x86_64: Enable interrupts in syscall

Kconfig

@@ -5,6 +5,8 @@ config KERNEL_FLAVOR
   default "vanilla"
 
 source "arch/$(ARCH)/Kconfig"
+source "boot/Kconfig"
 source "mem/Kconfig"
 source "sched/Kconfig"
 source "dev/Kconfig"
+source "tests/Kconfig"

arch/x86_64/Kconfig

@@ -4,6 +4,7 @@ config X86_64
   bool
   default y
   select ACPI
+  select HAVE_EFISTUB
   select HAVE_MMU
   select HAVE_MMU_GENERIC
   select HAVE_TLB_SHOOTDOWN_IPI

arch/x86_64/idt.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/irq.h>
@@ -101,12 +103,10 @@ void interrupt_handler(struct arch_context* ctx) {
         break;
     case IDT_TIMER:
         lapic_eoi(&arch_cpu->lapic);
+        arch_sched_preempt_disable();
         timer_interrupt();
-        if (cpu->sched.need_resched) {
-            arch_sched_preempt_disable();
-            if (arch_sched_preempt_enable())
-                sched_reschedule(&cpu->sched);
-        }
+        if (arch_sched_preempt_enable() && (cpu->sched.need_resched || cpu->sched.owe_preempt))
+            sched_reschedule(&cpu->sched);
         break;
     default:
         if (isr < 0x20)
@@ -114,7 +114,10 @@ void interrupt_handler(struct arch_context* ctx) {
         assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
         struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
         assert(line, "Unhandled interrupt on ISR %lu", isr);
+        arch_sched_preempt_disable();
         irq_line_raise(line);
+        if (arch_sched_preempt_enable() && cpu->sched.owe_preempt)
+            sched_reschedule(&cpu->sched);
     }
 }
 

arch/x86_64/include/bits/pmap.h

@@ -5,3 +5,20 @@
 #include <stdint.h>
 
 typedef uint64_t pte_t;
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};

arch/x86_64/include/x86_64/apic.h

@@ -2,6 +2,9 @@
 
 #pragma once
 
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
 #include <zinnia/mmio.h>
 #include <stdint.h>
 
@@ -81,3 +84,8 @@ void lapic_send_ipi(
 );
 
 void pic_disable();
+
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic);
+void apic_free_vector(uint8_t vector);
+
+INIT_EXTERN(ioapic_stage);

arch/x86_64/include/x86_64/defs.h

@@ -187,6 +187,7 @@ enum : uint64_t {
     CR4_OSFXSR = 1ull << 9,
     CR4_OSXMMEXCPT = 1ull << 10,
     CR4_UMIP = 1ull << 11,
+    CR4_LA57 = 1ull << 12,
     CR4_VMXE = 1ull << 13,
     CR4_SMXE = 1ull << 14,
     CR4_FSGSBASE = 1ull << 16,

arch/x86_64/include/x86_64/pmap.h

@@ -2,20 +2,3 @@
 
 #pragma once
 #include <zinnia/pmap.h>
-
-enum : pte_t {
-    ARCH_FLAG_PRESENT = 1 << 0,
-    ARCH_FLAG_READ_WRITE = 1 << 1,
-    ARCH_FLAG_USER_MODE = 1 << 2,
-    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
-    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
-    ARCH_FLAG_ACCESSED = 1 << 5,
-    ARCH_FLAG_DIRTY = 1 << 6,
-    ARCH_FLAG_SIZE = 1 << 7,
-    ARCH_FLAG_GLOBAL = 1 << 8,
-    ARCH_FLAG_AVAILABLE = 1 << 9,
-    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
-    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
-
-    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
-};

arch/x86_64/ioapic.c

@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "ioapic: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/percpu.h>
+
+constexpr size_t ISA_IRQS = 16;
+
+constexpr mmio_reg32 IOAPIC_REG_SELECT = {.offset = 0x00};
+constexpr mmio_reg32 IOAPIC_REG_WINDOW = {.offset = 0x10};
+
+constexpr uint32_t IOAPIC_VERSION = 0x01;
+constexpr uint32_t IOAPIC_REDIRECT = 0x10;
+
+constexpr uint64_t IOAPIC_RTE_POLARITY_LOW = 1 << 13;
+constexpr uint64_t IOAPIC_RTE_TRIGGER_LEVEL = 1 << 15;
+constexpr uint64_t IOAPIC_RTE_MASKED = 1 << 16;
+
+struct ioapic_line {
+    struct irq_line line;
+    struct ioapic* ioapic;
+    uint32_t pin;
+    uint32_t lapic;
+    uint8_t vector;
+};
+
+struct ioapic {
+    void* regs;
+    struct ioapic_line** lines;
+    struct spinlock lock;
+    uint32_t gsi_base;
+    uint32_t num_pins;
+};
+
+struct isa_override {
+    uint32_t gsi;
+    uint16_t flags;
+    bool present;
+};
+
+static struct ioapic* ioapics;
+static size_t num_ioapics;
+static struct isa_override isa_overrides[ISA_IRQS];
+static struct spinlock lines_lock;
+
+static uint32_t ioapic_read(struct ioapic* ioapic, uint32_t reg) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    return mmio_reg_read(ioapic->regs, IOAPIC_REG_WINDOW);
+}
+
+static void ioapic_write(struct ioapic* ioapic, uint32_t reg, uint32_t value) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_WINDOW, value);
+}
+
+static void write_rte(struct ioapic* ioapic, uint32_t pin, uint64_t rte) {
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, IOAPIC_RTE_MASKED);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2 + 1, rte >> 32);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, rte);
+}
+
+static void update_mask(struct ioapic_line* line, bool masked) {
+    struct ioapic* ioapic = line->ioapic;
+    spin_lock(&ioapic->lock);
+    uint32_t low = ioapic_read(ioapic, IOAPIC_REDIRECT + line->pin * 2);
+    low = masked ? low | IOAPIC_RTE_MASKED : low & ~IOAPIC_RTE_MASKED;
+    ioapic_write(ioapic, IOAPIC_REDIRECT + line->pin * 2, low);
+    spin_unlock(&ioapic->lock);
+}
+
+static enum irq_mode ioapic_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    struct ioapic_line* line = containerof(self, struct ioapic_line, line);
+    uint64_t rte = line->vector | ((uint64_t)line->lapic << 56);
+    if (polarity == IRQ_POLARITY_LOW)
+        rte |= IOAPIC_RTE_POLARITY_LOW;
+    if (trigger == IRQ_TRIGGER_LEVEL)
+        rte |= IOAPIC_RTE_TRIGGER_LEVEL;
+
+    spin_lock(&line->ioapic->lock);
+    write_rte(line->ioapic, line->pin, rte);
+    spin_unlock(&line->ioapic->lock);
+    return IRQ_MODE_EOI | IRQ_MODE_MASKABLE;
+}
+
+static void ioapic_mask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), true);
+}
+
+static void ioapic_unmask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), false);
+}
+
+static void ioapic_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_get_gsi(uint32_t gsi, struct irq_line** out) {
+    // Find the IOAPIC that handles this GSI.
+    struct ioapic* ioapic = nullptr;
+    for (size_t i = 0; i < num_ioapics; i++) {
+        if (gsi >= ioapics[i].gsi_base && gsi < ioapics[i].gsi_base + ioapics[i].num_pins) {
+            ioapic = &ioapics[i];
+            break;
+        }
+    }
+    if (!ioapic)
+        return ENOENT;
+    const uint32_t pin = gsi - ioapic->gsi_base;
+
+    spin_lock(&lines_lock);
+    struct ioapic_line* line = ioapic->lines[pin];
+    if (line) {
+        spin_unlock(&lines_lock);
+        *out = &line->line;
+        return 0;
+    }
+    spin_unlock(&lines_lock);
+
+    line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+    line->line.set_config = ioapic_set_config;
+    line->line.mask = ioapic_mask;
+    line->line.unmask = ioapic_unmask;
+    line->line.eoi = ioapic_eoi;
+    line->ioapic = ioapic;
+    line->pin = pin;
+
+    errno_t status = apic_alloc_vector(&line->line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
+
+    spin_lock(&lines_lock);
+    if (ioapic->lines[pin]) {
+        spin_unlock(&lines_lock);
+        apic_free_vector(line->vector);
+        mem_free(line);
+        return arch_irq_get_gsi(gsi, out);
+    }
+    ioapic->lines[pin] = line;
+    spin_unlock(&lines_lock);
+
+    *out = &line->line;
+    return 0;
+}
+
+bool arch_irq_isa_override(
+    uint8_t isa_irq,
+    uint32_t* out_gsi,
+    enum irq_trigger_mode* out_trigger,
+    enum irq_polarity* out_polarity
+) {
+    if (isa_irq >= ISA_IRQS || !isa_overrides[isa_irq].present)
+        return false;
+
+    const struct isa_override* iso = &isa_overrides[isa_irq];
+    *out_gsi = iso->gsi;
+    switch (iso->flags & ACPI_MADT_POLARITY_MASK) {
+    case ACPI_MADT_POLARITY_ACTIVE_HIGH:
+        *out_polarity = IRQ_POLARITY_HIGH;
+        break;
+    case ACPI_MADT_POLARITY_ACTIVE_LOW:
+        *out_polarity = IRQ_POLARITY_LOW;
+        break;
+    }
+    switch (iso->flags & ACPI_MADT_TRIGGERING_MASK) {
+    case ACPI_MADT_TRIGGERING_EDGE:
+        *out_trigger = IRQ_TRIGGER_EDGE;
+        break;
+    case ACPI_MADT_TRIGGERING_LEVEL:
+        *out_trigger = IRQ_TRIGGER_LEVEL;
+        break;
+    }
+    return true;
+}
+
+static uacpi_iteration_decision count_ioapics(uacpi_handle user, struct acpi_entry_hdr* hdr) {
+    size_t* count = user;
+    if (hdr->type == ACPI_MADT_ENTRY_TYPE_IOAPIC)
+        (*count)++;
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static uacpi_iteration_decision parse_madt_entry(uacpi_handle, struct acpi_entry_hdr* hdr) {
+    switch (hdr->type) {
+    case ACPI_MADT_ENTRY_TYPE_IOAPIC: {
+        // Map the IOAPIC and mask all of its pins.
+        const struct acpi_madt_ioapic* entry = (const struct acpi_madt_ioapic*)hdr;
+        uintptr_t vaddr = vm_alloc(arch_mem_page_size());
+        if (!vaddr)
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        pmap_map(&kernel_vma.pmap, vaddr, entry->address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+
+        struct ioapic* ioapic = &ioapics[num_ioapics];
+        ioapic->regs = (void*)vaddr;
+        ioapic->gsi_base = entry->gsi_base;
+        ioapic->num_pins = ((ioapic_read(ioapic, IOAPIC_VERSION) >> 16) & 0xFF) + 1;
+        spin_init(&ioapic->lock);
+        ioapic->lines = mem_alloc(ioapic->num_pins * sizeof(*ioapic->lines), 0);
+        if (!ioapic->lines) {
+            pmap_unmap(&kernel_vma.pmap, vaddr);
+            vm_free(vaddr, arch_mem_page_size());
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        }
+
+        for (uint32_t pin = 0; pin < ioapic->num_pins; pin++)
+            write_rte(ioapic, pin, IOAPIC_RTE_MASKED);
+
+        num_ioapics++;
+        log("IOAPIC %u at %#x handles GSIs %u-%u\n",
+            entry->id,
+            entry->address,
+            ioapic->gsi_base,
+            ioapic->gsi_base + ioapic->num_pins - 1);
+        break;
+    }
+    case ACPI_MADT_ENTRY_TYPE_INTERRUPT_SOURCE_OVERRIDE:
+        auto iso = (const struct acpi_madt_interrupt_source_override*)hdr;
+        if (iso->bus == 0 && iso->source < ISA_IRQS) {
+            isa_overrides[iso->source] = (struct isa_override){
+                .gsi = iso->gsi,
+                .flags = iso->flags,
+                .present = true,
+            };
+        }
+        break;
+    }
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static void ioapic_init(struct boot_info*) {
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log_warn("No MADT found\n");
+        return;
+    }
+
+    size_t capacity = 0;
+    uacpi_status status = uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), count_ioapics, &capacity);
+    if (status != UACPI_STATUS_OK)
+        log_warn("Failed to parse the MADT: %s\n", uacpi_status_to_string(status));
+
+    if (capacity) {
+        ioapics = mem_alloc(capacity * sizeof(*ioapics), 0);
+        if (!ioapics) {
+            log_err("Unable to allocate %zu controllers\n", capacity);
+            uacpi_table_unref(&table);
+            return;
+        }
+    }
+
+    uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), parse_madt_entry, nullptr);
+    uacpi_table_unref(&table);
+}
+INIT_TASK(ioapic_stage, "arch.x86_64.ioapic", ioapic_init);
+INIT_DEPENDS(ioapic_stage, acpi_stage, percpu_stage);
+INIT_ENTAILS(ioapic_stage, acpi_namespace_stage);

arch/x86_64/irq.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/percpu.h>
@@ -59,11 +61,7 @@ static void apic_msi_eoi(struct irq_line* self) {
     lapic_eoi(&percpu_ref(arch_data)->lapic);
 }
 
-errno_t arch_irq_alloc_msi(struct msi_line** out) {
-    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
-    if (!line)
-        return ENOMEM;
-
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic) {
     irq_lock();
     struct arch_percpu* arch_cpu = percpu_ref(arch_data);
     size_t slot;
@@ -73,20 +71,39 @@ errno_t arch_irq_alloc_msi(struct msi_line** out) {
     }
     if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
         irq_unlock();
-        mem_free(line);
         return ENOSPC;
     }
 
+    *out_vector = slot + 0x20;
+    *out_lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = line;
+    irq_unlock();
+    return 0;
+}
+
+void apic_free_vector(uint8_t vector) {
+    irq_lock();
+    percpu_ref(arch_data)->irq_lines[vector - 0x20] = nullptr;
+    irq_unlock();
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
     line->msi.line.set_config = apic_msi_set_config;
     line->msi.line.mask = apic_msi_mask;
     line->msi.line.unmask = apic_msi_unmask;
     line->msi.line.eoi = apic_msi_eoi;
     line->msi.msg_addr = apic_msi_msg_addr;
     line->msi.msg_data = apic_msi_msg_data;
-    line->vector = slot + 0x20;
-    line->lapic = arch_cpu->lapic.id;
-    arch_cpu->irq_lines[slot] = &line->msi.line;
-    irq_unlock();
+
+    errno_t status = apic_alloc_vector(&line->msi.line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
 
     *out = &line->msi;
     return 0;

arch/x86_64/meson.build

@@ -10,6 +10,7 @@ kernel_sources += files(
     'gdt.c',
     'hpet.c',
     'idt.c',
+    'ioapic.c',
     'irq.c',
     'mem.c',
     'percpu.c',

arch/x86_64/serial.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>

arch/x86_64/syscall.c

@@ -40,6 +40,7 @@ void arch_syscall_stub() {
         "    push r15\n"
         "    xor rbp, rbp\n"
         "    mov rdi, rsp\n"          // SysV ABI first argument.
+        "    sti\n"
         "    call syscall_dispatch\n" // Call syscall handler.
         "    cli\n"
         "    pop r15\n"

boot/Kconfig

@@ -0,0 +1,19 @@
+config HAVE_EFISTUB
+  bool
+
+menu "Boot loaders"
+
+config BOOT_EFISTUB
+  bool "EFI stub"
+  depends on HAVE_EFISTUB
+  default y
+  help
+    Kernel loader for UEFI systems.
+
+config BOOT_LIMINE
+  bool "Limine"
+  default n
+  help
+    ELF loader with the Limine protocol.
+
+endmenu

boot/efi/efi.h

@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+
+typedef uint64_t efi_status;
+typedef void* efi_handle;
+typedef uint16_t char16;
+
+constexpr efi_status EFI_SUCCESS = 0;
+constexpr efi_status EFI_ERROR = 1ull << 63;
+constexpr efi_status EFI_LOAD_ERROR = EFI_ERROR | 1;
+constexpr efi_status EFI_INVALID_PARAMETER = EFI_ERROR | 2;
+constexpr efi_status EFI_UNSUPPORTED = EFI_ERROR | 3;
+constexpr efi_status EFI_BUFFER_TOO_SMALL = EFI_ERROR | 5;
+constexpr efi_status EFI_OUT_OF_RESOURCES = EFI_ERROR | 9;
+constexpr efi_status EFI_NOT_FOUND = EFI_ERROR | 14;
+
+struct efi_guid {
+    uint32_t data1;
+    uint16_t data2;
+    uint16_t data3;
+    uint8_t data4[8];
+};
+
+enum efi_memory_type : uint32_t {
+    EFI_RESERVED_MEMORY_TYPE,
+    EFI_LOADER_CODE,
+    EFI_LOADER_DATA,
+    EFI_BOOT_SERVICES_CODE,
+    EFI_BOOT_SERVICES_DATA,
+    EFI_RUNTIME_SERVICES_CODE,
+    EFI_RUNTIME_SERVICES_DATA,
+    EFI_CONVENTIONAL_MEMORY,
+    EFI_UNUSABLE_MEMORY,
+    EFI_ACPI_RECLAIM_MEMORY,
+    EFI_ACPI_MEMORY_NVS,
+    EFI_MEMORY_MAPPED_IO,
+    EFI_MEMORY_MAPPED_IO_PORT_SPACE,
+    EFI_PAL_CODE,
+    EFI_PERSISTENT_MEMORY,
+    EFI_UNACCEPTED_MEMORY,
+};
+
+enum efi_allocate_type : uint32_t {
+    EFI_ALLOCATE_ANY_PAGES,
+    EFI_ALLOCATE_MAX_ADDRESS,
+    EFI_ALLOCATE_ADDRESS,
+};
+
+enum efi_locate_search_type : uint32_t {
+    EFI_LOCATE_ALL_HANDLES,
+    EFI_LOCATE_BY_REGISTER_NOTIFY,
+    EFI_LOCATE_BY_PROTOCOL,
+};
+
+struct efi_memory_descriptor {
+    uint32_t type;
+    uint64_t physical_start;
+    uint64_t virtual_start;
+    uint64_t number_of_pages;
+    uint64_t attribute;
+};
+
+struct efi_table_header {
+    uint64_t signature;
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t crc32;
+    uint32_t reserved;
+};
+
+struct efi_simple_text_output_protocol {
+    void* reset;
+    efi_status(EFIAPI* output_string)(struct efi_simple_text_output_protocol* self, const char16* string);
+};
+
+struct efi_boot_services {
+    struct efi_table_header hdr;
+    void* raise_tpl;
+    void* restore_tpl;
+    efi_status(EFIAPI* allocate_pages)(
+        enum efi_allocate_type type,
+        enum efi_memory_type memory_type,
+        size_t pages,
+        uint64_t* memory
+    );
+    efi_status(EFIAPI* free_pages)(uint64_t memory, size_t pages);
+    efi_status(EFIAPI* get_memory_map)(
+        size_t* map_size,
+        struct efi_memory_descriptor* map,
+        size_t* map_key,
+        size_t* descriptor_size,
+        uint32_t* descriptor_version
+    );
+    efi_status(EFIAPI* allocate_pool)(enum efi_memory_type pool_type, size_t size, void** buffer);
+    efi_status(EFIAPI* free_pool)(void* buffer);
+    void* create_event;
+    void* set_timer;
+    void* wait_for_event;
+    void* signal_event;
+    void* close_event;
+    void* check_event;
+    void* install_protocol_interface;
+    void* reinstall_protocol_interface;
+    void* uninstall_protocol_interface;
+    efi_status(EFIAPI* handle_protocol)(efi_handle handle, const struct efi_guid* protocol, void** interface);
+    void* reserved;
+    void* register_protocol_notify;
+    void* locate_handle;
+    void* locate_device_path;
+    void* install_configuration_table;
+    void* load_image;
+    void* start_image;
+    void* exit;
+    void* unload_image;
+    efi_status(EFIAPI* exit_boot_services)(efi_handle image_handle, size_t map_key);
+    void* get_next_monotonic_count;
+    void* stall;
+    void* set_watchdog_timer;
+    void* connect_controller;
+    void* disconnect_controller;
+    void* open_protocol;
+    void* close_protocol;
+    void* open_protocol_information;
+    void* protocols_per_handle;
+    efi_status(EFIAPI* locate_handle_buffer)(
+        enum efi_locate_search_type search_type,
+        const struct efi_guid* protocol,
+        void* search_key,
+        size_t* no_handles,
+        efi_handle** buffer
+    );
+    efi_status(EFIAPI* locate_protocol)(const struct efi_guid* protocol, void* registration, void** interface);
+};
+
+struct efi_configuration_table {
+    struct efi_guid vendor_guid;
+    void* vendor_table;
+};
+
+struct efi_system_table {
+    struct efi_table_header hdr;
+    char16* firmware_vendor;
+    uint32_t firmware_revision;
+    efi_handle console_in_handle;
+    void* con_in;
+    efi_handle console_out_handle;
+    struct efi_simple_text_output_protocol* con_out;
+    efi_handle standard_error_handle;
+    void* std_err;
+    void* runtime_services;
+    struct efi_boot_services* boot_services;
+    size_t number_of_table_entries;
+    struct efi_configuration_table* configuration_table;
+};
+
+struct efi_loaded_image_protocol {
+    uint32_t revision;
+    efi_handle parent_handle;
+    struct efi_system_table* system_table;
+    efi_handle device_handle;
+    void* file_path;
+    void* reserved;
+    uint32_t load_options_size;
+    void* load_options;
+    void* image_base;
+    uint64_t image_size;
+    enum efi_memory_type image_code_type;
+    enum efi_memory_type image_data_type;
+    void* unload;
+};
+
+constexpr uint64_t EFI_FILE_MODE_READ = 1;
+
+struct efi_file_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open)(
+        struct efi_file_protocol* self,
+        struct efi_file_protocol** new_handle,
+        const char16* file_name,
+        uint64_t open_mode,
+        uint64_t attributes
+    );
+    efi_status(EFIAPI* close)(struct efi_file_protocol* self);
+    void* delete;
+    efi_status(EFIAPI* read)(struct efi_file_protocol* self, size_t* buffer_size, void* buffer);
+    void* write;
+    efi_status(EFIAPI* get_position)(struct efi_file_protocol* self, uint64_t* position);
+    efi_status(EFIAPI* set_position)(struct efi_file_protocol* self, uint64_t position);
+};
+
+struct efi_simple_file_system_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open_volume)(struct efi_simple_file_system_protocol* self, struct efi_file_protocol** root);
+};
+
+enum efi_pixel_format : uint32_t {
+    EFI_PIXEL_RGBX_8,
+    EFI_PIXEL_BGRX_8,
+    EFI_PIXEL_BIT_MASK,
+    EFI_PIXEL_BLT_ONLY,
+};
+
+struct efi_pixel_bitmask {
+    uint32_t red;
+    uint32_t green;
+    uint32_t blue;
+    uint32_t reserved;
+};
+
+struct efi_gop_mode_info {
+    uint32_t version;
+    uint32_t horizontal_resolution;
+    uint32_t vertical_resolution;
+    enum efi_pixel_format pixel_format;
+    struct efi_pixel_bitmask pixel_information;
+    uint32_t pixels_per_scan_line;
+};
+
+struct efi_gop_mode {
+    uint32_t max_mode;
+    uint32_t mode;
+    struct efi_gop_mode_info* info;
+    size_t size_of_info;
+    uint64_t frame_buffer_base;
+    size_t frame_buffer_size;
+};
+
+struct efi_graphics_output_protocol {
+    void* query_mode;
+    void* set_mode;
+    void* blt;
+    struct efi_gop_mode* mode;
+};
+
+static const struct efi_guid EFI_LOADED_IMAGE_PROTOCOL_GUID =
+    {0x5b1b'31a1, 0x9562, 0x11d2, {0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID =
+    {0x964e'5b22, 0x6459, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID =
+    {0x9042'a9de, 0x23dc, 0x4a38, {0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a}};
+static const struct efi_guid EFI_ACPI_20_TABLE_GUID =
+    {0x8868'e871, 0xe4f1, 0x11d3, {0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81}};
+static const struct efi_guid EFI_ACPI_10_TABLE_GUID =
+    {0xeb9d'2d30, 0x2d88, 0x11d3, {0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d}};

boot/efi/header.c

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+#include "pe.h"
+
+extern const char __pe_code_size[];
+extern const char __pe_data_sizes[];
+extern const char __pe_entry[];
+extern const char __pe_image_sizes[];
+extern const char __pe_base_relocs[];
+extern const char __pe_text_virtual[];
+extern const char __pe_text_raw[];
+extern const char __pe_reloc_virtual[];
+extern const char __pe_reloc_raw[];
+extern const char __pe_rodata_virtual[];
+extern const char __pe_rodata_raw[];
+extern const char __pe_data_virtual[];
+extern const char __pe_data_raw[];
+
+struct efistub_pe_header {
+    struct pe_dos_header dos;
+    uint32_t signature;
+    struct pe_coff_header coff;
+    struct pe_optional_header64 optional;
+    struct pe_section_header sections[4];
+};
+
+static_assert(offsetof(struct efistub_pe_header, optional) % 8 == 0);
+static_assert(offsetof(struct efistub_pe_header, sections) % 8 == 0);
+
+[[__used, __section(".header")]]
+const struct efistub_pe_header efistub_pe_header = {
+    .dos =
+        {
+            .e_magic = PE_DOS_MAGIC,
+            .e_lfanew = offsetof(struct efistub_pe_header, signature),
+        },
+    .signature = PE_SIGNATURE,
+    .coff =
+        {
+            .machine = EFI_ARCH_PE_MACHINE,
+            .number_of_sections = 4,
+            .size_of_optional_header = sizeof(struct pe_optional_header64),
+            .characteristics = PE_FILE_EXECUTABLE_IMAGE | PE_FILE_LINE_NUMS_STRIPPED | PE_FILE_DEBUG_STRIPPED,
+        },
+    .optional =
+        {
+            .magic_and_code_size = (uintptr_t)__pe_code_size + PE_OPTIONAL_MAGIC_PE32_PLUS,
+            .data_sizes = (uintptr_t)__pe_data_sizes,
+            .entry_and_code_base = (uintptr_t)__pe_entry,
+            .section_alignment = 0x1000,
+            .file_alignment = 0x1000,
+            .image_and_header_sizes = (uintptr_t)__pe_image_sizes,
+            .subsystem = PE_SUBSYSTEM_EFI_APPLICATION,
+            .dll_characteristics = PE_DLL_NX_COMPAT,
+            .number_of_rva_and_sizes = PE_NUM_DATA_DIRECTORIES,
+            .data_directories =
+                {
+                    [PE_DIRECTORY_BASE_RELOC] = {.layout = (uintptr_t)__pe_base_relocs},
+                },
+        },
+    .sections = {
+        {
+            .name = ".text",
+            .virtual_layout = (uintptr_t)__pe_text_virtual,
+            .raw_layout = (uintptr_t)__pe_text_raw,
+            .characteristics = PE_SCN_CNT_CODE | PE_SCN_MEM_EXECUTE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".reloc",
+            .virtual_layout = (uintptr_t)__pe_reloc_virtual,
+            .raw_layout = (uintptr_t)__pe_reloc_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_DISCARDABLE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".rdata",
+            .virtual_layout = (uintptr_t)__pe_rodata_virtual,
+            .raw_layout = (uintptr_t)__pe_rodata_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".data",
+            .virtual_layout = (uintptr_t)__pe_data_virtual,
+            .raw_layout = (uintptr_t)__pe_data_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ | PE_SCN_MEM_WRITE,
+        },
+    },
+};
+
+[[__used, __section(".reloc")]]
+const struct pe_base_reloc_block efistub_base_relocs = {
+    .page_rva = 0,
+    .block_size = sizeof(struct pe_base_reloc_block),
+};

boot/efi/main.c

@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: ISC
+
+#include <string.h>
+#include "stub.h"
+
+constexpr size_t MEM_MAP_SLACK = 64;
+
+struct efi_boot_services* bs;
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    enum efi_memory_type efi_type = EFI_LOADER_DATA;
+    if (usage == PHYS_STATIC)
+        efi_type = MEM_STATIC;
+    else if (usage == PHYS_RECLAIMABLE)
+        efi_type = MEM_RECLAIMABLE;
+
+    const size_t pages = ALIGN_UP(size, PAGE_SIZE) / PAGE_SIZE;
+    if (bs->allocate_pages(EFI_ALLOCATE_ANY_PAGES, efi_type, pages, out))
+        return false;
+    memset((void*)*out, 0, pages * PAGE_SIZE);
+    return true;
+}
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)phys;
+}
+
+static efi_status open_file_on(efi_handle handle, const char16* path, struct efi_file_protocol** out) {
+    struct efi_simple_file_system_protocol* fs;
+    efi_status status = bs->handle_protocol(handle, &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID, (void**)&fs);
+    if (status)
+        return status;
+
+    struct efi_file_protocol* root;
+    status = fs->open_volume(fs, &root);
+    if (status)
+        return status;
+
+    status = root->open(root, out, path, EFI_FILE_MODE_READ, 0);
+    if (status && path[0] == '\\')
+        status = root->open(root, out, path + 1, EFI_FILE_MODE_READ, 0);
+    root->close(root);
+    return status;
+}
+
+static efi_status read_file(struct efi_file_protocol* file, enum phys_mem_usage usage, uint64_t* base, uint64_t* size) {
+    uint64_t total;
+    uint64_t done = 0;
+    efi_status status = file->set_position(file, UINT64_MAX);
+    if (!status)
+        status = file->get_position(file, &total);
+    if (!status)
+        status = file->set_position(file, 0);
+    if (!status && !loader_alloc(usage, total ? total : 1, base))
+        status = EFI_OUT_OF_RESOURCES;
+    while (!status && done < total) {
+        size_t chunk = total - done;
+        status = file->read(file, &chunk, (void*)(*base + done));
+        if (!chunk)
+            break;
+        done += chunk;
+    }
+    file->close(file);
+    *size = done;
+    return status;
+}
+
+static void color_from_mask(uint32_t mask, struct boot_color* out) {
+    if (!mask)
+        return;
+    out->shift = __builtin_ctz(mask);
+    out->size = __builtin_ctzll(~(uint64_t)(mask >> out->shift));
+}
+
+static enum phys_mem_usage convert_mem_type(uint32_t type) {
+    switch (type) {
+    case EFI_LOADER_CODE:
+    case EFI_LOADER_DATA:
+    case EFI_BOOT_SERVICES_CODE:
+    case EFI_BOOT_SERVICES_DATA:
+    case EFI_CONVENTIONAL_MEMORY:
+        return PHYS_USABLE;
+    case EFI_ACPI_RECLAIM_MEMORY:
+    case MEM_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case MEM_STATIC:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+efi_status efi_main(efi_handle image_handle, struct efi_system_table* st) {
+    bs = st->boot_services;
+
+    // Check if we support this configuration.
+    if (!arch_check_cpu())
+        return EFI_UNSUPPORTED;
+
+    struct efi_loaded_image_protocol* image;
+    efi_status status = bs->handle_protocol(image_handle, &EFI_LOADED_IMAGE_PROTOCOL_GUID, (void**)&image);
+    if (status)
+        return status;
+
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, "EFISTUB"))
+        return EFI_LOAD_ERROR;
+    struct boot_info* info = kernel.info;
+
+    // Reserve room for the memory map.
+    size_t map_size = 0;
+    size_t map_key;
+    size_t desc_size;
+    uint32_t desc_version;
+    status = bs->get_memory_map(&map_size, nullptr, &map_key, &desc_size, &desc_version);
+    if (status != EFI_BUFFER_TOO_SMALL)
+        return status;
+
+    const size_t map_capacity = map_size + MEM_MAP_SLACK * desc_size;
+    uint64_t map;
+    if (!loader_alloc(PHYS_USABLE, map_capacity, &map))
+        return EFI_OUT_OF_RESOURCES;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_USABLE, map_capacity / desc_size * sizeof(struct phys_mem), &mem))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Use the load options as the command line.
+    const char16* opts = image->load_options;
+    const size_t opts_len = image->load_options_size / sizeof(char16);
+    size_t cmdline_len = 0;
+    for (size_t i = 0; opts && i < opts_len && opts[i]; i++) {
+        if (opts[i] < 0x20 || opts[i] > 0x7e || cmdline_len + 1 >= sizeof(info->cmdline)) {
+            cmdline_len = 0;
+            break;
+        }
+        info->cmdline[cmdline_len++] = (char)opts[i];
+    }
+    info->cmdline[cmdline_len] = 0;
+
+    // Without load options, fall back to a command line file on the boot device.
+    struct efi_file_protocol* cmdline_file;
+    if (!cmdline_len && !open_file_on(image->device_handle, u"\\EFI\\zinnia\\cmdline", &cmdline_file)) {
+        uint64_t base;
+        uint64_t size;
+        status = read_file(cmdline_file, PHYS_USABLE, &base, &size);
+        if (status)
+            return status;
+
+        const char* text = (const char*)base;
+        while (cmdline_len < size && text[cmdline_len] && text[cmdline_len] != '\n' && text[cmdline_len] != '\r') {
+            if (text[cmdline_len] < 0x20 || text[cmdline_len] > 0x7e)
+                return EFI_INVALID_PARAMETER;
+            if (cmdline_len + 1 >= sizeof(info->cmdline))
+                return EFI_BUFFER_TOO_SMALL;
+            info->cmdline[cmdline_len] = text[cmdline_len];
+            cmdline_len++;
+        }
+        while (cmdline_len && info->cmdline[cmdline_len - 1] == ' ')
+            cmdline_len--;
+        info->cmdline[cmdline_len] = 0;
+    }
+
+    // Load every initrd= file.
+    const char* cmdline = info->cmdline;
+    for (size_t i = 0; cmdline[i];) {
+        if (cmdline[i] == ' ') {
+            i++;
+            continue;
+        }
+
+        const size_t start = i;
+        bool quoted = false;
+        while (cmdline[i] && (quoted || cmdline[i] != ' ')) {
+            if (cmdline[i] == '"')
+                quoted = !quoted;
+            i++;
+        }
+        const char* token = cmdline + start;
+        const size_t len = i - start;
+        if (len <= 7 || memcmp(token, "initrd=", 7))
+            continue;
+
+        const size_t path_len = len - 7;
+        if (path_len >= BOOT_PATH_MAX)
+            return EFI_INVALID_PARAMETER;
+        char path[BOOT_PATH_MAX];
+        char16 wide[BOOT_PATH_MAX];
+        for (size_t j = 0; j < path_len; j++) {
+            path[j] = token[7 + j];
+            wide[j] = path[j] == '/' ? '\\' : (char16)path[j];
+        }
+        path[path_len] = 0;
+        wide[path_len] = 0;
+
+        // Try the boot device first, then all other file systems.
+        struct efi_file_protocol* file;
+        status = open_file_on(image->device_handle, wide, &file);
+        size_t num_handles;
+        efi_handle* handles;
+        if (status &&
+            !bs->locate_handle_buffer(
+                EFI_LOCATE_BY_PROTOCOL,
+                &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID,
+                nullptr,
+                &num_handles,
+                &handles
+            )) {
+            for (size_t j = 0; j < num_handles && status; j++) {
+                if (handles[j] != image->device_handle)
+                    status = open_file_on(handles[j], wide, &file);
+            }
+            bs->free_pool(handles);
+        }
+        if (status)
+            return status;
+
+        uint64_t base;
+        uint64_t size;
+        status = read_file(file, PHYS_STATIC, &base, &size);
+        if (status)
+            return status;
+
+        if (!loader_add_file(&kernel, path, base, size))
+            return EFI_OUT_OF_RESOURCES;
+    }
+
+    // Framebuffer
+    struct efi_graphics_output_protocol* gop;
+    if (!bs->locate_protocol(&EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, nullptr, (void**)&gop) && gop && gop->mode &&
+        gop->mode->info) {
+        const struct efi_gop_mode_info* mode = gop->mode->info;
+        struct boot_framebuffer* fb = &info->fb;
+        switch (mode->pixel_format) {
+        case EFI_PIXEL_RGBX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){0, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){16, 8};
+            break;
+        case EFI_PIXEL_BGRX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){16, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){0, 8};
+            break;
+        case EFI_PIXEL_BIT_MASK: {
+            const struct efi_pixel_bitmask* m = &mode->pixel_information;
+            const uint32_t all = m->red | m->green | m->blue | m->reserved;
+            if (!all)
+                break;
+            fb->bytes_per_pixel = (32 - __builtin_clz(all) + 7) / 8;
+            color_from_mask(m->red, &fb->red);
+            color_from_mask(m->green, &fb->green);
+            color_from_mask(m->blue, &fb->blue);
+            break;
+        }
+        default:
+            break;
+        }
+
+        if (fb->bytes_per_pixel) {
+            fb->base = gop->mode->frame_buffer_base;
+            fb->width = mode->horizontal_resolution;
+            fb->height = mode->vertical_resolution;
+            fb->pitch = mode->pixels_per_scan_line * fb->bytes_per_pixel;
+        }
+    }
+
+    // Find the RSDP.
+    for (size_t i = 0; i < st->number_of_table_entries; i++) {
+        const struct efi_configuration_table* table = &st->configuration_table[i];
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_20_TABLE_GUID, sizeof(struct efi_guid))) {
+            info->rsdp = (uint64_t)table->vendor_table;
+            break;
+        }
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_10_TABLE_GUID, sizeof(struct efi_guid)))
+            info->rsdp = (uint64_t)table->vendor_table;
+    }
+
+    // Map everything up to the highest non-reserved address.
+    map_size = map_capacity;
+    status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+    if (status)
+        return status;
+
+    uint64_t top = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        if (convert_mem_type(desc->type) != PHYS_RESERVED)
+            top = MAX(top, desc->physical_start + desc->number_of_pages * PAGE_SIZE);
+    }
+
+    uint64_t root;
+    if (!loader_page_tables(&kernel, top, (uint64_t)arch_enter_kernel, &root))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Exit boot services.
+    for (size_t attempt = 0;; attempt++) {
+        map_size = map_capacity;
+        status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+        if (status)
+            return status;
+        status = bs->exit_boot_services(image_handle, map_key);
+        if (!status)
+            break;
+        if (attempt == 8)
+            return status;
+    }
+
+    // Hand the final memory map to the kernel.
+    struct phys_mem* mem_map = (struct phys_mem*)mem;
+    size_t num_mem = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        mem_map[num_mem++] = (struct phys_mem){
+            .address = desc->physical_start,
+            .length = desc->number_of_pages * PAGE_SIZE,
+            .usage = convert_mem_type(desc->type),
+        };
+    }
+    loader_set_mem_map(&kernel, mem_map, num_mem);
+
+    arch_enter_kernel(root, kernel.entry);
+}

boot/efi/meson.build

@@ -0,0 +1,42 @@
+efistub_sources = files(
+    'header.c',
+    'main.c',
+    'reloc.c',
+)
+
+efistub_lds = meson.current_source_dir() / 'stub.ld'
+
+subdir(target_machine.cpu_family())
+
+efistub_elf = executable(
+    'efistub.elf',
+    sources: [loader_sources, efistub_sources, loader_kernel],
+    include_directories: [
+        common_includes,
+        loader_includes,
+        include_directories('.', target_machine.cpu_family()),
+    ],
+    c_args: loader_c_args,
+    link_args: [
+        '-nostdlib',
+        '-Wl,-T,' + efistub_lds,
+        '-Wl,--no-dynamic-linker',
+        '-Wl,-z,text',
+        '-Wl,-z,norelro',
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: efistub_lds,
+    pie: true,
+)
+
+zinnia_efi = custom_target(
+    'zinnia.efi',
+    input: efistub_elf,
+    output: 'zinnia.efi',
+    command: [objcopy, '-O', 'binary', '@INPUT@', '@OUTPUT@'],
+    build_by_default: true,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/efi/pe.h

@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint16_t PE_DOS_MAGIC = 0x5a4d;
+constexpr uint32_t PE_SIGNATURE = 0x0000'4550;
+constexpr uint16_t PE_OPTIONAL_MAGIC_PE32_PLUS = 0x020b;
+constexpr uint16_t PE_SUBSYSTEM_EFI_APPLICATION = 10;
+constexpr size_t PE_NUM_DATA_DIRECTORIES = 16;
+constexpr size_t PE_DIRECTORY_BASE_RELOC = 5;
+
+enum pe_file_characteristics : uint16_t {
+    PE_FILE_EXECUTABLE_IMAGE = 0x0002,
+    PE_FILE_LINE_NUMS_STRIPPED = 0x0004,
+    PE_FILE_DEBUG_STRIPPED = 0x0200,
+};
+
+enum pe_dll_characteristics : uint16_t {
+    PE_DLL_NX_COMPAT = 0x0100,
+};
+
+enum pe_section_characteristics : uint32_t {
+    PE_SCN_CNT_CODE = 0x0000'0020,
+    PE_SCN_CNT_INITIALIZED_DATA = 0x0000'0040,
+    PE_SCN_MEM_DISCARDABLE = 0x0200'0000,
+    PE_SCN_MEM_EXECUTE = 0x2000'0000,
+    PE_SCN_MEM_READ = 0x4000'0000,
+    PE_SCN_MEM_WRITE = 0x8000'0000,
+};
+
+struct pe_dos_header {
+    uint16_t e_magic;
+    uint16_t e_cblp;
+    uint16_t e_cp;
+    uint16_t e_crlc;
+    uint16_t e_cparhdr;
+    uint16_t e_minalloc;
+    uint16_t e_maxalloc;
+    uint16_t e_ss;
+    uint16_t e_sp;
+    uint16_t e_csum;
+    uint16_t e_ip;
+    uint16_t e_cs;
+    uint16_t e_lfarlc;
+    uint16_t e_ovno;
+    uint16_t e_res[4];
+    uint16_t e_oemid;
+    uint16_t e_oeminfo;
+    uint16_t e_res2[10];
+    uint32_t e_lfanew;
+};
+
+struct pe_coff_header {
+    uint16_t machine;
+    uint16_t number_of_sections;
+    uint32_t time_date_stamp;
+    uint32_t pointer_to_symbol_table;
+    uint32_t number_of_symbols;
+    uint16_t size_of_optional_header;
+    uint16_t characteristics;
+};
+
+struct pe_data_directory {
+    union {
+        struct {
+            uint32_t virtual_address;
+            uint32_t size;
+        };
+        uint64_t layout;
+    };
+};
+
+struct pe_optional_header64 {
+    union {
+        struct {
+            uint16_t magic;
+            uint8_t major_linker_version;
+            uint8_t minor_linker_version;
+            uint32_t size_of_code;
+        };
+        uint64_t magic_and_code_size;
+    };
+    union {
+        struct {
+            uint32_t size_of_initialized_data;
+            uint32_t size_of_uninitialized_data;
+        };
+        uint64_t data_sizes;
+    };
+    union {
+        struct {
+            uint32_t address_of_entry_point;
+            uint32_t base_of_code;
+        };
+        uint64_t entry_and_code_base;
+    };
+    uint64_t image_base;
+    uint32_t section_alignment;
+    uint32_t file_alignment;
+    uint16_t major_operating_system_version;
+    uint16_t minor_operating_system_version;
+    uint16_t major_image_version;
+    uint16_t minor_image_version;
+    uint16_t major_subsystem_version;
+    uint16_t minor_subsystem_version;
+    uint32_t win32_version_value;
+    union {
+        struct {
+            uint32_t size_of_image;
+            uint32_t size_of_headers;
+        };
+        uint64_t image_and_header_sizes;
+    };
+    uint32_t checksum;
+    uint16_t subsystem;
+    uint16_t dll_characteristics;
+    uint64_t size_of_stack_reserve;
+    uint64_t size_of_stack_commit;
+    uint64_t size_of_heap_reserve;
+    uint64_t size_of_heap_commit;
+    uint32_t loader_flags;
+    uint32_t number_of_rva_and_sizes;
+    struct pe_data_directory data_directories[PE_NUM_DATA_DIRECTORIES];
+};
+
+struct pe_section_header {
+    char name[8];
+    union {
+        struct {
+            uint32_t virtual_size;
+            uint32_t virtual_address;
+        };
+        uint64_t virtual_layout;
+    };
+    union {
+        struct {
+            uint32_t size_of_raw_data;
+            uint32_t pointer_to_raw_data;
+        };
+        uint64_t raw_layout;
+    };
+    uint32_t pointer_to_relocations;
+    uint32_t pointer_to_line_numbers;
+    uint16_t number_of_relocations;
+    uint16_t number_of_line_numbers;
+    uint32_t characteristics;
+};
+
+struct pe_base_reloc_block {
+    uint32_t page_rva;
+    uint32_t block_size;
+    uint16_t entries[2];
+};
+
+static_assert(sizeof(struct pe_dos_header) == 64);
+static_assert(sizeof(struct pe_coff_header) == 20);
+static_assert(sizeof(struct pe_optional_header64) == 240);
+static_assert(offsetof(struct pe_optional_header64, size_of_image) == 56);
+static_assert(offsetof(struct pe_optional_header64, data_directories) == 112);
+static_assert(sizeof(struct pe_section_header) == 40);
+static_assert(sizeof(struct pe_base_reloc_block) == 12);

boot/efi/reloc.c

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+
+efi_status efistub_relocate(uintptr_t base, const elf64_dyn_t* dynamic) {
+    uintptr_t rela = 0;
+    size_t rela_size = 0;
+    size_t rela_ent = sizeof(elf64_rela_t);
+    uintptr_t relr = 0;
+    size_t relr_size = 0;
+
+    for (const elf64_dyn_t* dyn = dynamic; dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = base + dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        case DT_RELR:
+            relr = base + dyn->d_un.d_val;
+            break;
+        case DT_RELRSZ:
+            relr_size = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    for (size_t off = 0; rela && off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = (const elf64_rela_t*)(rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return EFI_LOAD_ERROR;
+        *(uint64_t*)(base + r->r_offset) = base + r->r_addend;
+    }
+
+    uint64_t* where = nullptr;
+    for (size_t off = 0; relr && off + sizeof(uint64_t) <= relr_size; off += sizeof(uint64_t)) {
+        uint64_t entry = *(const uint64_t*)(relr + off);
+        if (!(entry & 1)) {
+            where = (uint64_t*)(base + entry);
+            *where++ += base;
+            continue;
+        }
+        for (size_t i = 0; (entry >>= 1) != 0; i++) {
+            if (entry & 1)
+                where[i] += base;
+        }
+        where += 63;
+    }
+
+    return EFI_SUCCESS;
+}

boot/efi/stub.h

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+#include "loader.h"
+
+constexpr enum efi_memory_type MEM_STATIC = (enum efi_memory_type)0x8000'0000;
+constexpr enum efi_memory_type MEM_RECLAIMABLE = (enum efi_memory_type)0x8000'0001;
+
+extern struct efi_boot_services* bs;

boot/efi/stub.ld

@@ -0,0 +1,100 @@
+ENTRY(_start)
+
+PHDRS
+{
+    text    PT_LOAD    FLAGS(5);
+    rodata  PT_LOAD    FLAGS(4);
+    data    PT_LOAD    FLAGS(6);
+    dynamic PT_DYNAMIC FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0;
+
+    .header : {
+        __image_base = .;
+        KEEP(*(.header))
+    } :text
+
+    . = ALIGN(0x1000);
+    .text : {
+        __text_start = .;
+        *(.text .text.*)
+        . = ALIGN(0x1000);
+        __text_end = .;
+    } :text
+
+    .reloc : {
+        __reloc_start = .;
+        KEEP(*(.reloc))
+        __reloc_data_end = .;
+        . = ALIGN(0x1000);
+        __reloc_end = .;
+    } :rodata
+
+    .rodata : {
+        __rodata_start = .;
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    .hash : { *(.hash) } :rodata
+    .gnu.hash : { *(.gnu.hash) } :rodata
+    .dynsym : { *(.dynsym) } :rodata
+    .dynstr : { *(.dynstr) } :rodata
+    .rela.dyn : { *(.rela.*) } :rodata
+    .relr.dyn : { *(.relr.dyn) } :rodata
+
+    .rodata.end : {
+        . = ALIGN(0x1000);
+        __rodata_end = .;
+    } :rodata
+
+    .dynamic : {
+        __data_start = .;
+        *(.dynamic)
+    } :data :dynamic
+
+    .data : {
+        *(.data .data.*)
+        *(.got .got.*)
+        . = ALIGN(0x1000);
+        __data_raw_end = .;
+    } :data
+
+    .bss (NOLOAD) : {
+        __bss_start = .;
+        *(.bss .bss.*)
+        *(COMMON)
+        . = ALIGN(0x1000);
+        __bss_end = .;
+    } :data
+
+    __image_end = .;
+
+    __pe_code_size = ABSOLUTE((__text_end - __text_start) << 32);
+    __pe_data_sizes = ABSOLUTE(((__reloc_end - __reloc_start) + (__rodata_end - __rodata_start) +
+                                (__data_raw_end - __data_start)) | ((__bss_end - __bss_start) << 32));
+    __pe_entry = ABSOLUTE(_start | (__text_start << 32));
+    __pe_image_sizes = ABSOLUTE(__image_end | (__text_start << 32));
+    __pe_base_relocs = ABSOLUTE(__reloc_start | ((__reloc_data_end - __reloc_start) << 32));
+    __pe_text_virtual = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_text_raw = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_reloc_virtual = ABSOLUTE((__reloc_data_end - __reloc_start) | (__reloc_start << 32));
+    __pe_reloc_raw = ABSOLUTE((__reloc_end - __reloc_start) | (__reloc_start << 32));
+    __pe_rodata_virtual = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_rodata_raw = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_data_virtual = ABSOLUTE((__bss_end - __data_start) | (__data_start << 32));
+    __pe_data_raw = ABSOLUTE((__data_raw_end - __data_start) | (__data_start << 32));
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/efi/x86_64/efi_arch.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+#define EFIAPI __attribute__((ms_abi))
+
+constexpr uint16_t EFI_ARCH_PE_MACHINE = 0x8664;

boot/efi/x86_64/entry.S

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl _start
+    .type _start, @function
+_start:
+    // Align the stack pointer
+    push rbp
+    mov rbp, rsp
+    push rdi
+    push rsi
+    push rbx
+    push r12
+    and rsp, -16
+    cld
+
+    mov rbx, rcx
+    mov r12, rdx
+
+    // Clear .bss
+    lea rdi, [rip + __bss_start]
+    lea rcx, [rip + __bss_end]
+    sub rcx, rdi
+    xor eax, eax
+    rep stosb
+
+    // Self-relocate
+    lea rdi, [rip + __image_base]
+    lea rsi, [rip + _DYNAMIC]
+    call efistub_relocate
+    test rax, rax
+    jnz 1f
+
+    mov rdi, rbx
+    mov rsi, r12
+    call efi_main
+
+1:
+    lea rsp, [rbp - 32]
+    pop r12
+    pop rbx
+    pop rsi
+    pop rdi
+    pop rbp
+    ret
+    .size _start, . - _start

boot/efi/x86_64/meson.build

@@ -0,0 +1 @@
+efistub_sources += files('entry.S')

boot/kernel.ld

@@ -7,6 +7,7 @@ PHDRS {
     dynamic PT_DYNAMIC  FLAGS(6);
     percpu  PT_LOAD     FLAGS(6);
     cmdline 0x60000005;
+    boot    0x60000008;
 }
 
 SECTIONS
@@ -49,6 +50,11 @@ SECTIONS
         KEEP(*(.initgraph.edges))
         __ld_initgraph_edges_end = .;
 
+        . = ALIGN(8);
+        __ld_ktest_start = .;
+        KEEP(*(.ktest))
+        __ld_ktest_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -62,6 +68,10 @@ SECTIONS
         *(.data .data.*)
     } :data
 
+    .boot : {
+        KEEP(*(.boot.info))
+    } :data :boot
+
     .dynsym : {
         __ld_dynsym_start = .;
         KEEP(*(.dynsym*))

boot/limine/entry.c

@@ -1,135 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/cmdline.h>
-#include <zinnia/fdt.h>
-#include <zinnia/init.h>
-#include <zinnia/print.h>
-#include <zinnia/utils.h>
-#include <string.h>
-#include "limine.h"
-
-[[__initdata_sorted("limine.0")]]
-static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
-[[__initdata_sorted("limine.1")]]
-static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
-[[__initdata_sorted("limine.2")]]
-static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
-
-#define LIMINE_REQUEST(request, tag, rev) \
-    [[__initdata_sorted("limine.1")]] \
-    static volatile struct limine_##request request = { \
-        .id = tag, \
-        .revision = rev, \
-        .response = nullptr, \
-    }
-
-LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
-LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
-LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
-LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
-LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
-LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
-LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
-LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
-
-[[__initdata]]
-static struct phys_mem mem[128];
-[[__initdata]]
-static struct boot_file files[8];
-
-[[__init]]
-void kernel_entry() {
-    kernel_early_init();
-
-    struct boot_info info = {};
-
-    const struct limine_memmap_response* mm_res = memmap_request.response;
-    const struct limine_hhdm_response* hhdm_res = hhdm_request.response;
-    const struct limine_module_response* module_res = module_request.response;
-    const struct limine_executable_address_response* exec_res = executable_address_request.response;
-    const struct limine_executable_file_response* exec_file_res = executable_file_request.response;
-    const struct limine_rsdp_response* rsdp_res = rsdp_request.response;
-
-    assert(mm_res, "memmap_request returned no response!");
-    assert(hhdm_res, "hhdm_request returned no response!");
-
-    info.cmdline = exec_file_res->executable_file->string;
-    info.kernel_file.data = (phys_t)((uintptr_t)exec_file_res->executable_file->address - hhdm_res->offset);
-    info.kernel_file.length = exec_file_res->executable_file->size;
-
-    info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
-    for (size_t i = 0; i < info.num_mem_maps; i++) {
-        mem[i].address = mm_res->entries[i]->base;
-        mem[i].length = mm_res->entries[i]->length;
-
-        switch (mm_res->entries[i]->type) {
-        case LIMINE_MEMMAP_USABLE:
-            mem[i].usage = PHYS_USABLE;
-            break;
-        case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
-        case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
-            mem[i].usage = PHYS_RECLAIMABLE;
-            break;
-        case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
-            mem[i].usage = PHYS_STATIC;
-            break;
-        default:
-            mem[i].usage = PHYS_RESERVED;
-            break;
-        }
-    }
-
-    if (module_res) {
-        info.files = files;
-        info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
-        for (size_t i = 0; i < info.num_files; i++) {
-            files[i].data = (phys_t)module_res->modules[i]->address - hhdm_res->offset;
-            files[i].length = module_res->modules[i]->size;
-            strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1);
-            files[i].path[sizeof(files[i].path) - 1] = 0;
-        }
-    }
-
-    info.mem_map = mem;
-    info.phys_base = (phys_t)exec_res->physical_base;
-    info.virt_base = (uintptr_t)exec_res->virtual_base;
-    info.hhdm_base = (uintptr_t)hhdm_res->offset;
-
-    const struct limine_dtb_response* dtb_res = dtb_request.response;
-    if (dtb_res && dtb_res->dtb_ptr) {
-        info.dtb = (phys_t)((uintptr_t)dtb_res->dtb_ptr - hhdm_res->offset);
-        info.dtb_size = fdt_read_u32((const uint8_t*)dtb_res->dtb_ptr + 4);
-    }
-
-    const struct limine_framebuffer_response* fb_res = framebuffer_request.response;
-    if (fb_res) {
-        // Only use the first framebuffer.
-        if (fb_res->framebuffer_count >= 1) {
-            const struct limine_framebuffer* fb = framebuffer_request.response->framebuffers[0];
-            info.fb.base = (phys_t)((uintptr_t)fb->address - hhdm_res->offset);
-            info.fb.width = fb->width;
-            info.fb.height = fb->height;
-            info.fb.pitch = fb->pitch;
-            info.fb.bytes_per_pixel = fb->bpp / 8;
-
-            info.fb.red.shift = fb->red_mask_shift;
-            info.fb.red.size = fb->red_mask_size;
-
-            info.fb.green.shift = fb->green_mask_shift;
-            info.fb.green.size = fb->green_mask_size;
-
-            info.fb.blue.shift = fb->blue_mask_shift;
-            info.fb.blue.size = fb->blue_mask_size;
-        }
-    }
-
-#if CONFIG_ACPI
-    if (rsdp_request.response) {
-        info.rsdp = (phys_t)rsdp_request.response->address;
-        if (info.rsdp >= info.hhdm_base)
-            info.rsdp -= info.hhdm_base;
-    }
-#endif
-
-    kernel_main(&info);
-}

boot/limine/loader.ld

@@ -0,0 +1,46 @@
+ENTRY(loader_main)
+
+PHDRS
+{
+    text    PT_LOAD FLAGS(5);
+    rodata  PT_LOAD FLAGS(4);
+    data    PT_LOAD FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0xffffffffc0000000;
+
+    .text : {
+        *(.text .text.*)
+    } :text
+
+    . = ALIGN(0x1000);
+    .rodata : {
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    . = ALIGN(0x1000);
+    .data : {
+        KEEP(*(.limine_requests_start))
+        KEEP(*(.limine_requests))
+        KEEP(*(.limine_requests_end))
+        *(.data .data.*)
+    } :data
+
+    .bss : {
+        *(.bss .bss.*)
+        *(COMMON)
+    } :data
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/limine/main.c

@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <string.h>
+#include "limine.h"
+#include "loader.h"
+
+[[__used, __section(".limine_requests_start")]]
+static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
+[[__used, __section(".limine_requests")]]
+static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
+[[__used, __section(".limine_requests_end")]]
+static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
+
+#define LIMINE_REQUEST(request, tag, rev) \
+    [[__used, __section(".limine_requests")]] \
+    static volatile struct limine_##request request = { \
+        .id = tag, \
+        .revision = rev, \
+        .response = nullptr, \
+    }
+
+LIMINE_REQUEST(bootloader_info_request, LIMINE_BOOTLOADER_INFO_REQUEST_ID, 0);
+LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
+LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
+LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
+LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
+LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
+LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
+
+static uint64_t hhdm_offset;
+static const struct limine_memmap_entry* pool;
+static uint64_t pool_static_end;
+static uint64_t pool_reclaimable_start;
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)(phys + hhdm_offset);
+}
+
+static uint64_t to_phys(const void* ptr) {
+    const uint64_t addr = (uint64_t)ptr;
+    return addr >= hhdm_offset ? addr - hhdm_offset : addr;
+}
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    size = ALIGN_UP(size, PAGE_SIZE);
+    if (pool_reclaimable_start - pool_static_end < size)
+        return false;
+
+    if (usage == PHYS_STATIC) {
+        *out = pool_static_end;
+        pool_static_end += size;
+    } else {
+        pool_reclaimable_start -= size;
+        *out = pool_reclaimable_start;
+    }
+    memset(loader_ptr(*out), 0, size);
+    return true;
+}
+
+static enum phys_mem_usage convert_mem_type(uint64_t type) {
+    switch (type) {
+    case LIMINE_MEMMAP_USABLE:
+        return PHYS_USABLE;
+    case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
+    case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+static bool init_pool(const struct limine_memmap_response* memmap) {
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry->type == LIMINE_MEMMAP_USABLE && (!pool || entry->length > pool->length))
+            pool = entry;
+    }
+    if (!pool)
+        return false;
+
+    pool_static_end = pool->base;
+    pool_reclaimable_start = pool->base + pool->length;
+    return true;
+}
+
+static uint64_t mem_map_top(const struct limine_memmap_response* memmap) {
+    uint64_t top = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (convert_mem_type(entry->type) != PHYS_RESERVED)
+            top = MAX(top, entry->base + entry->length);
+    }
+    return top;
+}
+
+static size_t convert_mem_map(const struct limine_memmap_response* memmap, struct phys_mem* out) {
+    size_t n = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry != pool) {
+            out[n++] = (struct phys_mem){entry->base, entry->length, convert_mem_type(entry->type)};
+            continue;
+        }
+
+        const uint64_t end = entry->base + entry->length;
+        out[n++] = (struct phys_mem){entry->base, pool_static_end - entry->base, PHYS_STATIC};
+        out[n++] = (struct phys_mem){pool_static_end, pool_reclaimable_start - pool_static_end, PHYS_USABLE};
+        out[n++] = (struct phys_mem){pool_reclaimable_start, end - pool_reclaimable_start, PHYS_RECLAIMABLE};
+    }
+    return n;
+}
+
+static void find_framebuffer(struct boot_framebuffer* out) {
+    const struct limine_framebuffer_response* res = framebuffer_request.response;
+    if (!res || !res->framebuffer_count)
+        return;
+
+    const struct limine_framebuffer* fb = res->framebuffers[0];
+    out->base = to_phys(fb->address);
+    out->width = fb->width;
+    out->height = fb->height;
+    out->pitch = fb->pitch;
+    out->bytes_per_pixel = fb->bpp / 8;
+    out->red = (struct boot_color){fb->red_mask_shift, fb->red_mask_size};
+    out->green = (struct boot_color){fb->green_mask_shift, fb->green_mask_size};
+    out->blue = (struct boot_color){fb->blue_mask_shift, fb->blue_mask_size};
+}
+
+static bool load() {
+    const struct limine_memmap_response* memmap = memmap_request.response;
+    const struct limine_executable_address_response* exec = executable_address_request.response;
+    if (!LIMINE_BASE_REVISION_SUPPORTED(limine_revision) || !memmap || !hhdm_request.response || !exec)
+        return false;
+    hhdm_offset = hhdm_request.response->offset;
+
+    if (!arch_check_cpu() || !init_pool(memmap))
+        return false;
+
+    const struct limine_bootloader_info_response* info = bootloader_info_request.response;
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, info ? info->name : "Limine"))
+        return false;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_RECLAIMABLE, (memmap->entry_count + 2) * sizeof(struct phys_mem), &mem))
+        return false;
+
+    const struct limine_executable_file_response* file = executable_file_request.response;
+    if (file && file->executable_file->string) {
+        const char* cmdline = file->executable_file->string;
+        memcpy(kernel.info->cmdline, cmdline, MIN(strlen(cmdline), sizeof(kernel.info->cmdline) - 1));
+    }
+
+    const struct limine_module_response* modules = module_request.response;
+    for (size_t i = 0; modules && i < modules->module_count; i++) {
+        const struct limine_file* module = modules->modules[i];
+        if (!loader_add_file(&kernel, module->path, to_phys(module->address), module->size))
+            return false;
+    }
+
+    find_framebuffer(&kernel.info->fb);
+    if (rsdp_request.response)
+        kernel.info->rsdp = to_phys(rsdp_request.response->address);
+    if (dtb_request.response && dtb_request.response->dtb_ptr)
+        kernel.info->dtb = to_phys(dtb_request.response->dtb_ptr);
+
+    const uint64_t trampoline = (uint64_t)arch_enter_kernel;
+    uint64_t root;
+    if (!loader_page_tables(
+            &kernel,
+            mem_map_top(memmap),
+            exec->physical_base + (trampoline - exec->virtual_base),
+            &root
+        ))
+        return false;
+
+    struct phys_mem* mem_map = loader_ptr(mem);
+    loader_set_mem_map(&kernel, mem_map, convert_mem_map(memmap, mem_map));
+    arch_enter_kernel(root, kernel.entry);
+}
+
+[[noreturn]]
+void loader_main() {
+    load();
+    for (;;)
+        asm volatile("");
+}

boot/limine/meson.build

@@ -1 +1,25 @@
-kernel_sources += files('entry.c')
+limine_lds = meson.current_source_dir() / 'loader.ld'
+
+limine_c_args = [loader_c_args, '-fno-pic']
+if target_machine.cpu_family() == 'x86_64'
+    limine_c_args += '-mcmodel=kernel'
+endif
+
+executable(
+    'zinnia-limine',
+    sources: [loader_sources, files('main.c'), loader_kernel],
+    include_directories: [common_includes, loader_includes],
+    c_args: limine_c_args,
+    link_args: [
+        '-nostdlib',
+        '-static',
+        '-Wl,-T,' + limine_lds,
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: limine_lds,
+    pie: false,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/loader/elf.c

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <string.h>
+#include "loader.h"
+
+static bool in_image(const struct kernel_image* image, uint64_t vaddr, uint64_t len) {
+    return vaddr >= image->virt && len <= image->size && vaddr - image->virt <= image->size - len;
+}
+
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr) {
+    return loader_ptr(image->phys + (vaddr - image->virt));
+}
+
+static bool apply_relocations(const struct kernel_image* image, uint64_t dynamic_vaddr) {
+    if (!in_image(image, dynamic_vaddr, sizeof(elf64_dyn_t)))
+        return false;
+
+    uint64_t rela = 0;
+    uint64_t rela_size = 0;
+    uint64_t rela_ent = sizeof(elf64_rela_t);
+    for (const elf64_dyn_t* dyn = kernel_ptr(image, dynamic_vaddr); dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    if (!rela)
+        return true;
+    if (rela_ent < sizeof(elf64_rela_t) || !in_image(image, rela, rela_size))
+        return false;
+
+    for (uint64_t off = 0; off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = kernel_ptr(image, rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return false;
+        if (!in_image(image, r->r_offset, sizeof(uint64_t)))
+            return false;
+        *(uint64_t*)kernel_ptr(image, r->r_offset) = (uint64_t)r->r_addend;
+    }
+
+    return true;
+}
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out) {
+    elf64_ehdr_t ehdr;
+    if (size < sizeof(ehdr))
+        return false;
+    memcpy(&ehdr, elf, sizeof(ehdr));
+
+    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) || ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS ||
+        ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
+        return false;
+    if (ehdr.e_machine != ELF_ARCH_MACHINE)
+        return false;
+    if (ehdr.e_phentsize != sizeof(elf64_phdr_t) || ehdr.e_phoff > size ||
+        (size - ehdr.e_phoff) / sizeof(elf64_phdr_t) < ehdr.e_phnum)
+        return false;
+
+    const elf64_phdr_t* phdrs = (const elf64_phdr_t*)(elf + ehdr.e_phoff);
+
+    uint64_t lo = UINT64_MAX;
+    uint64_t hi = 0;
+    uint64_t dynamic_vaddr = 0;
+    uint64_t info_vaddr = 0;
+    uint64_t info_size = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type == PT_DYNAMIC)
+            dynamic_vaddr = ph->p_vaddr;
+        if (ph->p_type == PT_BOOT_INFO) {
+            info_vaddr = ph->p_vaddr;
+            info_size = ph->p_memsz;
+        }
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        if (ph->p_filesz > ph->p_memsz || ph->p_offset > size || size - ph->p_offset < ph->p_filesz)
+            return false;
+        if (ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE) < lo)
+            lo = ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE);
+        if (ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE) > hi)
+            hi = ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE);
+    }
+    if (lo >= hi)
+        return false;
+
+    out->virt = lo;
+    out->size = hi - lo;
+    out->entry = ehdr.e_entry;
+    if (!in_image(out, out->entry, 1))
+        return false;
+    if (info_size != sizeof(struct boot_info) || !in_image(out, info_vaddr, info_size))
+        return false;
+
+    if (!loader_alloc(PHYS_STATIC, out->size, &out->phys))
+        return false;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        memcpy(kernel_ptr(out, ph->p_vaddr), elf + ph->p_offset, ph->p_filesz);
+    }
+    out->info = kernel_ptr(out, info_vaddr);
+
+    if (dynamic_vaddr)
+        return apply_relocations(out, dynamic_vaddr);
+    return true;
+}

boot/loader/lib.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+size_t strlen(const char* str) {
+    if (!str)
+        return 0;
+
+    size_t len = 0;
+    while (str[len])
+        len++;
+    return len;
+}
+
+int memcmp(const void* s1, const void* s2, size_t size) {
+    const uint8_t* a = s1;
+    const uint8_t* b = s2;
+    for (size_t i = 0; i < size; i++) {
+        if (a[i] != b[i])
+            return a[i] < b[i] ? -1 : 1;
+    }
+    return 0;
+}
+
+void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t* s = src;
+    for (size_t i = 0; i < n; i++)
+        d[i] = s[i];
+    return dest;
+}
+
+void* memmove(void* dstptr, const void* srcptr, size_t size) {
+    uint8_t* d = dstptr;
+    const uint8_t* s = srcptr;
+    if (d < s) {
+        for (size_t i = 0; i < size; i++)
+            d[i] = s[i];
+    } else {
+        for (size_t i = size; i > 0; i--)
+            d[i - 1] = s[i - 1];
+    }
+    return dstptr;
+}
+
+void* memset(void* dest, int value, size_t n) {
+    uint8_t* d = dest;
+    for (size_t i = 0; i < n; i++)
+        d[i] = (uint8_t)value;
+    return dest;
+}

boot/loader/loader.c

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: ISC
+
+#include "loader.h"
+#include <string.h>
+#include "loader_arch.h"
+
+extern const uint8_t __kernel_start[];
+extern const uint8_t __kernel_end[];
+
+bool loader_load_kernel(struct kernel_image* out, const char* name) {
+    if (!elf_load(__kernel_start, __kernel_end - __kernel_start, out))
+        return false;
+
+    struct boot_info* info = out->info;
+    memcpy(info->loader, name, MIN(strlen(name), sizeof(info->loader) - 1));
+    info->hhdm_base = ARCH_HHDM_BASE;
+    info->phys_base = out->phys;
+    info->virt_base = out->virt;
+    return true;
+}
+
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length) {
+    struct boot_info* info = kernel->info;
+    if (info->num_files == ARRAY_SIZE(info->files))
+        return false;
+
+    struct boot_file* file = &info->files[info->num_files++];
+    file->data = base;
+    file->length = length;
+    const size_t path_len = MIN(strlen(path), sizeof(file->path) - 1);
+    memcpy(file->path, path, path_len);
+    file->path[path_len] = 0;
+    return true;
+}
+
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len) {
+    for (size_t i = 1; i < len; i++) {
+        const struct phys_mem entry = map[i];
+        size_t j = i;
+        while (j > 0 && map[j - 1].address > entry.address) {
+            map[j] = map[j - 1];
+            j--;
+        }
+        map[j] = entry;
+    }
+
+    size_t merged = 0;
+    for (size_t i = 0; i < len; i++) {
+        if (!map[i].length)
+            continue;
+        struct phys_mem* prev = merged ? &map[merged - 1] : nullptr;
+        if (prev && prev->usage == map[i].usage && prev->address + prev->length == map[i].address)
+            prev->length += map[i].length;
+        else
+            map[merged++] = map[i];
+    }
+
+    struct boot_info* info = kernel->info;
+    info->num_mem_maps = MIN(merged, ARRAY_SIZE(info->mem_map));
+    memcpy(info->mem_map, map, info->num_mem_maps * sizeof(*map));
+}
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+) {
+    uint64_t root;
+    if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &root))
+        return false;
+
+    const uint64_t trampoline_virt = ALIGN_DOWN((uint64_t)arch_enter_kernel, PAGE_SIZE);
+    if (!arch_paging_map(root, ARCH_HHDM_BASE, 0, ALIGN_UP(mem_top, ARCH_HUGE_PAGE_SIZE), ARCH_HUGE_PAGE_SIZE) ||
+        !arch_paging_map(root, kernel->virt, kernel->phys, kernel->size, PAGE_SIZE) ||
+        !arch_paging_map(root, trampoline_virt, ALIGN_DOWN(trampoline_phys, PAGE_SIZE), 2 * PAGE_SIZE, PAGE_SIZE))
+        return false;
+
+    *out_root = root;
+    return true;
+}

boot/loader/loader.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/boot.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint64_t PAGE_SIZE = 0x1000;
+
+struct kernel_image {
+    uint64_t phys;
+    uint64_t virt;
+    uint64_t size;
+    uint64_t entry;
+    struct boot_info* info;
+};
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out);
+void* loader_ptr(uint64_t phys);
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out);
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr);
+
+bool loader_load_kernel(struct kernel_image* out, const char* name);
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length);
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len);
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+);
+
+bool arch_check_cpu();
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size);
+
+[[noreturn]]
+void arch_enter_kernel(uint64_t root, uint64_t entry);

boot/loader/x86_64/enter.S

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl arch_enter_kernel
+    .type arch_enter_kernel, @function
+    .balign 16
+arch_enter_kernel:
+    cli
+    mov cr3, rdi
+    jmp rsi
+    .size arch_enter_kernel, . - arch_enter_kernel

boot/loader/x86_64/loader_arch.h

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+constexpr uint64_t ARCH_HHDM_BASE = 0xffff'8000'0000'0000;
+constexpr uint64_t ARCH_HUGE_PAGE_SIZE = 0x20'0000;

boot/loader/x86_64/meson.build

@@ -0,0 +1,11 @@
+loader_sources += files(
+    'enter.S',
+    'paging.c',
+)
+
+loader_c_args += [
+    '-mno-red-zone',
+    '-masm=intel',
+]
+
+loader_objcopy_format = 'elf64-x86-64'

boot/loader/x86_64/paging.c

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/pmap.h>
+#include <x86_64/defs.h>
+#include "loader.h"
+#include "loader_arch.h"
+
+static bool next_table(uint64_t* table, size_t index, uint64_t** out) {
+    if (!(table[index] & ARCH_FLAG_PRESENT)) {
+        uint64_t page;
+        if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &page))
+            return false;
+        table[index] = page | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    }
+    *out = loader_ptr(table[index] & ARCH_PTE_ADDR_MASK);
+    return true;
+}
+
+bool arch_check_cpu() {
+    uint64_t cr4;
+    asm volatile("mov %0, cr4" : "=r"(cr4));
+    if (cr4 & CR4_LA57)
+        return false;
+    return true;
+}
+
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size) {
+    const int leaf_level = page_size == ARCH_HUGE_PAGE_SIZE ? 1 : 0;
+
+    for (uint64_t off = 0; off < size; off += page_size) {
+        const uint64_t va = virt + off;
+        uint64_t* table = loader_ptr(root);
+        for (int level = 3; level > leaf_level; level--) {
+            if (!next_table(table, (va >> (12 + 9 * level)) & 0x1ff, &table))
+                return false;
+        }
+
+        uint64_t entry = (phys + off) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        if (leaf_level == 1)
+            entry |= ARCH_FLAG_SIZE;
+        table[(va >> (12 + 9 * leaf_level)) & 0x1ff] = entry;
+    }
+
+    return true;
+}

boot/meson.build

@@ -1,6 +1,44 @@
-kernel_sources += files(
-    'cmdline.c',
-    'main.c',
+objcopy = find_program('objcopy')
+zinnia_bootdir = zinnia_datadir / 'boot'
+
+loader_sources = files(
+    'loader/elf.c',
+    'loader/lib.c',
+    'loader/loader.c',
 )
 
-subdir('limine')
+loader_c_args = [
+    '-ffreestanding',
+    '-nostdinc',
+    '-fno-stack-protector',
+    '-fno-omit-frame-pointer',
+    '-mgeneral-regs-only',
+]
+
+loader_includes = include_directories('loader', 'loader' / target_machine.cpu_family())
+
+loader_objcopy_format = 'unknown'
+
+subdir('loader' / target_machine.cpu_family())
+
+loader_kernel = custom_target(
+    'loader-kernel.o',
+    input: kernel,
+    output: 'loader-kernel.o',
+    command: [
+        objcopy,
+        '-I', 'binary',
+        '-O', loader_objcopy_format,
+        '--rename-section', '.data=.kernel,alloc,load,readonly,data,contents',
+        '@INPUT@',
+        '@OUTPUT@',
+    ],
+)
+
+if kconfig.get('CONFIG_BOOT_EFISTUB', 'n') == 'y'
+    subdir('efi')
+endif
+
+if kconfig.get('CONFIG_BOOT_LIMINE', 'n') == 'y'
+    subdir('limine')
+endif

dev/acpi/include/zinnia/acpi.h

@@ -2,14 +2,21 @@
 
 #pragma once
 
+#include <zinnia/errno.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>
 #include <zinnia/types.h>
+#include <zinnia/workqueue.h>
 #include <stdint.h>
 
 INIT_EXTERN(acpi_stage);
+INIT_EXTERN(acpi_namespace_stage);
 
 extern phys_t acpi_rsdp_addr;
+extern struct workqueue* acpi_workqueue;
+
+errno_t acpi_reboot();
+errno_t acpi_power_off();
 
 struct acpi_match {
     const char* hid;

dev/acpi/init.c

@@ -1,16 +1,25 @@
+// SPDX-License-Identifier: ISC
+
 #define pr_fmt(fmt) "acpi: " fmt
 
 #include <zinnia/acpi.h>
 #include <zinnia/alloc.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/irq.h>
+#include <zinnia/pci.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
 #include <zinnia/types.h>
+#include <zinnia/workqueue.h>
+#include <uacpi/event.h>
+#include <uacpi/sleep.h>
 #include <uacpi/status.h>
 #include <uacpi/tables.h>
 #include <uacpi/uacpi.h>
 
 phys_t acpi_rsdp_addr = 0;
+struct workqueue* acpi_workqueue;
 
 static char table_buf[0x1000];
 
@@ -43,3 +52,66 @@ static bool acpi_stage_init(struct boot_info* info) {
 INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
 INIT_DEPENDS(acpi_stage, mem_stage);
 INIT_ENTAILS(acpi_stage, percpu_stage);
+
+static bool namespace_done;
+
+[[__init]]
+static void acpi_namespace_init(struct boot_info*) {
+    errno_t error = workqueue_create("acpi", SCHED_PRIO_WORK_HIGH, &acpi_workqueue);
+    if (error) {
+        log_err("Failed to create the ACPI workqueue (%i)\n", error);
+        return;
+    }
+
+    uacpi_status status = uacpi_initialize(0);
+    if (status != UACPI_STATUS_OK) {
+        log_err("Initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_load();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace loading failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_initialize();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_finalize_gpe_initialization();
+    if (status != UACPI_STATUS_OK)
+        log_warn("GPE initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+
+    namespace_done = true;
+}
+INIT_TASK(acpi_namespace_stage, "dev.acpi.namespace", acpi_namespace_init);
+INIT_DEPENDS(acpi_namespace_stage, acpi_stage, workqueue_stage);
+#if CONFIG_PCI
+INIT_DEPENDS(acpi_namespace_stage, pci_stage);
+#endif
+
+errno_t acpi_reboot() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_reboot();
+    log_err("Failed to reset the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}
+
+errno_t acpi_power_off() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_prepare_for_sleep_state(UACPI_SLEEP_STATE_S5);
+    if (status == UACPI_STATUS_OK) {
+        irq_lock();
+        status = uacpi_enter_sleep_state(UACPI_SLEEP_STATE_S5);
+        irq_unlock();
+    }
+    log_err("Failed to power off the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}

dev/acpi/meson.build

@@ -15,6 +15,5 @@ kernel_includes += include_directories('include')
 
 kernel_c_args += [
     '-DUACPI_FORMATTED_LOGGING',
-    '-DUACPI_BAREBONES_MODE', # TODO: Remove
     '-DUACPI_OVERRIDE_TYPES',
 ]

dev/acpi/uacpi_glue.c

@@ -1,13 +1,24 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/acpi.h>
 #include <zinnia/alloc.h>
+#include <zinnia/clock.h>
 #include <zinnia/console.h>
 #include <zinnia/init.h>
+#include <zinnia/irq.h>
 #include <zinnia/mmio.h>
+#include <zinnia/pci.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/semaphore.h>
+#include <zinnia/spin.h>
 #include <zinnia/utils.h>
 #include <zinnia/virt.h>
 #include <zinnia/vma.h>
+#include <zinnia/workqueue.h>
+#include <arch/irq.h>
 #include <arch/mem.h>
 #include <stdarg.h>
 #include <uacpi/kernel_api.h>
@@ -71,3 +82,250 @@ void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
 void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
     log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
 }
+
+uacpi_status uacpi_kernel_pci_device_open(uacpi_pci_address address, uacpi_handle* out_handle) {
+#if CONFIG_PCI
+    struct pci_address addr = {
+        .domain = address.segment,
+        .bus = address.bus,
+        .device = address.device,
+        .function = address.function,
+    };
+    struct pci_access* access = pci_access_find(&addr);
+    if (!access)
+        return UACPI_STATUS_NOT_FOUND;
+
+    struct pci_device* dev = mem_alloc(sizeof(*dev), 0);
+    if (!dev)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    dev->address = addr;
+    dev->access = access;
+    *out_handle = dev;
+    return UACPI_STATUS_OK;
+#else
+    return UACPI_STATUS_NOT_FOUND;
+#endif
+}
+
+void uacpi_kernel_pci_device_close(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+#if CONFIG_PCI
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        *value = pci_read##width(device, offset); \
+        return UACPI_STATUS_OK; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        pci_write##width(device, offset, value); \
+        return UACPI_STATUS_OK; \
+    }
+#else
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#endif
+
+PCI_READ(8)
+PCI_READ(16)
+PCI_READ(32)
+PCI_WRITE(8)
+PCI_WRITE(16)
+PCI_WRITE(32)
+
+void* uacpi_kernel_alloc(uacpi_size size) {
+    return mem_alloc(size, ALLOC_NOZERO);
+}
+
+void uacpi_kernel_free(void* mem) {
+    if (mem)
+        mem_free(mem);
+}
+
+uacpi_u64 uacpi_kernel_get_nanoseconds_since_boot() {
+    return clock_get_elapsed_ns();
+}
+
+void uacpi_kernel_stall(uacpi_u8 usec) {
+    clock_spin_ns(usec * 1000ull);
+}
+
+void uacpi_kernel_sleep(uacpi_u64 msec) {
+    sched_sleep_until(clock_get_elapsed_ns() + msec * 1'000'000);
+}
+
+uacpi_handle uacpi_kernel_create_semaphore(uacpi_u32 initial_units) {
+    sem_t* sem = mem_alloc(sizeof(*sem), 0);
+    if (sem)
+        sem_init(sem, initial_units);
+    return sem;
+}
+
+void uacpi_kernel_free_semaphore(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_status uacpi_kernel_wait_for_semaphore(uacpi_handle handle, uacpi_u16 timeout) {
+    sem_t* sem = handle;
+    if (timeout == 0)
+        return sem_trywait(sem) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+    if (timeout == 0xFFFF) {
+        sem_wait(sem);
+        return UACPI_STATUS_OK;
+    }
+
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout * 1'000'000ull;
+    return sem_timedwait(sem, &deadline) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+}
+
+void uacpi_kernel_signal_semaphore(uacpi_handle handle) {
+    sem_post(handle);
+}
+
+uacpi_thread_id uacpi_kernel_get_thread_id() {
+    return percpu_get()->sched.current;
+}
+
+uacpi_interrupt_state uacpi_kernel_disable_interrupts() {
+    irq_lock();
+    return 0;
+}
+
+void uacpi_kernel_restore_interrupts(uacpi_interrupt_state) {
+    irq_unlock();
+}
+
+uacpi_status uacpi_kernel_handle_firmware_request(uacpi_firmware_request* req) {
+    switch (req->type) {
+    case UACPI_FIRMWARE_REQUEST_TYPE_BREAKPOINT:
+        log_warn("acpi: Ignoring AML breakpoint\n");
+        break;
+    case UACPI_FIRMWARE_REQUEST_TYPE_FATAL:
+        log_err(
+            "acpi: Fatal firmware error: type %u, code %#x, arg %#llx\n",
+            req->fatal.type,
+            req->fatal.code,
+            (unsigned long long)req->fatal.arg
+        );
+        break;
+    }
+    return UACPI_STATUS_OK;
+}
+
+struct acpi_irq {
+    struct irq_handler handler;
+    uacpi_interrupt_handler func;
+    uacpi_handle ctx;
+};
+
+static enum irq_status acpi_irq_raise(struct irq_handler* self) {
+    struct acpi_irq* irq = containerof(self, struct acpi_irq, handler);
+    return irq->func(irq->ctx) == UACPI_INTERRUPT_HANDLED ? IRQ_STATUS_HANDLED : IRQ_STATUS_IGNORED;
+}
+
+uacpi_status uacpi_kernel_install_interrupt_handler(
+    uacpi_u32 irq,
+    uacpi_interrupt_handler handler,
+    uacpi_handle ctx,
+    uacpi_handle* out_irq_handle
+) {
+    uint32_t gsi = irq;
+    enum irq_trigger_mode trigger = IRQ_TRIGGER_LEVEL;
+    enum irq_polarity polarity = IRQ_POLARITY_LOW;
+    if (irq <= UINT8_MAX)
+        arch_irq_isa_override(irq, &gsi, &trigger, &polarity);
+
+    struct irq_line* line;
+    errno_t status = arch_irq_get_gsi(gsi, &line);
+    if (status) {
+        log_err("acpi: Unable to get GSI %u for IRQ %u (%i)\n", gsi, irq, status);
+        return UACPI_STATUS_UNIMPLEMENTED;
+    }
+
+    struct acpi_irq* acpi_irq = mem_alloc(sizeof(*acpi_irq), 0);
+    if (!acpi_irq)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    acpi_irq->handler.raise = acpi_irq_raise;
+    acpi_irq->func = handler;
+    acpi_irq->ctx = ctx;
+
+    irq_line_attach(line, &acpi_irq->handler);
+    irq_line_program(line, trigger, polarity);
+    log("acpi: Installed handler for IRQ %u on GSI %u\n", irq, gsi);
+
+    *out_irq_handle = acpi_irq;
+    return UACPI_STATUS_OK;
+}
+
+uacpi_status uacpi_kernel_uninstall_interrupt_handler(uacpi_interrupt_handler handler, uacpi_handle irq_handle) {
+    struct acpi_irq* acpi_irq = irq_handle;
+    struct irq_line* line = acpi_irq->handler.line;
+    irq_line_detach(line, &acpi_irq->handler);
+    if (list_is_empty(&line->handlers) && line->mask)
+        line->mask(line);
+    mem_free(acpi_irq);
+    return UACPI_STATUS_OK;
+}
+
+uacpi_handle uacpi_kernel_create_spinlock() {
+    struct spinlock* lock = mem_alloc(sizeof(*lock), 0);
+    if (lock)
+        spin_init(lock);
+    return lock;
+}
+
+void uacpi_kernel_free_spinlock(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_cpu_flags uacpi_kernel_lock_spinlock(uacpi_handle handle) {
+    spin_lock(handle);
+    return 0;
+}
+
+void uacpi_kernel_unlock_spinlock(uacpi_handle handle, uacpi_cpu_flags) {
+    spin_unlock(handle);
+}
+
+struct acpi_work {
+    struct work work;
+    uacpi_work_handler handler;
+    uacpi_handle ctx;
+};
+
+static void acpi_work_fn(struct work* work) {
+    struct acpi_work* item = containerof(work, struct acpi_work, work);
+    const uacpi_work_handler handler = item->handler;
+    const uacpi_handle ctx = item->ctx;
+    handler(ctx);
+}
+
+uacpi_handle uacpi_kernel_create_work_item() {
+    struct acpi_work* item = mem_alloc(sizeof(*item), 0);
+    if (item)
+        work_init(&item->work, acpi_work_fn);
+    return item;
+}
+
+void uacpi_kernel_free_work_item(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+void uacpi_kernel_schedule_work(uacpi_work_type, uacpi_handle work_item, uacpi_work_handler handler, uacpi_handle ctx) {
+    struct acpi_work* item = work_item;
+    item->handler = handler;
+    item->ctx = ctx;
+    work_queue(acpi_workqueue, &item->work);
+}
+
+uacpi_status uacpi_kernel_wait_for_work_completion() {
+    workqueue_flush(acpi_workqueue);
+    return UACPI_STATUS_OK;
+}

dev/block/meson.build

@@ -6,6 +6,6 @@ kernel_sources += files(
     'ram.c',
 )
 
-if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
+if kconfig.get('CONFIG_NVME', 'n') != 'n'
     subdir('nvme')
 endif

dev/block/nvme/Kconfig

@@ -1,13 +1,4 @@
-menu "NVMe Support"
-
-config NVME_CORE
-  bool
-  default y
-
-config NVME_PCI
-  bool "NVMe via PCI transport"
+config NVME
+  bool "NVMe block devices"
   default y
   depends on PCI
-  depends on NVME_CORE
-
-endmenu

dev/block/nvme/command.c

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/mmio.h>
+#include "nvme.h"
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, cmd->do_write ? NVME_CMD_WRITE : NVME_CMD_READ) |
+        mmio_field(NVME_SQE_CID, cmd->cid) | mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_RW_SLBA_LOW, (uint32_t)cmd->start_lba);
+
+    uint32_t cdw11 = mmio_field(NVME_RW_SLBA_HIGH, (uint32_t)(cmd->start_lba >> 32));
+
+    uint32_t cdw12 = mmio_field(NVME_RW_NLB, (uint16_t)(cmd->num_lbas - 1));
+    uint32_t cdw13 = 0;
+    uint32_t cdw14 = 0;
+    uint32_t cdw15 = 0;
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->prp1);
+    mmio_reg_write(sqe, NVME_SQE_DPTR1, cmd->prp2);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+    mmio_reg_write(sqe, NVME_SQE_CDW12, cdw12);
+    mmio_reg_write(sqe, NVME_SQE_CDW13, cdw13);
+    mmio_reg_write(sqe, NVME_SQE_CDW14, cdw14);
+    mmio_reg_write(sqe, NVME_SQE_CDW15, cdw15);
+}
+
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_CMD_FLUSH) | mmio_field(NVME_SQE_CID, cmd->cid);
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+}
+
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_CQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_CQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_CQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_CQ_IV, cmd->irq_vector) |
+        mmio_field(NVME_CREATE_CQ_IEN, cmd->irqs_enabled ? 1 : 0) |
+        mmio_field(NVME_CREATE_CQ_PC, 1); // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->cq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_SQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_SQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_SQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_SQ_CQID, (uint16_t)cmd->cqid) |
+        mmio_field(NVME_CREATE_SQ_QPRIO, 0) | // Priority is the highest.
+        mmio_field(NVME_CREATE_SQ_PC, 1);     // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->sq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_IDENTIFY) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_IDENTIFY_CNTID, cmd->controller_id) | mmio_field(NVME_IDENTIFY_CNS, cmd->cns);
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->buffer);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+}

dev/block/nvme/controller.c

@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t ADMIN_QUEUE_SIZE = 32;
+constexpr size_t IO_QUEUE_SIZE_MAX = 1024;
+constexpr size_t IDENTIFY_SIZE = 4096;
+constexpr size_t NS_LIST_LEN = 1024;
+
+constexpr mmio_reg8 ID_MDTS = {77, MMIO_LE};
+constexpr mmio_reg32 ID_NN = {516, MMIO_LE};
+constexpr mmio_reg64 ID_NSZE = {0, MMIO_LE};
+constexpr mmio_reg8 ID_FLBAS = {26, MMIO_LE};
+constexpr mmio_field8 ID_LBAF_IDX = {ID_FLBAS, 0, 4};
+constexpr mmio_reg32 ID_LBAF0 = {128, MMIO_LE};
+constexpr mmio_field32 ID_LBADS = {ID_LBAF0, 16, 8};
+
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl) {
+    const size_t cap = 512 * arch_mem_page_size();
+    if (ctrl->mdts != 0)
+        return MIN(ctrl->mdts, cap);
+    return cap;
+}
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out) {
+    // Read controller version.
+    uint32_t vs = mmio_reg_read(regs, NVME_REG_VS);
+    log("NVMe controller version %u.%u.%u\n",
+        mmio_field_get(NVME_VS_MJR, vs),
+        mmio_field_get(NVME_VS_MNR, vs),
+        mmio_field_get(NVME_VS_TER, vs));
+
+    uint64_t cap = mmio_reg_read(regs, NVME_REG_CAP);
+
+    // Check if our host page size is supported.
+    const size_t mps_max = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMAX, cap) + 12);
+    const size_t mps_min = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMIN, cap) + 12);
+    const size_t page_size = arch_mem_page_size();
+    if (page_size < mps_min || page_size > mps_max) {
+        log_err("Host page size is not supported on this NVMe!\n");
+        return ENOTSUP;
+    }
+
+    struct nvme_ctrl* ctrl = mem_alloc(sizeof(*ctrl), 0);
+    if (!ctrl)
+        return ENOMEM;
+
+    ctrl->regs = regs;
+    ctrl->dma_tag = dma_tag;
+    ctrl->doorbell_stride = (size_t)4 << mmio_field_get(NVME_CAP_DSTRD, cap);
+    ctrl->mps_min = mps_min;
+    ctrl->mps = __builtin_ctzl(page_size) - 12;
+    mutex_init(&ctrl->admin_lock);
+    spin_init(&ctrl->io_lock);
+    atomic_init(&ctrl->ns_count, 0);
+
+    *out = ctrl;
+    return 0;
+}
+
+// Sets the CC.EN flag.
+static errno_t set_status(struct nvme_ctrl* ctrl, bool enable) {
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+    // CAP.TO is in 500ms units.
+    const uint64_t timeout = MAX((uint64_t)mmio_field_get(NVME_CAP_TO, cap), 1) * 500'000'000;
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout;
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &= ~mmio_field(NVME_CC_EN, UINT32_MAX);
+    cc |= mmio_field(NVME_CC_EN, enable ? 1 : 0);
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    while (true) {
+        uint32_t csts = mmio_reg_read(ctrl->regs, NVME_REG_CSTS);
+
+        if (mmio_field_get(NVME_CSTS_CFS, csts) != 0)
+            return EIO;
+
+        bool ready = mmio_field_get(NVME_CSTS_RDY, csts) != 0;
+        if (ready == enable)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    return 0;
+}
+
+static errno_t admin_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe, struct nvme_completion* out) {
+    errno_t status = nvme_queue_submit_cmd(queue, sqe);
+    if (status)
+        return status;
+    return nvme_queue_next_completion(queue, out);
+}
+
+static errno_t identify(struct nvme_queue* queue, const struct nvme_identify_cmd* cmd) {
+    struct nvme_sqe sqe = {};
+    nvme_identify_cmd_write(cmd, &sqe);
+    struct nvme_completion comp;
+    errno_t status = admin_cmd(queue, &sqe, &comp);
+    if (status)
+        return status;
+    if (!nvme_status_is_success(comp.status)) {
+        log_err("Identification failed with status %x\n", comp.status);
+        return EIO;
+    }
+    return 0;
+}
+
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector) {
+    // Disable the controller.
+    errno_t status = set_status(ctrl, false);
+    if (status)
+        return status;
+
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+
+    const size_t queue_depth = MIN((size_t)mmio_field_get(NVME_CAP_MQES, cap) + 1, IO_QUEUE_SIZE_MAX);
+
+    // Create an admin queue first so we can create more queues.
+    struct nvme_queue* admin_queue;
+    status = nvme_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 0, ADMIN_QUEUE_SIZE, &admin_queue);
+    if (status)
+        return status;
+
+    // Set the admin queue sizes.
+    uint32_t aqa = 0;
+    aqa |= mmio_field(NVME_AQA_ACQS, admin_queue->depth - 1);
+    aqa |= mmio_field(NVME_AQA_ASQS, admin_queue->depth - 1);
+    mmio_reg_write(ctrl->regs, NVME_REG_AQA, aqa);
+
+    // Set the addresses to the admin submission and completion queue.
+    mmio_reg_write(ctrl->regs, NVME_REG_ASQ, admin_queue->sq.addr);
+    mmio_reg_write(ctrl->regs, NVME_REG_ACQ, admin_queue->cq.addr);
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &=
+        ~(mmio_field(NVME_CC_MPS, UINT32_MAX) | mmio_field(NVME_CC_IOCQES, UINT32_MAX) |
+          mmio_field(NVME_CC_IOSQES, UINT32_MAX));
+
+    cc |= mmio_field(NVME_CC_MPS, ctrl->mps);
+    cc |= mmio_field(NVME_CC_IOCQES, __builtin_ctzl(sizeof(struct nvme_cqe)));
+    cc |= mmio_field(NVME_CC_IOSQES, __builtin_ctzl(sizeof(struct nvme_sqe)));
+
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    // Re-enable the controller.
+    status = set_status(ctrl, true);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_io_queue* io_queue;
+    status =
+        nvme_io_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 1, queue_depth, !irqs_enabled, &io_queue);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_sqe sqe = {};
+    struct nvme_completion comp;
+    nvme_create_cq_cmd_write(
+        &(struct nvme_create_cq_cmd){
+            .cq_addr = io_queue->cq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .irqs_enabled = irqs_enabled,
+            .irq_vector = irqs_enabled ? io_irq_vector : 0,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    sqe = (struct nvme_sqe){};
+    nvme_create_sq_cmd_write(
+        &(struct nvme_create_sq_cmd){
+            .sq_addr = io_queue->sq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .cqid = io_queue->queue_id,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* old_admin = ctrl->admin_queue;
+    ctrl->admin_queue = admin_queue;
+    mutex_unlock(&ctrl->admin_lock);
+    if (old_admin)
+        nvme_queue_free(old_admin);
+
+    spin_lock(&ctrl->io_lock);
+    ctrl->io_queue = io_queue;
+    spin_unlock(&ctrl->io_lock);
+
+    return 0;
+
+fail_io:
+    nvme_io_queue_free(io_queue);
+fail_admin:
+    nvme_queue_free(admin_queue);
+    return status;
+}
+
+static void trim_ascii_end(char* str, size_t len) {
+    while (len > 0 &&
+           (str[len - 1] == ' ' || str[len - 1] == '\t' || str[len - 1] == '\n' || str[len - 1] == '\r' ||
+            str[len - 1] == '\f'))
+        len--;
+    str[len] = '\0';
+}
+
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl) {
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status)
+        return ENOMEM;
+
+    mutex_lock(&ctrl->admin_lock);
+    if (!ctrl->admin_queue) {
+        status = ENXIO;
+        goto out;
+    }
+
+    status = identify(
+        ctrl->admin_queue,
+        &(struct nvme_identify_cmd){
+            .buffer = identify_buffer.addr,
+            .controller_id = 0, // TODO: Use a real number here
+            .cns = 1,
+            .nsid = 0,
+        }
+    );
+    // Check if the identification was successful.
+    if (status)
+        goto out;
+
+    dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+    // Read the model strings.
+    const uint8_t* view = identify_buffer.virt;
+    memcpy(ctrl->serial, view + 4, sizeof(ctrl->serial) - 1);
+    memcpy(ctrl->model, view + 24, sizeof(ctrl->model) - 1);
+    memcpy(ctrl->revision, view + 64, sizeof(ctrl->revision) - 1);
+    trim_ascii_end(ctrl->serial, sizeof(ctrl->serial) - 1);
+    trim_ascii_end(ctrl->model, sizeof(ctrl->model) - 1);
+    trim_ascii_end(ctrl->revision, sizeof(ctrl->revision) - 1);
+    log("Model \"%s\"\n", ctrl->model);
+
+    atomic_store(&ctrl->ns_count, mmio_reg_read(identify_buffer.virt, ID_NN), ATOMIC_RELEASE);
+
+    uint8_t mdts = mmio_reg_read(identify_buffer.virt, ID_MDTS);
+    if (mdts != 0 && mdts < sizeof(size_t) * 8)
+        ctrl->mdts = ((size_t)1 << mdts) * ctrl->mps_min;
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    return status;
+}
+
+// Scans all existing namespaces.
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out) {
+    vec_init(out);
+
+    uint32_t* ns_list = mem_alloc(NS_LIST_LEN * sizeof(uint32_t), 0);
+    if (!ns_list)
+        return ENOMEM;
+
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status) {
+        mem_free(ns_list);
+        return ENOMEM;
+    }
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* aq = ctrl->admin_queue;
+    if (!aq) {
+        status = ENXIO;
+        goto out;
+    }
+
+    uint32_t nsid = 0;
+    while (true) {
+        const struct nvme_identify_cmd admin_ident_cmd = {
+            .buffer = identify_buffer.addr,
+            .controller_id = 0,
+            .cns = 2,
+            .nsid = nsid,
+        };
+        status = identify(aq, &admin_ident_cmd);
+
+        if (status)
+            goto out;
+
+        dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+        // Parse the identifier list until we hit an namespace value of 0.
+        size_t ns_list_len = 0;
+        const uint32_t* raw_list = identify_buffer.virt;
+        while (ns_list_len < NS_LIST_LEN && raw_list[ns_list_len] != 0) {
+            ns_list[ns_list_len] = raw_list[ns_list_len];
+            ns_list_len++;
+        }
+
+        for (size_t i = 0; i < ns_list_len; i++) {
+            const uint32_t id = ns_list[i];
+            const struct nvme_identify_cmd ns_ident_cmd = {
+                .buffer = identify_buffer.addr,
+                .controller_id = 0,
+                .cns = 0,
+                .nsid = id,
+            };
+            status = identify(aq, &ns_ident_cmd);
+
+            if (status)
+                goto out;
+
+            dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+            const uint64_t lba_count = mmio_reg_read(identify_buffer.virt, ID_NSZE);
+
+            // To get the block size of a namespace, we need to read a descriptor (see LBAF0-15).
+            // Get the format descriptor index used by this namespace.
+            const uint8_t lba_format_idx = mmio_field_get(ID_LBAF_IDX, mmio_reg_read(identify_buffer.virt, ID_FLBAS));
+
+            const mmio_reg32 lbaf = {ID_LBAF0.offset + lba_format_idx * sizeof(uint32_t), MMIO_LE};
+            const uint32_t lba_format = mmio_reg_read(identify_buffer.virt, lbaf);
+
+            uint8_t lba_shift = mmio_field_get(ID_LBADS, lba_format);
+            // If LBADS doesn't contain a non-zero value, we need to assume a default. Use 512 bytes here.
+            if (lba_shift == 0)
+                lba_shift = 9;
+
+            struct nvme_ns* ns;
+            status = nvme_ns_new(ctrl, id, lba_shift, lba_count, nvme_ctrl_max_transfer_bytes(ctrl), &ns);
+            if (status)
+                goto out;
+            status = vec_push(out, ns);
+            if (status) {
+                blockdev_unref(&ns->blk);
+                goto out;
+            }
+        }
+
+        // If there is no 0 in the buffer, there's more entries than can fit in `identify_buffer`.
+        if (ns_list_len < NS_LIST_LEN)
+            break;
+        nsid = ns_list[ns_list_len - 1];
+    }
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    mem_free(ns_list);
+    if (status) {
+        vec_foreach(out, i) {
+            blockdev_unref(&vec_elem_get(out, i)->blk);
+        }
+        vec_destroy(out);
+    }
+    return status;
+}
+
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl) {
+    spin_lock(&ctrl->io_lock);
+    struct nvme_io_queue* queue = ctrl->io_queue;
+    spin_unlock(&ctrl->io_lock);
+    return queue;
+}

dev/block/nvme/core.c

@@ -1,41 +0,0 @@
-#define pr_fmt(fmt) "nvme: " fmt
-
-#include <zinnia/fdt.h>
-#include <zinnia/module.h>
-#include <zinnia/pci.h>
-#include <zinnia/print.h>
-#include <zinnia/utils.h>
-#include <config.h>
-
-#if CONFIG_NVME_PCI
-#include "nvme_pci.h"
-#endif
-
-struct nvme_dev {
-#if CONFIG_NVME_PCI
-    struct pci_device pci_dev;
-#endif
-};
-
-errno_t nvme_init(struct module* mod) {
-    errno_t status = 0;
-
-#if CONFIG_NVME_PCI
-    status = pci_driver_register(&nvme_driver);
-#endif
-
-    return status;
-}
-
-errno_t nvme_exit(struct module* mod) {
-    // TODO: Unregister
-    return 0;
-}
-
-MODULE_ENTRY(nvme_init);
-MODULE_EXIT(nvme_exit);
-
-MODULE_DESC("NVMe Block Devices");
-MODULE_AUTHOR("Marvin Friedrich");
-MODULE_LICENSE("ISC");
-MODULE_VERSION("1.0");

dev/block/nvme/meson.build

@@ -1,12 +1,13 @@
-nvme_sources = files('core.c')
-
-if kconfig.get('CONFIG_NVME_PCI', 'n') == 'y'
-    nvme_sources += files('pci.c')
-endif
-
 modules += {
     'nvme': {
-        'modular': kconfig.get('CONFIG_NVME_CORE') == 'm',
-        'sources': nvme_sources,
+        'modular': kconfig.get('CONFIG_NVME') == 'm',
+        'sources': files(
+            'command.c',
+            'controller.c',
+            'namespace.c',
+            'pci.c',
+            'prp.c',
+            'queue.c',
+        ),
     },
 }

dev/block/nvme/namespace.c

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <stdckdint.h>
+#include "nvme.h"
+
+static errno_t nvme_ns_submit(struct blockdev* blk, struct bio* bio) {
+    struct nvme_ns* ns = containerof(blk, struct nvme_ns, blk);
+
+    if (bio->op != BIO_FLUSH) {
+        uint64_t end_lba;
+        if (ckd_add(&end_lba, bio->lba, (uint64_t)bio->num_lbas)) {
+            bio_complete(bio, EOVERFLOW, 0);
+            return 0;
+        }
+        if (end_lba > ns->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+    }
+
+    struct nvme_io_queue* queue = nvme_ctrl_io_queue(ns->ctrl);
+    if (!queue) {
+        bio_complete(bio, EIO, 0);
+        return 0;
+    }
+
+    size_t bytes = 0;
+    for (size_t i = 0; i < bio->num_segs; i++)
+        bytes += bio->segs[i].len;
+
+    _Atomic(bool) done = false;
+    const bool poll = queue->poll;
+
+    uint16_t cid = nvme_io_queue_submit(
+        queue,
+        bio,
+        (struct nvme_rw_cmd){
+            .prp1 = 0,
+            .prp2 = 0,
+            .cid = 0,
+            .do_write = bio->op == BIO_WRITE,
+            .start_lba = bio->lba,
+            .num_lbas = bio->num_lbas,
+            .bytes = bytes,
+            .control = 0,
+            .ds_mgmt = 0,
+            .ref_tag = 0,
+            .app_tag = 0,
+            .app_mask = 0,
+            .nsid = ns->nsid,
+        },
+        poll ? &done : nullptr
+    );
+
+    // If we're not interrupt-driven, poll the IO completion queue.
+    if (poll) {
+        const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
+        while (!atomic_load(&done, ATOMIC_ACQUIRE)) {
+            nvme_io_queue_drain(queue);
+            if (atomic_load(&done, ATOMIC_ACQUIRE))
+                break;
+            if (clock_get_elapsed_ns() >= deadline && nvme_io_queue_abandon(queue, cid, bio)) {
+                bio_complete(bio, EIO, 0);
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+static void nvme_ns_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct nvme_ns, blk));
+}
+
+static const struct blockdev_ops nvme_ns_ops = {
+    .submit = nvme_ns_submit,
+    .destroy = nvme_ns_destroy,
+};
+
+errno_t nvme_ns_new(
+    struct nvme_ctrl* ctrl,
+    uint32_t nsid,
+    uint8_t lba_shift,
+    uint64_t lba_count,
+    size_t max_transfer_bytes,
+    struct nvme_ns** out
+) {
+    log("Namespace %u: LBA size %zu bytes, %zu MBs total\n",
+        nsid,
+        (size_t)1 << lba_shift,
+        (size_t)(lba_count << lba_shift) / 1024 / 1024);
+
+    struct nvme_ns* ns = mem_alloc(sizeof(*ns), 0);
+    if (!ns)
+        return ENOMEM;
+
+    blockdev_init(&ns->blk, &nvme_ns_ops, NVME_MAJOR, nsid, (size_t)1 << lba_shift, lba_count);
+    ns->blk.limits = (struct blockdev_limits){
+        .max_lbas = MAX(max_transfer_bytes >> lba_shift, 1),
+        .max_segs = 512,
+    };
+    ns->ctrl = ctrl;
+    ns->nsid = nsid;
+    ns->lba_shift = lba_shift;
+    ns->lba_count = lba_count;
+    ns->max_transfer_bytes = max_transfer_bytes;
+
+    *out = ns;
+    return 0;
+}

dev/block/nvme/nvme.h

@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/dma.h>
+#include <zinnia/errno.h>
+#include <zinnia/event.h>
+#include <zinnia/mutex.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "spec.h"
+
+constexpr uint32_t NVME_MAJOR = 159;
+constexpr uint64_t NVME_COMPLETION_TIMEOUT_NS = 30'000'000'000;
+
+// An entry in the completion queue.
+struct nvme_completion {
+    uint32_t result;
+    uint16_t sq_head;
+    uint16_t sq_id;
+    uint16_t cmd_id;
+    bool phase_tag;
+    uint16_t status;
+};
+
+static inline bool nvme_status_is_success(uint16_t status) {
+    return status == 0;
+}
+
+struct nvme_rw_cmd {
+    uint64_t prp1;
+    uint64_t prp2;
+    uint16_t cid;
+    bool do_write;
+    uint64_t start_lba;
+    size_t num_lbas;
+    size_t bytes;
+    uint16_t control;
+    uint32_t ds_mgmt;
+    uint32_t ref_tag;
+    uint16_t app_tag;
+    uint16_t app_mask;
+    uint32_t nsid;
+};
+
+struct nvme_flush_cmd {
+    uint16_t cid;
+    uint32_t nsid;
+};
+
+struct nvme_create_cq_cmd {
+    dma_addr_t cq_addr;
+    size_t depth;
+    size_t qid;
+    bool irqs_enabled;
+    uint16_t irq_vector;
+};
+
+struct nvme_create_sq_cmd {
+    dma_addr_t sq_addr;
+    size_t depth;
+    size_t qid;
+    size_t cqid;
+};
+
+struct nvme_identify_cmd {
+    dma_addr_t buffer;
+    uint16_t controller_id;
+    uint8_t cns;
+    uint32_t nsid;
+};
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe);
+
+struct nvme_queue {
+    // Amount of queue entries.
+    size_t depth;
+    size_t doorbells_offset;
+    size_t doorbell_stride;
+    void* regs;
+    // Physical buffer for the completion queue.
+    struct dma_buffer cq;
+    // The index of the current completion queue entry.
+    size_t cq_head;
+    // Determines whether a completion queue entry is new.
+    uint8_t cq_phase;
+    // Physical buffer for the submission queue.
+    struct dma_buffer sq;
+    // The index of the current submission queue entry.
+    size_t sq_tail;
+};
+
+errno_t nvme_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    struct nvme_queue** out
+);
+void nvme_queue_free(struct nvme_queue* queue);
+errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe);
+errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out);
+
+struct nvme_inflight {
+    bool active;
+    struct bio* bio;
+    size_t lbas;
+    struct dma_map* map;
+    struct dma_buffer prps;
+    _Atomic(bool)* done;
+};
+
+struct nvme_io_queue {
+    size_t queue_id;
+    size_t depth;
+    size_t doorbells_offset;
+    size_t doorbell_stride;
+    void* regs;
+    bool poll;
+    struct dma_tag* tag;
+    struct dma_buffer cq;
+    struct dma_buffer sq;
+    struct spinlock sq_lock;
+    size_t sq_tail;
+    struct spinlock cq_lock;
+    size_t cq_head;
+    uint8_t cq_phase;
+    struct spinlock cmds_lock;
+    struct nvme_inflight* slots;
+    uint16_t* free;
+    size_t num_free;
+    struct event slots_free;
+};
+
+errno_t nvme_io_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    bool poll,
+    struct nvme_io_queue** out
+);
+void nvme_io_queue_free(struct nvme_io_queue* queue);
+uint16_t nvme_io_queue_submit(
+    struct nvme_io_queue* queue,
+    struct bio* bio,
+    struct nvme_rw_cmd cmd,
+    _Atomic(bool)* done
+);
+bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio);
+size_t nvme_io_queue_drain(struct nvme_io_queue* queue);
+
+errno_t nvme_build_prps(
+    struct dma_tag* tag,
+    struct bio* bio,
+    struct nvme_inflight* slot,
+    uint64_t* prp1,
+    uint64_t* prp2
+);
+void nvme_release_prps(struct nvme_inflight* slot);
+
+struct nvme_ns;
+
+struct nvme_ctrl {
+    void* regs;
+    struct dma_tag* dma_tag;
+    size_t doorbell_stride; // The amount of bytes between two doorbells.
+    size_t mps_min;
+    uint8_t mps;
+
+    char serial[20 + 1];  // Serial number of the controller.
+    char model[40 + 1];   // Model name of the controller.
+    char revision[8 + 1]; // Revision of the controller's firmware.
+
+    struct nvme_queue* admin_queue; // Queue to submit admin commands to.
+    mutex_t admin_lock;
+
+    struct nvme_io_queue* io_queue; // Queue to submit IO commands to.
+    struct spinlock io_lock;
+
+    _Atomic(uint32_t) ns_count; // Amount of namespaces.
+    size_t mdts;                // Maximum data transfer size.
+};
+
+typedef vec(struct nvme_ns*) nvme_ns_vec_t;
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out);
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl);
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector);
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl);
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out);
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl);
+
+struct nvme_ns {
+    struct blockdev blk;
+    struct nvme_ctrl* ctrl;
+    uint32_t nsid;
+    uint8_t lba_shift;
+    uint64_t lba_count;
+    size_t max_transfer_bytes;
+};
+
+errno_t nvme_ns_new(
+    struct nvme_ctrl* ctrl,
+    uint32_t nsid,
+    uint8_t lba_shift,
+    uint64_t lba_count,
+    size_t max_transfer_bytes,
+    struct nvme_ns** out
+);

dev/block/nvme/nvme_pci.h

@@ -1,5 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-extern struct pci_driver nvme_driver;

dev/block/nvme/pci.c

@@ -1,11 +1,134 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/module.h>
 #include <zinnia/pci.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+#include "nvme.h"
+
+static _Atomic(size_t) nvme_counter;
+
+struct nvme_irq_handler {
+    struct irq_handler handler;
+    struct nvme_ctrl* controller;
+};
+
+static void nvme_irq_thread(struct irq_handler* self) {
+    struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler);
+    struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller);
+    if (queue)
+        nvme_io_queue_drain(queue);
+}
+
+static errno_t map_bar(struct pci_bar* bar, void** out) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t len = ALIGN_UP(bar->size, page_size);
+    uintptr_t vaddr = vm_alloc(len);
+    if (!vaddr)
+        return ENOMEM;
+
+    for (size_t i = 0; i < len; i += page_size) {
+        errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, bar->address + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status)
+            return status;
+    }
+
+    *out = (void*)vaddr;
+    return 0;
+}
 
 static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
-    pci_log(dev, "Detected controller \n");
+    pci_log(dev, "Probing NVMe device\n");
 
-    return 0;
+    // Enable MMIO decoding and DMA. We only support MSI-X or polling, so keep legacy INTx disabled.
+    uint16_t cmd = pci_read16(dev, PCI_COMMAND);
+    pci_write16(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEM | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE);
+
+    struct irq_line* irq_line = nullptr;
+    if (pci_setup_msix(dev, &irq_line)) {
+        pci_log_warn(dev, "NVMe MSI-X setup failed, falling back to polling completions\n");
+        irq_line = nullptr;
+    }
+
+    struct pci_bar bar;
+    errno_t status = pci_bar_get(dev, 0, &bar);
+    if (status)
+        return ENXIO;
+    if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
+        return ENXIO;
+
+    void* regs;
+    status = map_bar(&bar, &regs);
+    if (status)
+        return status;
+
+    struct nvme_ctrl* controller;
+    status = nvme_ctrl_new_pci(regs, dev->dma_tag, &controller);
+    if (status) {
+        pci_log_err(dev, "Failed to probe controller: %i\n", status);
+        return ENODEV;
+    }
+
+    if (irq_line) {
+        struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0);
+        if (!handler)
+            return ENOMEM;
+        handler->handler.thread = nvme_irq_thread;
+        handler->controller = controller;
+
+        char name[32];
+        ksnprintf(name, sizeof(name), "nvme-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
+        status = irq_line_attach_task(irq_line, &handler->handler, name);
+        if (status) {
+            mem_free(handler);
+            return status;
+        }
+        if (irq_line->unmask)
+            irq_line->unmask(irq_line);
+    }
+
+    // Reset the controller to initialize all queues and other structures.
+    status = nvme_ctrl_reset(controller, irq_line != nullptr, 0);
+    if (status) {
+        pci_log_err(dev, "Failed to reset controller: %i\n", status);
+        return ENODEV;
+    }
+
+    status = nvme_ctrl_identify(controller);
+    if (status) {
+        pci_log_err(dev, "Failed to identify controller: %i\n", status);
+        return ENODEV;
+    }
+
+    nvme_ns_vec_t namespaces;
+    status = nvme_ctrl_scan_namespaces(controller, &namespaces);
+    if (status) {
+        pci_log_err(dev, "Failed to identify controller: %i\n", status);
+        return ENODEV;
+    }
+
+    const size_t nvme_id = atomic_fetch_add(&nvme_counter, 1, ATOMIC_SEQ_CST);
+    vec_foreach(&namespaces, i) {
+        struct nvme_ns* ns = vec_elem_get(&namespaces, i);
+        char path[32];
+        ksnprintf(path, sizeof(path), "nvme%zun%u", nvme_id, ns->nsid);
+        status = blockdev_register(path, &ns->blk);
+        if (status)
+            break;
+    }
+    vec_destroy(&namespaces);
+
+    return status;
 }
 
 static const struct pci_match nvme_match[] = {
@@ -15,9 +138,26 @@ static const struct pci_match nvme_match[] = {
 
 MATCH_TABLE("pci", nvme_match);
 
-struct pci_driver nvme_driver = {
-    .name = "nvme_pci",
+static struct pci_driver nvme_driver = {
+    .name = "nvme",
     .match = nvme_match,
     .num_matches = ARRAY_SIZE(nvme_match),
     .probe = nvme_pci_probe,
 };
+
+errno_t nvme_init(struct module* mod) {
+    return pci_driver_register(&nvme_driver);
+}
+
+errno_t nvme_exit(struct module* mod) {
+    // TODO: Unregister
+    return 0;
+}
+
+MODULE_ENTRY(nvme_init);
+MODULE_EXIT(nvme_exit);
+
+MODULE_DESC("NVMe Block Devices");
+MODULE_AUTHOR("Marvin Friedrich");
+MODULE_LICENSE("ISC");
+MODULE_VERSION("1.0");

dev/block/nvme/prp.c

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include "nvme.h"
+
+static size_t count_pages(struct dma_map* map, size_t page_size) {
+    size_t count = 0;
+    for (size_t i = 0; i < map->num_segs; i++) {
+        const struct dma_seg* seg = &map->segs[i];
+        if (seg->len == 0)
+            continue;
+        const dma_addr_t end = seg->addr + seg->len;
+        for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size)
+            count++;
+    }
+    return count;
+}
+
+static void fill_pages(struct dma_map* map, size_t page_size, uint64_t* out) {
+    bool first_page = true;
+    size_t idx = 0;
+    for (size_t i = 0; i < map->num_segs; i++) {
+        const struct dma_seg* seg = &map->segs[i];
+        if (seg->len == 0)
+            continue;
+        const dma_addr_t end = seg->addr + seg->len;
+        for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size) {
+            if (first_page)
+                first_page = false;
+            else
+                out[idx++] = page;
+        }
+    }
+}
+
+errno_t nvme_build_prps(
+    struct dma_tag* tag,
+    struct bio* bio,
+    struct nvme_inflight* slot,
+    uint64_t* prp1,
+    uint64_t* prp2
+) {
+    const size_t page_size = arch_mem_page_size();
+    slot->map = nullptr;
+    slot->prps = (struct dma_buffer){};
+
+    if (bio->num_segs == 0)
+        return EIO;
+
+    errno_t status = dma_map_create(tag, SIZE_MAX, bio->num_segs, SIZE_MAX, 0, &slot->map);
+    if (status)
+        return status;
+
+    status = dma_map_load_bio(slot->map, bio);
+    if (status)
+        goto fail;
+    if (slot->map->num_segs == 0) {
+        status = EIO;
+        goto fail;
+    }
+
+    dma_map_sync(slot->map, 0, slot->map->mapped_len, bio->op == BIO_WRITE ? DMA_SYNC_PREWRITE : DMA_SYNC_PREREAD);
+
+    *prp1 = slot->map->segs[0].addr;
+
+    // Enumerate the base of every page the transfer touches in order.
+    const size_t pages = count_pages(slot->map, page_size);
+    const size_t rest_pages = pages ? pages - 1 : 0;
+
+    if (rest_pages == 0) {
+        *prp2 = 0;
+        return 0;
+    }
+
+    if (rest_pages == 1) {
+        uint64_t second;
+        fill_pages(slot->map, page_size, &second);
+        *prp2 = second;
+        return 0;
+    }
+
+    // We don't want to chain across pages.
+    const size_t entries_per_page = page_size / sizeof(uint64_t);
+    if (rest_pages > entries_per_page) {
+        status = ENOMEM;
+        goto fail;
+    }
+
+    status = dma_buffer_alloc(tag, page_size, &slot->prps);
+    if (status)
+        goto fail;
+
+    fill_pages(slot->map, page_size, slot->prps.virt);
+    dma_buffer_sync(&slot->prps, 0, rest_pages * sizeof(uint64_t), DMA_SYNC_PREWRITE);
+
+    *prp2 = slot->prps.addr;
+    return 0;
+
+fail:
+    nvme_release_prps(slot);
+    return status;
+}
+
+void nvme_release_prps(struct nvme_inflight* slot) {
+    if (slot->map) {
+        dma_map_sync(
+            slot->map,
+            0,
+            slot->map->mapped_len,
+            slot->bio && slot->bio->op == BIO_WRITE ? DMA_SYNC_POSTWRITE : DMA_SYNC_POSTREAD
+        );
+        dma_map_unload(slot->map);
+        dma_map_destroy(slot->map);
+        slot->map = nullptr;
+    }
+    dma_buffer_free(&slot->prps);
+}

dev/block/nvme/queue.c

@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t DOORBELL_OFFSET = 0x1000;
+
+static void ring_tail(void* regs, size_t doorbells_offset, size_t tail) {
+    mmio_reg_write(regs, ((mmio_reg32){doorbells_offset, MMIO_LE}), (uint32_t)tail);
+}
+
+static void ring_head(void* regs, size_t doorbells_offset, size_t doorbell_stride, size_t head) {
+    mmio_reg_write(regs, ((mmio_reg32){doorbells_offset + doorbell_stride, MMIO_LE}), (uint32_t)head);
+}
+
+// Creates a new submission and completion queue pair.
+errno_t nvme_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    struct nvme_queue** out
+) {
+    const size_t align = 0x1000;
+    const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
+    const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
+
+    struct nvme_queue* queue = mem_alloc(sizeof(*queue), 0);
+    if (!queue)
+        return ENOMEM;
+
+    // Allocate memory the completion queue.
+    errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
+    if (status)
+        goto fail;
+
+    // Allocate memory for the submission queue.
+    status = dma_buffer_alloc(tag, sq_size, &queue->sq);
+    if (status)
+        goto fail;
+
+    // Calculate the offset of the doorbell registers. The stride is already precomputed here.
+    queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
+    queue->doorbell_stride = doorbell_stride;
+    queue->depth = depth;
+    queue->regs = regs;
+    queue->cq_head = 0;
+    queue->cq_phase = 1; // When the controller is enabled, the first phase is 1.
+    queue->sq_tail = 0;
+
+    *out = queue;
+    return 0;
+
+fail:
+    nvme_queue_free(queue);
+    return status;
+}
+
+void nvme_queue_free(struct nvme_queue* queue) {
+    dma_buffer_free(&queue->cq);
+    dma_buffer_free(&queue->sq);
+    mem_free(queue);
+}
+
+// Submits a command to this queue.
+errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe) {
+    struct nvme_sqe* entries = queue->sq.virt;
+    memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
+    dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
+
+    queue->sq_tail++;
+    if (queue->sq_tail == queue->depth)
+        queue->sq_tail = 0;
+
+    // Notify the controller of the new tail index.
+    ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
+
+    return 0;
+}
+
+// Reads the next completion entry from the queue.
+errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out) {
+    struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
+
+    // Wait until the phase for this entry has changed.
+    uint32_t dw3;
+    const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
+    while (true) {
+        dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
+        dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
+
+        // The controller will flip the phase bit of the current entry when writing.
+        if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) == queue->cq_phase)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    // Then, read the rest of the completion queue entry.
+    uint32_t dw0 = mmio_reg_read(entry, NVME_CQE_DW0);
+    uint32_t dw2 = mmio_reg_read(entry, NVME_CQE_DW2);
+
+    *out = (struct nvme_completion){
+        .result = dw0,
+        .sq_head = mmio_field_get(NVME_CQE_SQ_HEAD, dw2),
+        .sq_id = mmio_field_get(NVME_CQE_SQ_IDENT, dw2),
+        .cmd_id = mmio_field_get(NVME_CQE_CID, dw3),
+        .status = mmio_field_get(NVME_CQE_STATUS, dw3),
+        .phase_tag = mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != 0,
+    };
+
+    queue->cq_head++;
+    if (queue->cq_head == queue->depth) {
+        queue->cq_head = 0;
+        queue->cq_phase ^= 1; // Flip the phase bit.
+    }
+
+    // Notify the controller of the new head index.
+    ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
+
+    return 0;
+}
+
+errno_t nvme_io_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    bool poll,
+    struct nvme_io_queue** out
+) {
+    const size_t align = 0x1000;
+    const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
+    const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
+
+    struct nvme_io_queue* queue = mem_alloc(sizeof(*queue), 0);
+    if (!queue)
+        return ENOMEM;
+
+    errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
+    if (status)
+        goto fail;
+
+    status = dma_buffer_alloc(tag, sq_size, &queue->sq);
+    if (status)
+        goto fail;
+
+    const size_t slots = depth - 1;
+    queue->slots = mem_alloc(slots * sizeof(*queue->slots), 0);
+    queue->free = mem_alloc(slots * sizeof(*queue->free), 0);
+    if (!queue->slots || !queue->free) {
+        status = ENOMEM;
+        goto fail;
+    }
+    for (size_t i = 0; i < slots; i++)
+        queue->free[i] = slots - 1 - i;
+    queue->num_free = slots;
+
+    queue->queue_id = queue_id;
+    queue->depth = depth;
+    queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
+    queue->doorbell_stride = doorbell_stride;
+    queue->regs = regs;
+    queue->poll = poll;
+    queue->tag = tag;
+    queue->sq_tail = 0;
+    queue->cq_head = 0;
+    queue->cq_phase = 1;
+    spin_init(&queue->sq_lock);
+    spin_init(&queue->cq_lock);
+    spin_init(&queue->cmds_lock);
+    event_init(&queue->slots_free);
+
+    *out = queue;
+    return 0;
+
+fail:
+    nvme_io_queue_free(queue);
+    return status;
+}
+
+void nvme_io_queue_free(struct nvme_io_queue* queue) {
+    dma_buffer_free(&queue->cq);
+    dma_buffer_free(&queue->sq);
+    mem_free(queue->slots);
+    mem_free(queue->free);
+    mem_free(queue);
+}
+
+static uint16_t acquire_slot(struct nvme_io_queue* queue) {
+    while (true) {
+        spin_lock(&queue->cmds_lock);
+        if (queue->num_free) {
+            uint16_t cid = queue->free[--queue->num_free];
+            spin_unlock(&queue->cmds_lock);
+            return cid;
+        }
+        spin_unlock(&queue->cmds_lock);
+
+        if (queue->poll) {
+            nvme_io_queue_drain(queue);
+            continue;
+        }
+
+        nvme_io_queue_drain(queue);
+        event_reset(&queue->slots_free);
+        spin_lock(&queue->cmds_lock);
+        bool empty = queue->num_free == 0;
+        spin_unlock(&queue->cmds_lock);
+        if (empty)
+            event_wait(&queue->slots_free);
+    }
+}
+
+static void release_slot(struct nvme_io_queue* queue, uint16_t cid) {
+    spin_lock(&queue->cmds_lock);
+    queue->free[queue->num_free++] = cid;
+    spin_unlock(&queue->cmds_lock);
+}
+
+static void write_sqe(struct nvme_io_queue* queue, const struct nvme_sqe* sqe) {
+    spin_lock(&queue->sq_lock);
+    struct nvme_sqe* entries = queue->sq.virt;
+    memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
+    dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
+    queue->sq_tail++;
+    if (queue->sq_tail == queue->depth)
+        queue->sq_tail = 0;
+    ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
+    spin_unlock(&queue->sq_lock);
+}
+
+uint16_t nvme_io_queue_submit(
+    struct nvme_io_queue* queue,
+    struct bio* bio,
+    struct nvme_rw_cmd cmd,
+    _Atomic(bool)* done
+) {
+    const uint16_t cid = acquire_slot(queue);
+    struct nvme_inflight* slot = &queue->slots[cid];
+
+    struct nvme_sqe sqe = {};
+    if (bio->op == BIO_FLUSH) {
+        slot->map = nullptr;
+        slot->prps = (struct dma_buffer){};
+        nvme_flush_cmd_write(&(struct nvme_flush_cmd){.cid = cid, .nsid = cmd.nsid}, &sqe);
+    } else {
+        errno_t status = nvme_build_prps(queue->tag, bio, slot, &cmd.prp1, &cmd.prp2);
+        if (status) {
+            release_slot(queue, cid);
+            bio_complete(bio, EIO, 0);
+            if (done)
+                atomic_store(done, true, ATOMIC_RELEASE);
+            return cid;
+        }
+        cmd.cid = cid;
+        nvme_rw_cmd_write(&cmd, &sqe);
+    }
+
+    spin_lock(&queue->cmds_lock);
+    slot->active = true;
+    slot->bio = bio;
+    slot->lbas = bio->num_lbas;
+    slot->done = done;
+    spin_unlock(&queue->cmds_lock);
+
+    write_sqe(queue, &sqe);
+    return cid;
+}
+
+bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio) {
+    spin_lock(&queue->cmds_lock);
+    struct nvme_inflight* slot = &queue->slots[cid];
+    bool abandoned = slot->active && slot->bio == bio;
+    if (abandoned) {
+        slot->bio = nullptr;
+        slot->done = nullptr;
+    }
+    spin_unlock(&queue->cmds_lock);
+    return abandoned;
+}
+
+size_t nvme_io_queue_drain(struct nvme_io_queue* queue) {
+    spin_lock(&queue->cq_lock);
+    size_t completed = 0;
+
+    while (true) {
+        struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
+        dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
+        uint32_t dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
+        if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != queue->cq_phase)
+            break;
+
+        uint16_t cid = mmio_field_get(NVME_CQE_CID, dw3);
+        uint16_t status = mmio_field_get(NVME_CQE_STATUS, dw3);
+
+        queue->cq_head++;
+        if (queue->cq_head == queue->depth) {
+            queue->cq_head = 0;
+            queue->cq_phase ^= 1;
+        }
+
+        struct nvme_inflight inflight = {};
+        spin_lock(&queue->cmds_lock);
+        if (cid < queue->depth - 1 && queue->slots[cid].active) {
+            inflight = queue->slots[cid];
+            queue->slots[cid].active = false;
+            queue->free[queue->num_free++] = cid;
+        }
+        spin_unlock(&queue->cmds_lock);
+
+        if (inflight.active) {
+            nvme_release_prps(&inflight);
+            if (inflight.bio) {
+                if (nvme_status_is_success(status))
+                    bio_complete(inflight.bio, 0, inflight.lbas);
+                else
+                    bio_complete(inflight.bio, EIO, 0);
+            }
+            if (inflight.done)
+                atomic_store(inflight.done, true, ATOMIC_RELEASE);
+        }
+
+        completed++;
+    }
+
+    if (completed > 0)
+        ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
+
+    spin_unlock(&queue->cq_lock);
+
+    if (completed > 0)
+        event_set(&queue->slots_free);
+
+    return completed;
+}

dev/block/nvme/spec.h

@@ -1,3 +1,152 @@
 // SPDX-License-Identifier: ISC
 
 #pragma once
+
+#include <zinnia/compiler.h>
+#include <zinnia/mmio.h>
+#include <stddef.h>
+#include <stdint.h>
+
+// Submission queue entry.
+struct [[__packed]] nvme_sqe {
+    uint32_t cdw0;
+
+    uint32_t nsid; // Namespace Identifier
+    uint32_t cdw2;
+    uint32_t cdw3;
+
+    uint64_t mptr; // Metadata Pointer
+
+    uint64_t dptr0; // Data Pointer 0
+    uint64_t dptr1; // Data Pointer 1
+
+    uint32_t cdw10;
+    uint32_t cdw11;
+    uint32_t cdw12;
+    uint32_t cdw13;
+    uint32_t cdw14;
+    uint32_t cdw15;
+};
+static_assert(sizeof(struct nvme_sqe) == 0x40);
+
+constexpr mmio_reg32 NVME_SQE_CDW0 = {offsetof(struct nvme_sqe, cdw0), MMIO_LE};
+constexpr mmio_field32 NVME_SQE_CID = {NVME_SQE_CDW0, 16, 16}; // Command Identifier
+constexpr mmio_field32 NVME_SQE_PSDT = {NVME_SQE_CDW0, 14, 2}; // PRP or SGL for Data Transfer
+constexpr mmio_field32 NVME_SQE_FUSE = {NVME_SQE_CDW0, 8, 2};  // Fused Operation
+constexpr mmio_field32 NVME_SQE_OPC = {NVME_SQE_CDW0, 0, 8};   // Opcode
+
+constexpr mmio_reg32 NVME_SQE_NSID = {offsetof(struct nvme_sqe, nsid), MMIO_LE};   // Namespace Identifier
+constexpr mmio_reg64 NVME_SQE_MPTR = {offsetof(struct nvme_sqe, mptr), MMIO_LE};   // Metadata Pointer
+constexpr mmio_reg64 NVME_SQE_DPTR0 = {offsetof(struct nvme_sqe, dptr0), MMIO_LE}; // Data Pointer 0
+constexpr mmio_reg64 NVME_SQE_DPTR1 = {offsetof(struct nvme_sqe, dptr1), MMIO_LE}; // Data Pointer 1
+
+constexpr mmio_reg32 NVME_SQE_CDW10 = {offsetof(struct nvme_sqe, cdw10), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW11 = {offsetof(struct nvme_sqe, cdw11), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW12 = {offsetof(struct nvme_sqe, cdw12), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW13 = {offsetof(struct nvme_sqe, cdw13), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW14 = {offsetof(struct nvme_sqe, cdw14), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW15 = {offsetof(struct nvme_sqe, cdw15), MMIO_LE};
+
+constexpr uint8_t NVME_CREATE_CQ_OPCODE = 0x05;
+constexpr mmio_field32 NVME_CREATE_CQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
+constexpr mmio_field32 NVME_CREATE_CQ_QID = {NVME_SQE_CDW10, 0, 16};    // Queue Identifier
+constexpr mmio_field32 NVME_CREATE_CQ_IV = {NVME_SQE_CDW11, 16, 16};    // Interrupt Vector
+constexpr mmio_field32 NVME_CREATE_CQ_IEN = {NVME_SQE_CDW11, 1, 1};     // Interrupts Enabled
+constexpr mmio_field32 NVME_CREATE_CQ_PC = {NVME_SQE_CDW11, 0, 1};      // Physically Contiguous
+
+constexpr uint8_t NVME_CREATE_SQ_OPCODE = 0x01;
+constexpr mmio_field32 NVME_CREATE_SQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
+constexpr mmio_field32 NVME_CREATE_SQ_QID = {NVME_SQE_CDW10, 0, 16};    // Queue Identifier
+constexpr mmio_field32 NVME_CREATE_SQ_CQID = {NVME_SQE_CDW11, 16, 16};  // Completion Queue Identifier
+constexpr mmio_field32 NVME_CREATE_SQ_QPRIO = {NVME_SQE_CDW11, 1, 2};   // Queue Priority
+constexpr mmio_field32 NVME_CREATE_SQ_PC = {NVME_SQE_CDW11, 0, 1};      // Physically Contiguous
+
+constexpr mmio_field32 NVME_IDENTIFY_CNTID = {NVME_SQE_CDW10, 16, 16}; // Controller Identifier
+constexpr mmio_field32 NVME_IDENTIFY_CNS = {NVME_SQE_CDW10, 0, 8};     // Controller or Namespace Structure
+
+constexpr mmio_field32 NVME_RW_SLBA_LOW = {NVME_SQE_CDW10, 0, 32};  // Starting LBA Lower Bits
+constexpr mmio_field32 NVME_RW_SLBA_HIGH = {NVME_SQE_CDW11, 0, 32}; // Starting LBA Higher Bits
+constexpr mmio_field32 NVME_RW_LR = {NVME_SQE_CDW12, 31, 1};        // Limited Retry
+constexpr mmio_field32 NVME_RW_FUA = {NVME_SQE_CDW12, 30, 1};       // Force Unit Access
+constexpr mmio_field32 NVME_RW_PRINFO = {NVME_SQE_CDW12, 26, 4};    // Protection Information Field
+constexpr mmio_field32 NVME_RW_NLB = {NVME_SQE_CDW12, 0, 16};       // Number of Logical Blocks
+
+struct [[__packed]] nvme_cqe {
+    uint32_t dw0;
+    uint32_t dw1;
+    uint32_t dw2;
+    uint32_t dw3;
+};
+static_assert(sizeof(struct nvme_cqe) == 0x10);
+
+constexpr mmio_reg32 NVME_CQE_DW0 = {offsetof(struct nvme_cqe, dw0), MMIO_LE};
+constexpr mmio_reg32 NVME_CQE_DW2 = {offsetof(struct nvme_cqe, dw2), MMIO_LE};
+constexpr mmio_field32 NVME_CQE_SQ_IDENT = {NVME_CQE_DW2, 16, 16};
+constexpr mmio_field32 NVME_CQE_SQ_HEAD = {NVME_CQE_DW2, 0, 16};
+constexpr mmio_reg32 NVME_CQE_DW3 = {offsetof(struct nvme_cqe, dw3), MMIO_LE};
+constexpr mmio_field32 NVME_CQE_STATUS = {NVME_CQE_DW3, 17, 15};
+constexpr mmio_field32 NVME_CQE_PHASE_TAG = {NVME_CQE_DW3, 16, 1};
+constexpr mmio_field32 NVME_CQE_CID = {NVME_CQE_DW3, 0, 16};
+
+constexpr mmio_reg64 NVME_REG_CAP = {0x00, MMIO_LE};            // Controller Capabilities
+constexpr mmio_field64 NVME_CAP_MPSMAX = {NVME_REG_CAP, 52, 4}; // Memory Page Size Maximum
+constexpr mmio_field64 NVME_CAP_MPSMIN = {NVME_REG_CAP, 48, 4}; // Memory Page Size Minimum
+constexpr mmio_field64 NVME_CAP_BPS = {NVME_REG_CAP, 45, 1};    // Boot Partition Support
+constexpr mmio_field64 NVME_CAP_CSS = {NVME_REG_CAP, 37, 8};    // Command Sets Supported
+constexpr mmio_field64 NVME_CAP_NSSRS = {NVME_REG_CAP, 36, 1};  // NVM Subsystem Reset Supported
+constexpr mmio_field64 NVME_CAP_DSTRD = {NVME_REG_CAP, 32, 4};  // Doorbell Stride
+constexpr mmio_field64 NVME_CAP_TO = {NVME_REG_CAP, 24, 8};     // Timeout
+constexpr mmio_field64 NVME_CAP_AMS = {NVME_REG_CAP, 17, 2};    // Arbitration Mechanism Supported
+constexpr mmio_field64 NVME_CAP_CQR = {NVME_REG_CAP, 16, 1};    // Contiguous Queues Required
+constexpr mmio_field64 NVME_CAP_MQES = {NVME_REG_CAP, 0, 16};   // Maximum Queue Entries Supported
+
+constexpr mmio_reg32 NVME_REG_VS = {0x08, MMIO_LE};         // Version
+constexpr mmio_field32 NVME_VS_MJR = {NVME_REG_VS, 16, 16}; // Major Version Number
+constexpr mmio_field32 NVME_VS_MNR = {NVME_REG_VS, 8, 8};   // Minor Version Number
+constexpr mmio_field32 NVME_VS_TER = {NVME_REG_VS, 0, 8};   // Tertiary Version Number
+
+constexpr mmio_reg32 NVME_REG_INTMS = {0x0C, MMIO_LE}; // Interrupt Mask Set
+constexpr mmio_reg32 NVME_REG_INTMC = {0x10, MMIO_LE}; // Interrupt Mask Clear
+
+constexpr mmio_reg32 NVME_REG_CC = {0x14, MMIO_LE};           // Controller Configuration
+constexpr mmio_field32 NVME_CC_IOCQES = {NVME_REG_CC, 20, 4}; // I/O Completion Queue Entry Size
+constexpr mmio_field32 NVME_CC_IOSQES = {NVME_REG_CC, 16, 4}; // I/O Submission Queue Entry Size
+constexpr mmio_field32 NVME_CC_SHN = {NVME_REG_CC, 14, 2};    // Shutdown Notification
+constexpr mmio_field32 NVME_CC_AMS = {NVME_REG_CC, 11, 3};    // Arbitration Mechanism Selected
+constexpr mmio_field32 NVME_CC_MPS = {NVME_REG_CC, 7, 4};     // Memory Page Size
+constexpr mmio_field32 NVME_CC_CSS = {NVME_REG_CC, 4, 3};     // I/O Command Set Selected
+constexpr mmio_field32 NVME_CC_EN = {NVME_REG_CC, 0, 1};      // Enable
+
+constexpr mmio_reg32 NVME_REG_CSTS = {0x1C, MMIO_LE};           // Controller Status
+constexpr mmio_field32 NVME_CSTS_PP = {NVME_REG_CSTS, 5, 1};    // Processing Paused
+constexpr mmio_field32 NVME_CSTS_NSSRO = {NVME_REG_CSTS, 4, 1}; // NVM Subsystem Reset Occurred
+constexpr mmio_field32 NVME_CSTS_SHST = {NVME_REG_CSTS, 2, 2};  // Shutdown Status
+constexpr mmio_field32 NVME_CSTS_CFS = {NVME_REG_CSTS, 1, 1};   // Controller Fatal Status
+constexpr mmio_field32 NVME_CSTS_RDY = {NVME_REG_CSTS, 0, 1};   // Ready
+
+constexpr mmio_reg32 NVME_REG_AQA = {0x24, MMIO_LE}; // Admin Queue Attributes
+
+constexpr mmio_field32 NVME_AQA_ACQS = {NVME_REG_AQA, 0, 12};  // Admin Completion Queue Size
+constexpr mmio_field32 NVME_AQA_ASQS = {NVME_REG_AQA, 16, 12}; // Admin Submission Queue Size
+
+constexpr mmio_reg64 NVME_REG_ASQ = {0x28, MMIO_LE}; // Admin Submission Queue
+constexpr mmio_reg64 NVME_REG_ACQ = {0x30, MMIO_LE}; // Admin Completion Queue
+
+// Generic Command Set
+enum nvme_cmd : uint8_t {
+    NVME_CMD_FLUSH = 0x00,
+    NVME_CMD_WRITE = 0x01,
+    NVME_CMD_READ = 0x02,
+};
+
+// Admin Command Set
+enum nvme_admin_cmd : uint8_t {
+    NVME_ADMIN_DELETE_SQ = 0x00,
+    NVME_ADMIN_CREATE_SQ = 0x01,
+    NVME_ADMIN_DELETE_CQ = 0x04,
+    NVME_ADMIN_CREATE_CQ = 0x05,
+    NVME_ADMIN_IDENTIFY = 0x06,
+    NVME_ADMIN_ABORT = 0x08,
+    NVME_ADMIN_SET_FEATURES = 0x09,
+    NVME_ADMIN_GET_FEATURES = 0x0A,
+};

dev/pci/core.c

@@ -1,12 +1,21 @@
+// SPDX-License-Identifier: ISC
+
 #define pr_fmt(fmt) "pci: " fmt
 
 #include <zinnia/alloc.h>
 #include <zinnia/init.h>
+#include <zinnia/mmio.h>
 #include <zinnia/module.h>
 #include <zinnia/pci.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/spin.h>
+#include <zinnia/utils.h>
 #include <zinnia/vec.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
 #include <string.h>
 
 static struct spinlock access_lock = {};
@@ -168,6 +177,119 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) {
     return 0;
 }
 
+constexpr uint32_t PCI_MSI_CTRL_ENABLE = 1 << 16;
+constexpr uint32_t PCI_MSI_CTRL_MME = 0x7 << 20;
+constexpr uint32_t PCI_MSI_CTRL_64BIT = 1 << 23;
+constexpr uint32_t PCI_MSIX_CTRL_FUNCTION_MASK = 1 << 30;
+constexpr uint32_t PCI_MSIX_CTRL_ENABLE = 1u << 31;
+
+static void msix_set_state(struct pci_device* dev, uint8_t cap, bool status) {
+    uint32_t ctrl = pci_read32(dev, cap);
+    ctrl &= ~(PCI_MSIX_CTRL_ENABLE | PCI_MSIX_CTRL_FUNCTION_MASK);
+    if (status)
+        ctrl |= PCI_MSIX_CTRL_ENABLE;
+    pci_write32(dev, cap, ctrl);
+}
+
+errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out) {
+    // message address + data + vector control
+    constexpr size_t ENTRY_SIZE = 8 + 4 + 4;
+
+    const uint8_t cap = pci_cap_find(dev, PCI_CAP_MSIX);
+    if (cap == 0)
+        return ENODEV;
+
+    const uint32_t table = pci_read32(dev, cap + 4);
+    const uint8_t bir = table & 0x7;
+    const size_t table_offset = table & ~0x7;
+
+    pci_log(dev, "Using MSI-X for this device, table at BAR%hhu + %#zx\n", bir, table_offset);
+
+    // Map the first MSI-X table entry (16 bytes).
+    struct pci_bar bar;
+    errno_t status = pci_bar_get(dev, bir, &bar);
+    if (status)
+        return ENODEV;
+    if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
+        return EINVAL;
+
+    const size_t page_size = arch_mem_page_size();
+    const phys_t table_phys = bar.address + table_offset;
+    const phys_t page_base = ALIGN_DOWN(table_phys, page_size);
+    const size_t page_offset = table_phys - page_base;
+    const size_t map_len = ALIGN_UP(page_offset + ENTRY_SIZE, page_size);
+
+    uintptr_t vaddr = vm_alloc(map_len);
+    if (!vaddr)
+        return ENOMEM;
+    for (size_t i = 0; i < map_len; i += page_size) {
+        status = pmap_map(&kernel_vma.pmap, vaddr + i, page_base + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status)
+            return status;
+    }
+    void* entry = (void*)(vaddr + page_offset);
+
+    struct msi_line* msi;
+    status = arch_irq_alloc_msi(&msi);
+    if (status)
+        return status;
+    irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
+
+    // Write entry 0.
+    const uint64_t msg_addr = msi->msg_addr(msi);
+    const uint32_t msg_data = msi->msg_data(msi);
+    mmio_write32(entry + 0x00, (uint32_t)msg_addr);         // Address lo
+    mmio_write32(entry + 0x04, (uint32_t)(msg_addr >> 32)); // Address hi
+    mmio_write32(entry + 0x08, msg_data);                   // Data
+    mmio_write32(entry + 0x0C, 0);                          // Vector control
+
+    // Enable MSI-X on the device.
+    msix_set_state(dev, cap, true);
+
+    *out = &msi->line;
+    return 0;
+}
+
+errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out) {
+    uint8_t cap = pci_cap_find(dev, PCI_CAP_MSI);
+    if (cap == 0)
+        return ENODEV;
+
+    pci_log(dev, "Using MSI for this device\n");
+
+    struct msi_line* msi;
+    errno_t status = arch_irq_alloc_msi(&msi);
+    if (status)
+        return status;
+    irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
+
+    const uint64_t addr = msi->msg_addr(msi);
+    const uint16_t data = msi->msg_data(msi);
+
+    uint32_t ctrl = pci_read32(dev, cap);
+    pci_write32(dev, cap + 0x04, (uint32_t)addr);
+    uint16_t data_off = 0x08;
+    if (ctrl & PCI_MSI_CTRL_64BIT) {
+        pci_write32(dev, cap + 0x08, (uint32_t)(addr >> 32));
+        data_off = 0x0C;
+    }
+    pci_write32(dev, cap + data_off, data);
+
+    ctrl &= ~PCI_MSI_CTRL_MME;
+    ctrl |= PCI_MSI_CTRL_ENABLE;
+    pci_write32(dev, cap, ctrl);
+
+    *out = &msi->line;
+    return 0;
+}
+
+errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out) {
+    errno_t status = pci_setup_msix(dev, out);
+    if (status == ENODEV)
+        return pci_setup_msi(dev, out);
+    return status;
+}
+
 static void pci_device_name(const struct pci_device* dev, char buf[static 18]) {
     ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
 }
@@ -190,6 +312,7 @@ static bool scan_function(struct pci_access* access, struct pci_address addr) {
 
     dev->address = addr;
     dev->access = access;
+    dev->dma_tag = &dma_default_tag;
     dev->id.vendor_id = vendor_id;
     dev->id.device_id = device_id;
     dev->class.prog_if = pci_read8(dev, PCI_PROG_IF);

dev/video/fbcon/fbcon.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/console.h>

include/arch/irq.h

@@ -8,3 +8,10 @@ bool arch_irq_set_state(bool state);
 bool arch_irq_get_state();
 void arch_irq_wait();
 errno_t arch_irq_alloc_msi(struct msi_line** out);
+errno_t arch_irq_get_gsi(uint32_t gsi, struct irq_line** out);
+bool arch_irq_isa_override(
+    uint8_t isa_irq,
+    uint32_t* out_gsi,
+    enum irq_trigger_mode* out_trigger,
+    enum irq_polarity* out_polarity
+);

include/string.h

@@ -12,3 +12,4 @@ void* memset(void* dest, int value, size_t n);
 char* strncpy(char* restrict dst, const char* restrict src, size_t len);
 int strcmp(const char* str1, const char* str2);
 int strncmp(const char* str1, const char* str2, size_t len);
+char* strchr(const char* s, int c);

include/zinnia/alloc.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zinnia/boot.h>
 #include <zinnia/errno.h>
 #include <zinnia/initgraph.h>
 #include <zinnia/types.h>
@@ -14,19 +15,6 @@ enum alloc_flags {
     ALLOC_MEM20 = 1 << 2,  // Allocated memory needs to fit inside 20 bits.
 };
 
-enum phys_mem_usage {
-    PHYS_RESERVED,
-    PHYS_USABLE,
-    PHYS_STATIC,
-    PHYS_RECLAIMABLE,
-};
-
-struct phys_mem {
-    phys_t address;
-    size_t length;
-    enum phys_mem_usage usage;
-};
-
 // Base address of the HHDM.
 extern uintptr_t mem_hhdm_base;
 

include/zinnia/boot.h

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <bits/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+enum phys_mem_usage {
+    PHYS_RESERVED,
+    PHYS_USABLE,
+    PHYS_STATIC,
+    PHYS_RECLAIMABLE,
+};
+
+struct phys_mem {
+    phys_t address;
+    size_t length;
+    enum phys_mem_usage usage;
+};
+
+constexpr size_t BOOT_PATH_MAX = 128;
+
+struct boot_file {
+    phys_t data;
+    size_t length;
+    char path[BOOT_PATH_MAX];
+};
+
+struct boot_color {
+    uint8_t shift, size;
+};
+
+struct boot_framebuffer {
+    phys_t base;
+    size_t width, height, pitch;
+    uint8_t bytes_per_pixel;
+    struct boot_color red, green, blue;
+};
+
+constexpr size_t BOOT_LOADER_NAME_MAX = 64;
+constexpr size_t BOOT_CMDLINE_MAX = 4096;
+constexpr size_t BOOT_MEM_MAP_MAX = 128;
+constexpr size_t BOOT_FILES_MAX = 8;
+
+struct boot_info {
+    char loader[BOOT_LOADER_NAME_MAX];
+    char cmdline[BOOT_CMDLINE_MAX];
+    struct phys_mem mem_map[BOOT_MEM_MAP_MAX];
+    size_t num_mem_maps;
+    phys_t phys_base;
+    uintptr_t virt_base;
+    uintptr_t hhdm_base;
+    struct boot_file files[BOOT_FILES_MAX];
+    size_t num_files;
+    struct boot_framebuffer fb;
+    phys_t dtb;
+    size_t dtb_size;
+    phys_t rsdp;
+};

include/zinnia/dma.h

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/iovec.h>
+#include <zinnia/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+typedef uint64_t dma_addr_t;
+
+constexpr dma_addr_t DMA_ADDR_MAX = UINT64_MAX;
+
+struct bio;
+
+struct dma_tag;
+
+struct dma_ops {
+    errno_t (*map)(struct dma_tag* tag, phys_t phys, size_t len, dma_addr_t* out);
+    void (*unmap)(struct dma_tag* tag, dma_addr_t addr, size_t len);
+};
+
+struct dma_tag {
+    const struct dma_ops* ops;
+    void* priv;
+    dma_addr_t max_addr;
+};
+
+extern struct dma_tag dma_default_tag;
+
+void dma_tag_subregion(struct dma_tag* parent, dma_addr_t max_addr, struct dma_tag* out);
+
+struct dma_seg {
+    dma_addr_t addr;
+    size_t len;
+};
+
+struct dma_buffer {
+    struct dma_tag* tag;
+    phys_t phys;
+    dma_addr_t addr;
+    void* virt;
+    size_t len;
+};
+
+errno_t dma_buffer_alloc(struct dma_tag* tag, size_t len, struct dma_buffer* out);
+void dma_buffer_free(struct dma_buffer* buf);
+
+struct dma_map {
+    struct dma_tag* tag;
+    size_t size;
+    size_t max_segs;
+    size_t max_seg_size;
+    size_t boundary;
+    size_t mapped_len;
+    size_t num_segs;
+    struct dma_seg segs[];
+};
+
+enum dma_sync {
+    DMA_SYNC_PREREAD = 1 << 0,
+    DMA_SYNC_POSTREAD = 1 << 1,
+    DMA_SYNC_PREWRITE = 1 << 2,
+    DMA_SYNC_POSTWRITE = 1 << 3,
+};
+
+errno_t dma_map_create(
+    struct dma_tag* tag,
+    size_t size,
+    size_t max_segs,
+    size_t max_seg_size,
+    size_t boundary,
+    struct dma_map** out
+);
+void dma_map_destroy(struct dma_map* map);
+
+errno_t dma_map_load(struct dma_map* map, void* buf, size_t len);
+errno_t dma_map_load_bio(struct dma_map* map, struct bio* bio);
+errno_t dma_map_load_iter(struct dma_map* map, const iovec_iter_t* iter);
+void dma_map_unload(struct dma_map* map);
+
+void dma_map_sync(struct dma_map* map, size_t offset, size_t len, enum dma_sync ops);
+void dma_buffer_sync(struct dma_buffer* buf, size_t offset, size_t len, enum dma_sync ops);

include/zinnia/elf.h

@@ -2,8 +2,6 @@
 
 #pragma once
 
-#include <zinnia/errno.h>
-#include <zinnia/vfs.h>
 #include <arch/elf.h>
 #include <stdint.h>
 
@@ -84,6 +82,7 @@ enum : uint32_t {
     PT_MOD_CMDLINE = 0x60000005,
     PT_MOD_EXIT = 0x60000006,
     PT_MOD_MATCH = 0x60000007,
+    PT_BOOT_INFO = 0x60000008,
 };
 
 enum : uint16_t {

include/zinnia/init.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zinnia/boot.h>
 #include <zinnia/compiler.h>
 #include <zinnia/types.h>
 #include <zinnia/utils.h>
@@ -13,40 +14,7 @@
 #define __initdata              __used, __section(".init.data")
 #define __initdata_sorted(name) __used, __section(".init.data." name)
 
-struct boot_file {
-    phys_t data;
-    size_t length;
-    char path[128];
-};
-
-struct boot_color {
-    uint8_t shift, size;
-};
-
-struct boot_framebuffer {
-    phys_t base;
-    size_t width, height, pitch;
-    uint8_t bytes_per_pixel;
-    struct boot_color red, green, blue;
-};
-
-struct boot_info {
-    char* cmdline;
-    struct phys_mem* mem_map;
-    size_t num_mem_maps;
-    phys_t phys_base;
-    uintptr_t virt_base;
-    uintptr_t hhdm_base;
-    struct boot_file* files;
-    size_t num_files;
-    struct boot_framebuffer fb;
-    struct boot_file kernel_file;
-    phys_t dtb;
-    size_t dtb_size;
-#if CONFIG_ACPI
-    phys_t rsdp;
-#endif
-};
+extern struct boot_info boot_info;
 
 [[__visibility("hidden")]]
 extern uint8_t __ld_stack_top[];

include/zinnia/irq.h

@@ -12,6 +12,7 @@
 enum irq_status : uint8_t {
     IRQ_STATUS_IGNORED,
     IRQ_STATUS_HANDLED,
+    IRQ_STATUS_WAKE_THREAD,
 };
 
 enum irq_polarity : uint8_t {
@@ -29,9 +30,16 @@ enum irq_mode : uint8_t {
     IRQ_MODE_MASKABLE = 1 << 1,
 };
 
+struct irq_line;
+struct irq_task;
+
 struct irq_handler {
     enum irq_status (*raise)(struct irq_handler* self);
+    void (*thread)(struct irq_handler* self);
     list_link(struct irq_handler*) next;
+
+    struct irq_line* line;
+    _Atomic(bool) need;
 };
 
 struct irq_line {
@@ -43,6 +51,8 @@ struct irq_line {
     bool is_busy;
     list_head(struct irq_handler*) handlers;
     enum irq_mode mode;
+    enum irq_trigger_mode trigger;
+    struct irq_task* task;
 };
 
 struct msi_line {
@@ -61,6 +71,8 @@ extern struct irq_percpu irq_data;
 void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity);
 void irq_line_raise(struct irq_line* line);
 void irq_line_attach(struct irq_line* line, struct irq_handler* handler);
+void irq_line_detach(struct irq_line* line, struct irq_handler* handler);
+errno_t irq_line_attach_task(struct irq_line* line, struct irq_handler* handler, const char* name);
 
 void irq_lock();
 void irq_unlock();

include/zinnia/ktest.h

@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/compiler.h>
+#include <zinnia/utils.h>
+#include <stddef.h> // IWYU pragma: keep
+#include <stdint.h> // IWYU pragma: keep
+
+struct ktest {
+    const char* suite;
+    const char* name;
+    bool failed;
+    bool skipped;
+    const char* skip_reason;
+};
+
+struct ktest_case {
+    const char* suite;
+    const char* name;
+    void (*func)(struct ktest* test);
+};
+
+#define KTEST(suite_name, case_name) \
+    static void ktest_##suite_name##_##case_name(struct ktest* test); \
+    [[__used, __section(".ktest"), gnu::aligned(8)]] \
+    static const struct ktest_case ktest_##suite_name##_##case_name##_case = { \
+        .suite = #suite_name, \
+        .name = #case_name, \
+        .func = ktest_##suite_name##_##case_name, \
+    }; \
+    static void ktest_##suite_name##_##case_name(struct ktest* test)
+
+[[gnu::format(printf, 4, 5)]]
+void ktest_fail(struct ktest* test, const char* file, int line, const char* fmt, ...);
+
+void ktest_main();
+
+#define _KTEST_CHECK(cond, on_fail, ...) \
+    do { \
+        if (!(cond)) { \
+            ktest_fail(test, __FILE__, __LINE__, __VA_ARGS__); \
+            on_fail; \
+        } \
+    } while (0)
+
+#define _KTEST_CMP(a, op, b, on_fail) \
+    do { \
+        const typeof(true ? (a) : (b)) _ktest_a = (a); \
+        const typeof(true ? (a) : (b)) _ktest_b = (b); \
+        _KTEST_CHECK( \
+            _ktest_a op _ktest_b, \
+            on_fail, \
+            "expected %s %s %s, got %#llx and %#llx", \
+            #a, \
+            #op, \
+            #b, \
+            (unsigned long long)_ktest_a, \
+            (unsigned long long)_ktest_b \
+        ); \
+    } while (0)
+
+#define _KTEST_STREQ(a, b, on_fail) \
+    do { \
+        const char* _ktest_a = (a); \
+        const char* _ktest_b = (b); \
+        _KTEST_CHECK( \
+            _ktest_a && _ktest_b && !strcmp(_ktest_a, _ktest_b), \
+            on_fail, \
+            "expected %s == %s, got \"%s\" and \"%s\"", \
+            #a, \
+            #b, \
+            _ktest_a ? _ktest_a : "(null)", \
+            _ktest_b ? _ktest_b : "(null)" \
+        ); \
+    } while (0)
+
+#define KTEST_EXPECT_TRUE(cond)    _KTEST_CHECK(cond, (void)0, "expected %s", #cond)
+#define KTEST_EXPECT_FALSE(cond)   _KTEST_CHECK(!(cond), (void)0, "expected !(%s)", #cond)
+#define KTEST_EXPECT_EQ(a, b)      _KTEST_CMP(a, ==, b, (void)0)
+#define KTEST_EXPECT_NE(a, b)      _KTEST_CMP(a, !=, b, (void)0)
+#define KTEST_EXPECT_LT(a, b)      _KTEST_CMP(a, <, b, (void)0)
+#define KTEST_EXPECT_LE(a, b)      _KTEST_CMP(a, <=, b, (void)0)
+#define KTEST_EXPECT_GT(a, b)      _KTEST_CMP(a, >, b, (void)0)
+#define KTEST_EXPECT_GE(a, b)      _KTEST_CMP(a, >=, b, (void)0)
+#define KTEST_EXPECT_STREQ(a, b)   _KTEST_STREQ(a, b, (void)0)
+#define KTEST_EXPECT_NULL(ptr)     _KTEST_CHECK((ptr) == nullptr, (void)0, "expected %s to be null", #ptr)
+#define KTEST_EXPECT_NOT_NULL(ptr) _KTEST_CHECK((ptr) != nullptr, (void)0, "expected %s to be non-null", #ptr)
+
+#define KTEST_ASSERT_TRUE(cond)    _KTEST_CHECK(cond, return, "expected %s", #cond)
+#define KTEST_ASSERT_FALSE(cond)   _KTEST_CHECK(!(cond), return, "expected !(%s)", #cond)
+#define KTEST_ASSERT_EQ(a, b)      _KTEST_CMP(a, ==, b, return)
+#define KTEST_ASSERT_NE(a, b)      _KTEST_CMP(a, !=, b, return)
+#define KTEST_ASSERT_LT(a, b)      _KTEST_CMP(a, <, b, return)
+#define KTEST_ASSERT_LE(a, b)      _KTEST_CMP(a, <=, b, return)
+#define KTEST_ASSERT_GT(a, b)      _KTEST_CMP(a, >, b, return)
+#define KTEST_ASSERT_GE(a, b)      _KTEST_CMP(a, >=, b, return)
+#define KTEST_ASSERT_STREQ(a, b)   _KTEST_STREQ(a, b, return)
+#define KTEST_ASSERT_NULL(ptr)     _KTEST_CHECK((ptr) == nullptr, return, "expected %s to be null", #ptr)
+#define KTEST_ASSERT_NOT_NULL(ptr) _KTEST_CHECK((ptr) != nullptr, return, "expected %s to be non-null", #ptr)
+
+#define KTEST_SKIP(reason) \
+    do { \
+        (test)->skipped = true; \
+        (test)->skip_reason = (reason); \
+        return; \
+    } while (0)

include/zinnia/pci.h

@@ -4,6 +4,7 @@
 
 #include <zinnia/alloc.h>
 #include <zinnia/compiler.h>
+#include <zinnia/dma.h>
 #include <zinnia/errno.h>
 #include <zinnia/initgraph.h>
 #include <zinnia/print.h>
@@ -106,6 +107,7 @@ struct pci_device {
     struct pci_access* access;
     uint8_t header_type;
     struct pci_driver* driver;
+    struct dma_tag* dma_tag;
 };
 
 uint8_t pci_read8(struct pci_device* dev, uint16_t offset);
@@ -137,6 +139,12 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id);
 constexpr uint16_t PCI_ANY_ID = 0xffff;
 constexpr uint8_t PCI_ANY_CLASS = 0xff;
 
+struct irq_line;
+
+errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out);
+errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out);
+errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out);
+
 // Matches a device by ID and class.
 struct pci_match {
     uint16_t vendor_id;

include/zinnia/process.h

@@ -4,7 +4,6 @@
 
 #include <zinnia/atomic.h>
 #include <zinnia/errno.h>
-#include <zinnia/hashmap.h>
 #include <zinnia/ident.h>
 #include <zinnia/mutex.h>
 #include <zinnia/refcount.h>
@@ -26,9 +25,12 @@ struct process_rlimit {
     rlim_t max;
 };
 
-struct fd_entry {
-    struct vfs_file* file;
-    bool cloexec;
+struct fd_table {
+    struct vfs_file** files;
+    uint64_t* open;
+    uint64_t* cloexec;
+    size_t capacity;
+    size_t next_free;
 };
 
 struct process {
@@ -50,7 +52,7 @@ struct process {
     struct process_rlimit limit_nofile;
 
     mutex_t files_lock;
-    hashmap(int, struct fd_entry) files;
+    struct fd_table files;
 };
 
 struct process* process_kernel();

include/zinnia/sched.h

@@ -102,6 +102,7 @@ struct scheduler {
 
     struct timer preempt_timer;
     bool need_resched;
+    bool owe_preempt;
     uint64_t slice_start;
     uint64_t boost_epoch;
 };
@@ -132,6 +133,8 @@ void sched_wake_locked(struct task* task);
 
 void sched_notify(struct task* task);
 
+void sched_sleep_until(uint64_t deadline_ns);
+
 [[noreturn]]
 void sched_exit();
 

include/zinnia/semaphore.h

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+typedef struct {
+    _Atomic(uint32_t) count;
+} sem_t;
+
+void sem_init(sem_t* sem, uint32_t count);
+bool sem_trywait(sem_t* sem);
+bool sem_timedwait(sem_t* sem, const uint64_t* deadline_ns);
+void sem_wait(sem_t* sem);
+void sem_post(sem_t* sem);

lib/console.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/print.h>

lib/entry.c

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/boot.h>
+#include <zinnia/compiler.h>
+#include <zinnia/fdt.h>
+#include <zinnia/init.h>
+
+[[__section(".boot.info")]]
+struct boot_info boot_info;
+
+[[__init]]
+void kernel_entry() {
+    kernel_early_init();
+
+    assert(boot_info.num_mem_maps, "Loader did not provide a memory map");
+
+    if (boot_info.dtb)
+        boot_info.dtb_size = fdt_read_u32((const uint8_t*)(boot_info.hhdm_base + boot_info.dtb) + 4);
+
+    kernel_main(&boot_info);
+}

lib/irq.c

@@ -1,8 +1,17 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
 #include <zinnia/assert.h>
+#include <zinnia/atomic.h>
 #include <zinnia/irq.h>
 #include <zinnia/list.h>
+#include <zinnia/mutex.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <arch/cpu.h>
 #include <arch/irq.h>
 
 [[__percpu]]
@@ -29,16 +38,64 @@ void irq_lock() {
 
 void irq_unlock() {
     struct irq_percpu* state = percpu_ref(irq_data);
-    if (--state->depth == 0 && state->saved_state)
+    if (--state->depth == 0 && state->saved_state) {
+        struct scheduler* sched = &percpu_get()->sched;
+        if (sched->owe_preempt && !sched->preempt_level && sched->current)
+            sched_reschedule(sched);
         arch_irq_set_state(true);
+    }
 }
 
 void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
     irq_lock();
     line->mode = line->set_config(line, trigger, polarity);
+    line->trigger = trigger;
     irq_unlock();
 }
 
+struct irq_task {
+    struct irq_line* line;
+    struct task* task;
+    _Atomic(bool) need;
+    _Atomic(bool) masked;
+    _Atomic(bool) running;
+};
+
+static mutex_t irq_task_lock;
+
+static bool holds_mask(struct irq_line* line) {
+    return line->trigger == IRQ_TRIGGER_LEVEL && (line->mode & IRQ_MODE_MASKABLE) && line->mask && line->unmask;
+}
+
+static void irq_task_fn(uintptr_t arg) {
+    struct irq_task* it = (struct irq_task*)arg;
+    struct irq_line* line = it->line;
+
+    while (true) {
+        atomic_store(&it->running, true, ATOMIC_SEQ_CST);
+        while (atomic_exchange(&it->need, false, ATOMIC_ACQ_REL)) {
+            struct irq_handler* handler;
+            list_foreach(handler, &line->handlers, next) {
+                if (atomic_exchange(&handler->need, false, ATOMIC_ACQ_REL))
+                    handler->thread(handler);
+            }
+            if (atomic_exchange(&it->masked, false, ATOMIC_ACQ_REL))
+                line->unmask(line);
+        }
+        atomic_store(&it->running, false, ATOMIC_SEQ_CST);
+
+        if (!atomic_load(&it->need, ATOMIC_ACQUIRE))
+            sched_block(&percpu_get()->sched);
+    }
+}
+
+static void irq_task_schedule(struct irq_task* it) {
+    if (holds_mask(it->line) && !atomic_exchange(&it->masked, true, ATOMIC_ACQ_REL))
+        it->line->mask(it->line);
+    atomic_store(&it->need, true, ATOMIC_RELEASE);
+    sched_wake(it->task);
+}
+
 void irq_line_raise(struct irq_line* line) {
     assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
 
@@ -46,12 +103,21 @@ void irq_line_raise(struct irq_line* line) {
     line->is_busy = true;
 
     bool claimed = false;
+    bool wake = false;
     struct irq_handler* handler;
     list_foreach(handler, &line->handlers, next) {
-        if (handler->raise(handler) == IRQ_STATUS_HANDLED)
+        enum irq_status status = handler->raise ? handler->raise(handler) : IRQ_STATUS_WAKE_THREAD;
+        if (status == IRQ_STATUS_WAKE_THREAD && handler->thread && line->task) {
+            atomic_store(&handler->need, true, ATOMIC_RELEASE);
+            wake = true;
             claimed = true;
+        } else if (status != IRQ_STATUS_IGNORED) {
+            claimed = true;
+        }
     }
 
+    if (wake)
+        irq_task_schedule(line->task);
     if (!claimed)
         log("irq: Spurious interrupt\n");
 
@@ -63,7 +129,58 @@ void irq_line_raise(struct irq_line* line) {
 }
 
 void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
+    handler->line = line;
     irq_lock();
     list_insert_head(&line->handlers, handler, next);
     irq_unlock();
 }
+
+void irq_line_detach(struct irq_line* line, struct irq_handler* handler) {
+    irq_lock();
+    struct irq_handler** link = &list_first(&line->handlers);
+    while (*link && *link != handler)
+        link = &list_next(*link, next);
+    if (*link)
+        *link = list_next(handler, next);
+    irq_unlock();
+
+    while (__atomic_load_n(&line->is_busy, __ATOMIC_ACQUIRE))
+        arch_cpu_pause();
+
+    atomic_exchange(&handler->need, false, ATOMIC_SEQ_CST);
+    if (line->task) {
+        while (atomic_load(&line->task->running, ATOMIC_SEQ_CST))
+            sched_reschedule(&percpu_get()->sched);
+    }
+}
+
+errno_t irq_line_attach_task(struct irq_line* line, struct irq_handler* handler, const char* name) {
+    if (!handler->thread)
+        return EINVAL;
+
+    mutex_lock(&irq_task_lock);
+    if (!line->task) {
+        struct irq_task* it = mem_alloc(sizeof(*it), 0);
+        if (!it) {
+            mutex_unlock(&irq_task_lock);
+            return ENOMEM;
+        }
+        it->line = line;
+
+        char task_name[TASK_NAME_MAX];
+        ksnprintf(task_name, sizeof(task_name), "irq/%s", name);
+        errno_t status = task_create(task_name, process_kernel(), irq_task_fn, (uintptr_t)it, &it->task);
+        if (status) {
+            mem_free(it);
+            mutex_unlock(&irq_task_lock);
+            return status;
+        }
+        sched_set_priority(it->task, SCHED_PRIO_IRQ);
+        sched_add_task(&percpu_get()->sched, it->task);
+        line->task = it;
+    }
+    mutex_unlock(&irq_task_lock);
+
+    irq_line_attach(line, handler);
+    return 0;
+}

lib/main.c

@@ -11,6 +11,7 @@
 #include <zinnia/initgraph.h>
 #include <zinnia/initramfs.h>
 #include <zinnia/irq.h>
+#include <zinnia/ktest.h>
 #include <zinnia/module.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
@@ -48,6 +49,10 @@ static void kernel_main_task(uintptr_t arg0) {
     // Load all initramfs files.
     initramfs_load_boot_modules(info);
 
+#if CONFIG_KTEST
+    ktest_main();
+#endif
+
     const char* init_argv[] = {init_path, nullptr};
     const char* init_envp[] = {"HOME=/", "TERM=console", nullptr};
 
@@ -75,6 +80,7 @@ void kernel_main(struct boot_info* info) {
     cmdline_parse(info->cmdline);
 
     log("%s\n", zinnia_banner); // Say hello!
+    log("Boot loader: %s\n", info->loader);
     log("Command line: %s\n", info->cmdline);
 
     initgraph_run(sched_stage, info);

lib/meson.build

@@ -1,13 +1,16 @@
 kernel_sources += files(
     'clock.c',
+    'cmdline.c',
     'condvar.c',
     'console.c',
+    'entry.c',
     'event.c',
     'guid.c',
     'initgraph.c',
     'iovec.c',
     'irq.c',
     'ksyms.c',
+    'main.c',
     'module.c',
     'mutex.c',
     'panic.c',
@@ -20,6 +23,7 @@ kernel_sources += files(
     'ringbuf.c',
     'runtime.c',
     'rwlock.c',
+    'semaphore.c',
     'spin.c',
     'stdlib.c',
     'string.c',

lib/print.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/clock.h>
 #include <zinnia/console.h>
 #include <zinnia/panic.h>
@@ -232,6 +234,12 @@ check_fmt:
                 lutoa(va_arg(args, uint64_t), number, 16);
                 break;
             }
+            if (*fmt == 'X') {
+                for (char* c = number; *c; c++) {
+                    if (*c >= 'a' && *c <= 'f')
+                        *c -= 'a' - 'A';
+                }
+            }
 print_num:
             const char* digits = number;
             char sign = 0;
@@ -258,7 +266,7 @@ print_num:
             if (sign)
                 callback(ctx, &sign, 1);
             if (write_prefix)
-                callback(ctx, "0x", 2);
+                callback(ctx, *fmt == 'X' ? "0X" : "0x", 2);
             print_pad(callback, ctx, '0', zeros);
             callback(ctx, digits, len);
             if (left_align)
@@ -283,7 +291,7 @@ print_num:
             callback(ctx, number, len);
 
             fmt++;
-            written += len;
+            written += offset + len;
             break;
         }
         // The number of characters written so far is stored in the pointed location.

lib/process.c

@@ -3,7 +3,6 @@
 #include <zinnia/alloc.h>
 #include <zinnia/atomic.h>
 #include <zinnia/exec.h>
-#include <zinnia/hashmap.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <zinnia/process.h>
@@ -11,18 +10,103 @@
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
 #include <zinnia/vmo.h>
+#include <string.h>
+
+constexpr size_t PROCESS_INITIAL_FDS = 64;
 
 static struct process kernel_process;
 static atomic_bool kernel_process_ready;
 static atomic_int next_pid = 1;
 
+static size_t fd_words(size_t capacity) {
+    return capacity / 64;
+}
+
+static void fd_table_destroy(struct fd_table* t) {
+    mem_free(t->files);
+    mem_free(t->open);
+    mem_free(t->cloexec);
+    *t = (struct fd_table){};
+}
+
+static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
+    struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0);
+    uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0);
+    uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0);
+    if (!files || !open || !cloexec) {
+        mem_free(files);
+        mem_free(open);
+        mem_free(cloexec);
+        return ENOMEM;
+    }
+
+    if (t->capacity) {
+        memcpy(files, t->files, t->capacity * sizeof(*files));
+        memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open));
+        memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec));
+    }
+    mem_free(t->files);
+    mem_free(t->open);
+    mem_free(t->cloexec);
+
+    t->files = files;
+    t->open = open;
+    t->cloexec = cloexec;
+    t->capacity = capacity;
+    return 0;
+}
+
+static errno_t fd_table_grow(struct fd_table* t, size_t fd) {
+    if (fd < t->capacity)
+        return 0;
+    size_t capacity = MAX(PROCESS_INITIAL_FDS, t->capacity);
+    while (capacity <= fd)
+        capacity *= 2;
+    return fd_table_resize(t, MIN(capacity, PROCESS_MAX_FDS));
+}
+
+static size_t fd_find_free(struct fd_table* t, size_t start) {
+    for (size_t w = start / 64; w < fd_words(t->capacity); w++) {
+        uint64_t used = t->open[w];
+        if (w == start / 64)
+            used |= (1ull << (start % 64)) - 1;
+        if (~used)
+            return w * 64 + __builtin_ctzll(~used);
+    }
+    return MAX(start, t->capacity);
+}
+
+static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+    const uint64_t bit = 1ull << (fd % 64);
+    t->files[fd] = file;
+    t->open[fd / 64] |= bit;
+    if (cloexec)
+        t->cloexec[fd / 64] |= bit;
+    else
+        t->cloexec[fd / 64] &= ~bit;
+}
+
+static void fd_clear(struct fd_table* t, size_t fd) {
+    const uint64_t bit = 1ull << (fd % 64);
+    t->files[fd] = nullptr;
+    t->open[fd / 64] &= ~bit;
+    t->cloexec[fd / 64] &= ~bit;
+    if (fd < t->next_free)
+        t->next_free = fd;
+}
+
+static struct vfs_file* fd_lookup(struct process* proc, int fd) {
+    if (fd < 0 || (size_t)fd >= proc->files.capacity)
+        return nullptr;
+    return proc->files.files[fd];
+}
+
 static void process_init_common(struct process* proc) {
     refcount_init(&proc->rc);
     spin_init(&proc->tasks_lock);
     tailq_init(&proc->tasks);
     mutex_init(&proc->lock);
     mutex_init(&proc->files_lock);
-    hashmap_init(&proc->files);
     vec_init(&proc->identity.groups);
 }
 
@@ -37,17 +121,13 @@ struct process* process_kernel() {
 }
 
 static void close_files(struct process* proc) {
+    struct fd_table* t = &proc->files;
     mutex_lock(&proc->files_lock);
-    size_t i = 0;
-    while (i < proc->files.capacity) {
-        struct fd_entry entry = proc->files.values[i];
-        if (proc->files.hashes[i] == 0) {
-            i++;
-            continue;
-        }
-        hashmap_remove(&proc->files, proc->files.keys[i], hashmap_hash_default, hashmap_eq_default);
-        vfs_file_unref(entry.file);
+    for (size_t w = 0; w < fd_words(t->capacity); w++) {
+        for (uint64_t bits = t->open[w]; bits; bits &= bits - 1)
+            vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]);
     }
+    fd_table_destroy(t);
     mutex_unlock(&proc->files_lock);
 }
 
@@ -64,7 +144,6 @@ void process_unref(struct process* proc) {
     if (proc->space)
         vma_unref(proc->space);
     vec_destroy(&proc->identity.groups);
-    hashmap_destroy(&proc->files);
     mem_free(proc);
 }
 
@@ -192,10 +271,6 @@ errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloex
     return process_fd_alloc_min(proc, file, 0, cloexec, out_fd);
 }
 
-static struct fd_entry* fd_lookup(struct process* proc, int fd) {
-    return hashmap_get(&proc->files, fd, hashmap_hash_default, hashmap_eq_default);
-}
-
 size_t process_fd_limit(struct process* proc) {
     return MIN((size_t)PROCESS_MAX_FDS, (size_t)atomic_load(&proc->limit_nofile.cur, ATOMIC_RELAXED));
 }
@@ -204,22 +279,22 @@ errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int mi
     if (min_fd < 0 || (size_t)min_fd >= process_fd_limit(proc))
         return EINVAL;
 
+    struct fd_table* t = &proc->files;
     mutex_lock(&proc->files_lock);
-    int fd = min_fd;
-    while ((size_t)fd < process_fd_limit(proc) && fd_lookup(proc, fd))
-        fd++;
-
-    if ((size_t)fd >= process_fd_limit(proc)) {
+    const size_t fd = fd_find_free(t, MAX((size_t)min_fd, t->next_free));
+    if (fd >= process_fd_limit(proc)) {
         mutex_unlock(&proc->files_lock);
         return EMFILE;
     }
 
-    const struct fd_entry entry = {.file = file, .cloexec = cloexec};
-    errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default);
+    errno_t status = fd_table_grow(t, fd);
     if (status) {
         mutex_unlock(&proc->files_lock);
         return status;
     }
+    fd_install(t, fd, file, cloexec);
+    if ((size_t)min_fd <= t->next_free)
+        t->next_free = fd + 1;
     vfs_file_ref(file);
     mutex_unlock(&proc->files_lock);
 
@@ -231,20 +306,15 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file,
     if (fd < 0 || (size_t)fd >= process_fd_limit(proc))
         return EBADF;
 
-    struct vfs_file* displaced = nullptr;
-    const struct fd_entry entry = {.file = file, .cloexec = cloexec};
+    struct fd_table* t = &proc->files;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* slot = fd_lookup(proc, fd);
-    if (slot) {
-        displaced = slot->file;
-        *slot = entry;
-    } else {
-        errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default);
-        if (status) {
-            mutex_unlock(&proc->files_lock);
-            return status;
-        }
+    errno_t status = fd_table_grow(t, fd);
+    if (status) {
+        mutex_unlock(&proc->files_lock);
+        return status;
     }
+    struct vfs_file* displaced = t->files[fd];
+    fd_install(t, fd, file, cloexec);
     vfs_file_ref(file);
     mutex_unlock(&proc->files_lock);
 
@@ -256,9 +326,8 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file,
 errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) {
     errno_t status = EBADF;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        *out_cloexec = entry->cloexec;
+    if (fd_lookup(proc, fd)) {
+        *out_cloexec = (proc->files.cloexec[fd / 64] >> (fd % 64)) & 1;
         status = 0;
     }
     mutex_unlock(&proc->files_lock);
@@ -268,9 +337,9 @@ errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec)
 errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
     errno_t status = EBADF;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        entry->cloexec = cloexec;
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file) {
+        fd_install(&proc->files, fd, file, cloexec);
         status = 0;
     }
     mutex_unlock(&proc->files_lock);
@@ -278,25 +347,19 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
 }
 
 struct vfs_file* process_fd_get(struct process* proc, int fd) {
-    struct vfs_file* file = nullptr;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        file = entry->file;
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file)
         vfs_file_ref(file);
-    }
     mutex_unlock(&proc->files_lock);
     return file;
 }
 
 errno_t process_fd_close(struct process* proc, int fd) {
-    struct vfs_file* file = nullptr;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        file = entry->file;
-        hashmap_remove(&proc->files, fd, hashmap_hash_default, hashmap_eq_default);
-    }
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file)
+        fd_clear(&proc->files, fd);
     mutex_unlock(&proc->files_lock);
 
     if (!file)

lib/semaphore.c

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/atomic.h>
+#include <zinnia/park.h>
+#include <zinnia/semaphore.h>
+
+void sem_init(sem_t* sem, uint32_t count) {
+    atomic_store(&sem->count, count, ATOMIC_RELAXED);
+}
+
+bool sem_trywait(sem_t* sem) {
+    uint32_t count = atomic_load(&sem->count, ATOMIC_RELAXED);
+    while (count) {
+        if (atomic_compare_exchange_weak(&sem->count, &count, count - 1, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+            return true;
+    }
+    return false;
+}
+
+static bool sem_empty(void* ctx) {
+    sem_t* sem = ctx;
+    return atomic_load(&sem->count, ATOMIC_RELAXED) == 0;
+}
+
+bool sem_timedwait(sem_t* sem, const uint64_t* deadline_ns) {
+    while (!sem_trywait(sem)) {
+        if (park_timeout(sem, sem_empty, nullptr, sem, 0, deadline_ns).status == PARK_TIMED_OUT)
+            return sem_trywait(sem);
+    }
+    return true;
+}
+
+void sem_wait(sem_t* sem) {
+    sem_timedwait(sem, nullptr);
+}
+
+void sem_post(sem_t* sem) {
+    atomic_fetch_add(&sem->count, 1, ATOMIC_RELEASE);
+    unpark_one(sem, nullptr, nullptr);
+}

lib/string.c

@@ -5,6 +5,9 @@
 
 [[__hot]]
 size_t strlen(const char* str) {
+    if (!str)
+        return 0;
+
     size_t result = 0;
     while (*str++) {
         result++;
@@ -14,6 +17,9 @@ size_t strlen(const char* str) {
 
 [[__hot]]
 size_t strnlen(const char* str, size_t len) {
+    if (!str || len == 0)
+        return 0;
+
     size_t result = 0;
     while (result < len && *str++) {
         result++;

mem/dma.c

@@ -0,0 +1,272 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/dma.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+
+static errno_t dma_noop_map(struct dma_tag*, phys_t phys, size_t, dma_addr_t* out) {
+    *out = phys;
+    return 0;
+}
+
+static void dma_noop_unmap(struct dma_tag*, dma_addr_t, size_t) {
+    // Does nothing.
+}
+
+static const struct dma_ops dma_noop_ops = {
+    .map = dma_noop_map,
+    .unmap = dma_noop_unmap,
+};
+
+struct dma_tag dma_default_tag = {
+    .ops = &dma_noop_ops,
+    .max_addr = DMA_ADDR_MAX,
+};
+
+void dma_tag_subregion(struct dma_tag* parent, dma_addr_t max_addr, struct dma_tag* out) {
+    *out = *parent;
+    out->max_addr = MIN(parent->max_addr, max_addr);
+}
+
+static errno_t tag_alloc_flags(struct dma_tag* tag, enum alloc_flags* out) {
+    if (tag->max_addr == DMA_ADDR_MAX)
+        *out = 0;
+    else if (tag->max_addr >= UINT32_MAX)
+        *out = ALLOC_MEM32;
+    else if (tag->max_addr >= ((dma_addr_t)1 << 20) - 1)
+        *out = ALLOC_MEM20;
+    else
+        return EINVAL;
+    return 0;
+}
+
+errno_t dma_buffer_alloc(struct dma_tag* tag, size_t len, struct dma_buffer* out) {
+    if (len == 0)
+        return EINVAL;
+
+    enum alloc_flags flags;
+    errno_t status = tag_alloc_flags(tag, &flags);
+    if (status)
+        return status;
+
+    const size_t num_pages = ROUND_UP(len, arch_mem_page_size());
+    phys_t phys;
+    status = mem_phys_alloc(num_pages, flags, &phys);
+    if (status)
+        return status;
+
+    const size_t size = num_pages * arch_mem_page_size();
+    dma_addr_t addr;
+    status = tag->ops->map(tag, phys, size, &addr);
+    if (status) {
+        mem_phys_free(phys, num_pages);
+        return status;
+    }
+    if (addr + size - 1 > tag->max_addr) {
+        tag->ops->unmap(tag, addr, size);
+        mem_phys_free(phys, num_pages);
+        return ENOMEM;
+    }
+
+    *out = (struct dma_buffer){
+        .tag = tag,
+        .phys = phys,
+        .addr = addr,
+        .virt = HHDM_PTR(phys),
+        .len = len,
+    };
+    return 0;
+}
+
+void dma_buffer_free(struct dma_buffer* buf) {
+    if (buf->virt == nullptr)
+        return;
+
+    const size_t num_pages = ROUND_UP(buf->len, arch_mem_page_size());
+    buf->tag->ops->unmap(buf->tag, buf->addr, num_pages * arch_mem_page_size());
+    mem_phys_free(buf->phys, num_pages);
+    *buf = (struct dma_buffer){0};
+}
+
+errno_t dma_map_create(
+    struct dma_tag* tag,
+    size_t size,
+    size_t max_segs,
+    size_t max_seg_size,
+    size_t boundary,
+    struct dma_map** out
+) {
+    if (max_segs == 0 || max_seg_size == 0)
+        return EINVAL;
+    if (boundary & (boundary - 1))
+        return EINVAL;
+
+    struct dma_map* map = mem_alloc(sizeof(*map) + max_segs * sizeof(struct dma_seg), 0);
+    if (map == nullptr)
+        return ENOMEM;
+
+    map->tag = tag;
+    map->size = size;
+    map->max_segs = max_segs;
+    map->max_seg_size = max_seg_size;
+    map->boundary = boundary;
+    map->mapped_len = 0;
+    map->num_segs = 0;
+    *out = map;
+    return 0;
+}
+
+void dma_map_destroy(struct dma_map* map) {
+    mem_free(map);
+}
+
+static errno_t map_add_range(struct dma_map* map, phys_t phys, size_t len) {
+    if (len == 0)
+        return 0;
+
+    dma_addr_t addr;
+    errno_t status = map->tag->ops->map(map->tag, phys, len, &addr);
+    if (status)
+        return status;
+
+    if (addr + len - 1 > map->tag->max_addr) {
+        map->tag->ops->unmap(map->tag, addr, len);
+        return EINVAL;
+    }
+
+    while (len > 0) {
+        size_t chunk = MIN(len, map->max_seg_size);
+        if (map->boundary) {
+            const dma_addr_t limit = ALIGN_DOWN(addr, map->boundary) + map->boundary;
+            chunk = MIN(chunk, limit - addr);
+        }
+
+        struct dma_seg* last = map->num_segs ? &map->segs[map->num_segs - 1] : nullptr;
+        if (last && last->addr + last->len == addr && last->len + chunk <= map->max_seg_size &&
+            (!map->boundary || ALIGN_DOWN(last->addr, map->boundary) == ALIGN_DOWN(addr + chunk - 1, map->boundary))) {
+            last->len += chunk;
+        } else {
+            if (map->num_segs == map->max_segs) {
+                map->tag->ops->unmap(map->tag, addr, len);
+                return EFBIG;
+            }
+            map->segs[map->num_segs++] = (struct dma_seg){.addr = addr, .len = chunk};
+        }
+
+        map->mapped_len += chunk;
+        addr += chunk;
+        len -= chunk;
+    }
+    return 0;
+}
+
+static errno_t map_add_virt(struct dma_map* map, struct pmap* pmap, uintptr_t vaddr, size_t len) {
+    const size_t page_size = arch_mem_page_size();
+    while (len > 0) {
+        const size_t offset = vaddr % page_size;
+        const size_t chunk = MIN(len, page_size - offset);
+
+        phys_t phys;
+        if (pmap_extract(pmap, vaddr - offset, &phys))
+            return EFAULT;
+
+        errno_t status = map_add_range(map, phys + offset, chunk);
+        if (status)
+            return status;
+
+        vaddr += chunk;
+        len -= chunk;
+    }
+    return 0;
+}
+
+static errno_t map_load_finish(struct dma_map* map, errno_t status) {
+    if (status)
+        dma_map_unload(map);
+    return status;
+}
+
+errno_t dma_map_load(struct dma_map* map, void* buf, size_t len) {
+    if (map->num_segs || len > map->size)
+        return EINVAL;
+
+    return map_load_finish(map, map_add_virt(map, &kernel_vma.pmap, (uintptr_t)buf, len));
+}
+
+errno_t dma_map_load_bio(struct dma_map* map, struct bio* bio) {
+    if (map->num_segs)
+        return EINVAL;
+
+    size_t total = 0;
+    for (size_t i = 0; i < bio->num_segs; i++)
+        total += bio->segs[i].len;
+    if (total > map->size)
+        return EINVAL;
+
+    errno_t status = 0;
+    for (size_t i = 0; i < bio->num_segs && !status; i++)
+        status = map_add_range(map, bio->segs[i].addr, bio->segs[i].len);
+
+    return map_load_finish(map, status);
+}
+
+errno_t dma_map_load_iter(struct dma_map* map, const iovec_iter_t* iter) {
+    if (map->num_segs || iter->total_len - iter->total_offset > map->size)
+        return EINVAL;
+
+    struct pmap* pmap = &kernel_vma.pmap;
+    if (iter->user)
+        pmap = &percpu_get()->sched.current->space->pmap;
+
+    errno_t status = 0;
+    size_t offset = iter->current_offset;
+    for (size_t i = iter->current_idx; i < iter->iovecs_len && !status; i++) {
+        const iovec_t* iov = &iter->iovecs[i];
+        if (offset >= iov->len) {
+            offset = 0;
+            continue;
+        }
+
+        const uintptr_t base = iov->base + offset;
+        const size_t len = iov->len - offset;
+        offset = 0;
+
+        if (iter->user && (!arch_mem_addr_is_user(base) || !arch_mem_addr_is_user(base + len - 1))) {
+            status = EFAULT;
+            break;
+        }
+
+        status = map_add_virt(map, pmap, base, len);
+    }
+
+    return map_load_finish(map, status);
+}
+
+void dma_map_unload(struct dma_map* map) {
+    for (size_t i = 0; i < map->num_segs; i++)
+        map->tag->ops->unmap(map->tag, map->segs[i].addr, map->segs[i].len);
+    map->num_segs = 0;
+    map->mapped_len = 0;
+}
+
+static void dma_sync(enum dma_sync ops) {
+    if (ops & (DMA_SYNC_PREREAD | DMA_SYNC_PREWRITE))
+        atomic_fence(ATOMIC_RELEASE);
+    if (ops & (DMA_SYNC_POSTREAD | DMA_SYNC_POSTWRITE))
+        atomic_fence(ATOMIC_ACQUIRE);
+}
+
+void dma_map_sync(struct dma_map* map, size_t offset, size_t len, enum dma_sync ops) {
+    dma_sync(ops);
+}
+
+void dma_buffer_sync(struct dma_buffer* buf, size_t offset, size_t len, enum dma_sync ops) {
+    dma_sync(ops);
+}

mem/meson.build

@@ -1,6 +1,6 @@
 kernel_sources += files(
     'amap.c',
-    'fault.c',
+    'dma.c',
     'init.c',
     'page.c',
     'pmm.c',

meson.build

@@ -2,7 +2,7 @@ project(
     'zinnia',
     license: 'ISC',
     version: '0.1.0',
-    meson_version: '>=1.5.0',
+    meson_version: '>=1.6.0',
     default_options: [
         'c_std=gnu23',
         'werror=true',
@@ -13,6 +13,7 @@ project(
 
 if get_option('headers_only')
     install_subdir('include/uapi', install_dir: get_option('includedir'))
+    install_subdir('include/sys', install_dir: get_option('includedir'))
     pkg = import('pkgconfig')
     pkg.generate(name: 'zinnia', description: 'Zinnia user API')
     subdir_done()
@@ -22,6 +23,7 @@ add_languages('c', native: false, required: true)
 
 # Check if we're using a sane compiler.
 c_compiler = meson.get_compiler('c')
+python = import('python').find_installation('python3')
 
 # We require a compiler that supports the GNU C dialect.
 assert(c_compiler.has_define('__GNUC__'))
@@ -94,7 +96,6 @@ modules = {}
 
 # These dirs go first so we can add arch-specific flags to drivers.
 subdir('arch' / target_machine.cpu_family())
-subdir('boot')
 subdir('lib')
 subdir('mem')
 subdir('sched')
@@ -104,6 +105,10 @@ subdir('syscalls')
 subdir('dev')
 subdir('fs')
 
+if kconfig.get('CONFIG_KTEST', 'n') == 'y'
+    subdir('tests/kernel')
+endif
+
 configure_file(
     configuration: cfg,
     output: 'config.h',
@@ -174,6 +179,12 @@ kernel = executable(
     install_dir: zinnia_datadir,
 )
 
+subdir('boot')
+
+if kconfig.get('CONFIG_KTEST', 'n') == 'y'
+    subdir('tests')
+endif
+
 foreach name, v : modular_modules
     shared_library(
         name,

meson.options

@@ -18,3 +18,10 @@ option(
     value: '.config',
     description: 'Relative path to the generated Kconfig file',
 )
+
+option(
+    'ovmf',
+    type: 'string',
+    value: '',
+    description: 'OVMF firmware image used to run the QEMU tests',
+)

sched/core.c

@@ -4,6 +4,7 @@
 #include <zinnia/assert.h>
 #include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/park.h>
 #include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
@@ -157,3 +158,8 @@ void task_lock(struct task* task) {
 void task_unlock(struct task* task) {
     spin_unlock(atomic_load(&task->lockp, ATOMIC_RELAXED));
 }
+
+void sched_sleep_until(uint64_t deadline_ns) {
+    const uint8_t key = 0;
+    park_timeout(&key, nullptr, nullptr, nullptr, 0, &deadline_ns);
+}

sched/sched_dumb.c

@@ -84,6 +84,12 @@ void sched_init(struct scheduler* sched) {
 
 // Asks the CPU of `sched` to reschedule as soon as it has interrupts enabled.
 static void kick(struct scheduler* sched) {
+    if (sched == &percpu_get()->sched) {
+        sched->owe_preempt = true;
+        if (percpu_ref(irq_data)->depth == 0 && irq_get_state() && !sched->preempt_level && sched->current)
+            sched_reschedule(sched);
+        return;
+    }
     if (!atomic_exchange(&sched->kick_pending, true, ATOMIC_RELAXED))
         arch_sched_kick(cpu_of(sched));
 }
@@ -217,6 +223,7 @@ static void finish_switch(struct task* prev) {
 
 static void do_reschedule(struct scheduler* sched, uint64_t now) {
     atomic_store(&sched->kick_pending, false, ATOMIC_RELAXED);
+    sched->owe_preempt = false;
     struct task* from = sched->current;
     struct task* to = next_task(sched, now);
 

syscalls/dispatch.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
 #include <zinnia/print.h>

tests/Kconfig

@@ -0,0 +1,10 @@
+menu "Testing"
+
+config KTEST
+  bool "Kernel test suite"
+  depends on BOOT_EFISTUB
+  default n
+  help
+    Builds the kernel unit tests. To run them, add "ktest=1" to the cmdline.
+
+endmenu

tests/cmdline.in

@@ -0,0 +1 @@
+@CMDLINE@

tests/kernel/example.c

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/ktest.h>
+
+KTEST(example, arithmetic) {
+    KTEST_EXPECT_EQ(1 + 1, 2);
+    KTEST_EXPECT_LT(1, 2);
+}

tests/kernel/meson.build

@@ -0,0 +1,4 @@
+kernel_sources += files(
+    'example.c',
+    'runner.c',
+)

tests/kernel/runner.c

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/cmdline.h>
+#include <zinnia/ktest.h>
+#include <zinnia/print.h>
+#include <stdarg.h>
+
+extern const struct ktest_case __ld_ktest_start[];
+extern const struct ktest_case __ld_ktest_end[];
+
+static bool ktest_enabled = false;
+
+static void ktest_option(bool value) {
+    ktest_enabled = value;
+}
+CMDLINE_OPTION("ktest", "Run all kernel tests", ktest_option);
+
+void ktest_fail(struct ktest* test, const char* file, int line, const char* fmt, ...) {
+    char buf[256];
+    va_list args;
+    va_start(args, fmt);
+    kvsnprintf(buf, sizeof(buf), fmt, args);
+    va_end(args);
+
+    test->failed = true;
+    log("# %s.%s: %s:%d: %s\n", test->suite, test->name, file, line, buf);
+}
+
+void ktest_main() {
+    if (!ktest_enabled)
+        return;
+
+    size_t index = 0;
+    for (const struct ktest_case* c = __ld_ktest_start; c < __ld_ktest_end; c++) {
+        struct ktest test = {.suite = c->suite, .name = c->name};
+        c->func(&test);
+
+        index++;
+        if (test.failed)
+            log("not ok %zu %s.%s\n", index, c->suite, c->name);
+        else if (test.skipped)
+            log("ok %zu %s.%s # SKIP %s\n", index, c->suite, c->name, test.skip_reason ? test.skip_reason : "");
+        else
+            log("ok %zu %s.%s\n", index, c->suite, c->name);
+    }
+
+    log("1..%zu\n", index);
+}

tests/meson.build

@@ -0,0 +1,125 @@
+fs = import('fs')
+
+assert(
+    kconfig.get('CONFIG_BOOT_EFISTUB', 'n') == 'y',
+    'CONFIG_KTEST depends on CONFIG_BOOT_EFISTUB',
+)
+
+assert(
+    target_machine.cpu_family() == 'x86_64',
+    'CONFIG_KTEST is only supported on x86_64',
+)
+
+# All userspace tests are a single executable, so they have to be idempotent.
+utest_init = executable(
+    'init',
+    sources: files(
+        'user/lib/crt0.S',
+        'user/lib/utest.c',
+        'user/memory.c',
+        'user/system.c',
+    ),
+    include_directories: include_directories('../include'),
+    c_args: [
+        '-ffreestanding',
+        '-nostdinc',
+        '-fno-pic',
+        '-fno-stack-protector',
+        '-masm=intel',
+    ],
+    link_args: [
+        '-nostdlib',
+        '-static',
+        '-Wl,--no-dynamic-linker',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,-z,noexecstack',
+        '-Wl,--build-id=none',
+    ],
+    pie: false,
+)
+
+test_initramfs = custom_target(
+    'initramfs.tar',
+    input: utest_init,
+    output: 'initramfs.tar',
+    command: [
+        find_program('tar'),
+        '--format=ustar',
+        '-cf', '@OUTPUT@',
+        '-C', '@OUTDIR@',
+        'init',
+    ],
+)
+
+# Try to find an OVMF.
+ovmf = get_option('ovmf')
+if ovmf == ''
+    foreach path : [
+        '/usr/share/qemu/ovmf-x86_64-4m.bin',
+        '/usr/share/qemu/ovmf-x86_64.bin',
+        '/usr/share/ovmf/OVMF.fd',
+        '/usr/share/edk2/x64/OVMF.4m.fd',
+        '/usr/share/edk2/x64/OVMF.fd',
+        '/usr/share/edk2/ovmf/OVMF.fd',
+    ]
+        if ovmf == '' and fs.is_file(path)
+            ovmf = path
+        endif
+    endforeach
+endif
+assert(ovmf != '', 'CONFIG_KTEST needs an OVMF image, set it with -Dovmf=')
+
+qemu = [
+    find_program('qemu-system-x86_64'),
+    '-M', 'q35',
+    '-m', '512M',
+    '-smp', '4',
+    '-accel', 'kvm',
+    '-accel', 'tcg',
+    '-cpu', 'max',
+    '-nic', 'none',
+    '-display', 'none',
+    '-monitor', 'none',
+    '-serial', 'stdio',
+    '-no-reboot',
+    '-bios', ovmf,
+]
+
+test_images = {
+    'kernel': {
+        'cmdline': 'ktest=1',
+        'files': [],
+    },
+    'user': {
+        'cmdline': 'init=/init initrd=/initramfs.tar',
+        'files': [test_initramfs],
+    },
+}
+
+# Build an ESP with the cmdline file and an initramfs.
+foreach name, image : test_images
+    conf = configure_file(
+        input: 'cmdline.in',
+        output: name + '.cmdline',
+        configuration: {'CMDLINE': image['cmdline']},
+    )
+
+    esp = [
+        zinnia_efi.full_path() + '=EFI/BOOT/BOOTX64.EFI',
+        conf.full_path() + '=EFI/zinnia/cmdline',
+    ]
+
+    foreach file : image['files']
+        esp += file.full_path() + '=' + fs.name(file.full_path())
+    endforeach
+
+    test(
+        name,
+        python,
+        args: [files('run.py'), esp, '--', qemu],
+        depends: [zinnia_efi, image['files']],
+        protocol: 'tap',
+        suite: 'qemu',
+        timeout: 60,
+    )
+endforeach

tests/run.py

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: ISC
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+NOISE_RE = re.compile(r"\x1b\[[0-9;]*m|\r|^\[[ \d.]+\] (?:syslog\(\w+\): )?")
+PLAN_RE = re.compile(r"^1\.\.\d+")
+TAP_RE = re.compile(r"^(?:ok|not ok|#|1\.\.\d+|Bail out!)")
+
+sep = sys.argv.index("--")
+with tempfile.TemporaryDirectory(prefix="zinnia-esp-") as esp:
+    for spec in sys.argv[1:sep]:
+        src, dest = spec.split("=", 1)
+        path = os.path.join(esp, dest)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        shutil.copyfile(src, path)
+
+    proc = subprocess.Popen(
+        sys.argv[sep + 1 :] + ["-drive", f"format=raw,file=fat:{esp},if=virtio,readonly=on"],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        text=True,
+        errors="replace",
+    )
+    for line in proc.stdout:
+        line = NOISE_RE.sub("", line).rstrip()
+        print(line if TAP_RE.match(line) else f"# {line}", flush=True)
+        if PLAN_RE.match(line):
+            proc.kill()
+            proc.wait()
+            sys.exit(0)
+    proc.wait()
+sys.exit(1)

tests/user/lib/crt0.S

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: ISC
+
+.intel_syntax noprefix
+.text
+
+.globl _start
+.type _start, @function
+_start:
+    xor ebp, ebp
+    and rsp, -16
+    call main
+    mov edi, eax
+    xor eax, eax
+    syscall
+    ud2
+.size _start, . - _start
+
+.section .note.GNU-stack, "", @progbits

tests/user/lib/syscall.h

@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+enum syscall_number : uintptr_t {
+#define SYSCALL_DECLARE(number, name, num_args) SYS_##name = number,
+#include <zinnia/syscall_list.inc>
+#undef SYSCALL_DECLARE
+};
+
+enum syslog_kind : uintptr_t {
+    SYSLOG_ERR = 3,
+    SYSLOG_INFO = 6,
+};
+
+struct sys_ret {
+    uintptr_t value;
+    uintptr_t error;
+};
+
+static inline struct sys_ret syscall6(
+    enum syscall_number num,
+    uintptr_t a0,
+    uintptr_t a1,
+    uintptr_t a2,
+    uintptr_t a3,
+    uintptr_t a4,
+    uintptr_t a5
+) {
+    register uintptr_t r10 asm("r10") = a3;
+    register uintptr_t r8 asm("r8") = a4;
+    register uintptr_t r9 asm("r9") = a5;
+    uintptr_t value;
+    uintptr_t error;
+    asm volatile("syscall"
+                 : "=a"(value), "=d"(error)
+                 : "a"(num), "D"(a0), "S"(a1), "d"(a2), "r"(r10), "r"(r8), "r"(r9)
+                 : "rcx", "r11", "memory");
+    return (struct sys_ret){value, error};
+}
+
+static inline struct sys_ret syscall3(enum syscall_number num, uintptr_t a0, uintptr_t a1, uintptr_t a2) {
+    return syscall6(num, a0, a1, a2, 0, 0, 0);
+}
+
+static inline struct sys_ret syscall4(enum syscall_number num, uintptr_t a0, uintptr_t a1, uintptr_t a2, uintptr_t a3) {
+    return syscall6(num, a0, a1, a2, a3, 0, 0);
+}
+
+static inline struct sys_ret syscall5(
+    enum syscall_number num,
+    uintptr_t a0,
+    uintptr_t a1,
+    uintptr_t a2,
+    uintptr_t a3,
+    uintptr_t a4
+) {
+    return syscall6(num, a0, a1, a2, a3, a4, 0);
+}
+
+[[noreturn]]
+static inline void sys_exit(int status) {
+    syscall3(SYS_exit, (uintptr_t)status, 0, 0);
+    __builtin_unreachable();
+}
+
+static inline struct sys_ret sys_syslog(enum syslog_kind kind, const char* msg, size_t len) {
+    return syscall3(SYS_syslog, kind, (uintptr_t)msg, len);
+}
+
+static inline struct sys_ret sys_mmap(void* addr, size_t len, uint32_t prot, uint32_t flags, int fd, int64_t off) {
+    return syscall6(SYS_mmap, (uintptr_t)addr, len, prot, flags, (uintptr_t)fd, (uintptr_t)off);
+}
+
+static inline struct sys_ret sys_munmap(void* addr, size_t len) {
+    return syscall3(SYS_munmap, (uintptr_t)addr, len, 0);
+}
+
+static inline struct sys_ret sys_openat(int dirfd, const char* path, uint32_t flags, uint32_t mode) {
+    return syscall4(SYS_openat, (uintptr_t)dirfd, (uintptr_t)path, flags, mode);
+}
+
+static inline struct sys_ret sys_close(int fd) {
+    return syscall3(SYS_close, (uintptr_t)fd, 0, 0);
+}
+
+static inline struct sys_ret sys_pread(int fd, void* buf, size_t len, uint64_t off) {
+    return syscall4(SYS_pread, (uintptr_t)fd, (uintptr_t)buf, len, off);
+}
+
+static inline struct sys_ret sys_fstat(int fd, void* buf) {
+    return syscall3(SYS_fstat, (uintptr_t)fd, (uintptr_t)buf, 0);
+}
+
+static inline struct sys_ret sys_getuname(void* buf) {
+    return syscall3(SYS_getuname, (uintptr_t)buf, 0, 0);
+}
+
+static inline struct sys_ret sys_setuname(const void* buf) {
+    return syscall3(SYS_setuname, (uintptr_t)buf, 0, 0);
+}
+
+static inline struct sys_ret sys_mprotect(void* addr, size_t len, uint32_t prot) {
+    return syscall3(SYS_mprotect, (uintptr_t)addr, len, prot);
+}
+
+static inline struct sys_ret sys_madvise(void* addr, size_t len, int advice) {
+    return syscall3(SYS_madvise, (uintptr_t)addr, len, (uintptr_t)advice);
+}
+
+static inline struct sys_ret sys_msync(void* addr, size_t len, int flags) {
+    return syscall3(SYS_msync, (uintptr_t)addr, len, (uintptr_t)flags);
+}
+
+static inline struct sys_ret sys_pwrite(int fd, const void* buf, size_t len, uint64_t off) {
+    return syscall4(SYS_pwrite, (uintptr_t)fd, (uintptr_t)buf, len, off);
+}
+
+static inline struct sys_ret sys_readv(int fd, const void* iov, size_t count) {
+    return syscall3(SYS_readv, (uintptr_t)fd, (uintptr_t)iov, count);
+}
+
+static inline struct sys_ret sys_writev(int fd, const void* iov, size_t count) {
+    return syscall3(SYS_writev, (uintptr_t)fd, (uintptr_t)iov, count);
+}
+
+static inline struct sys_ret sys_seek(int fd, int64_t off, int whence) {
+    return syscall3(SYS_seek, (uintptr_t)fd, (uintptr_t)off, (uintptr_t)whence);
+}
+
+static inline struct sys_ret sys_fstatat(int dirfd, const char* path, void* buf, uint32_t flags) {
+    return syscall4(SYS_fstatat, (uintptr_t)dirfd, (uintptr_t)path, (uintptr_t)buf, flags);
+}
+
+static inline struct sys_ret sys_faccessat(int dirfd, const char* path, uint32_t mode, uint32_t flags) {
+    return syscall4(SYS_faccessat, (uintptr_t)dirfd, (uintptr_t)path, mode, flags);
+}
+
+static inline struct sys_ret sys_fcntl(int fd, int cmd, uintptr_t arg) {
+    return syscall3(SYS_fcntl, (uintptr_t)fd, (uintptr_t)cmd, arg);
+}
+
+static inline struct sys_ret sys_ftruncate(int fd, int64_t len) {
+    return syscall3(SYS_ftruncate, (uintptr_t)fd, (uintptr_t)len, 0);
+}
+
+static inline struct sys_ret sys_truncate(const char* path, int64_t len) {
+    return syscall3(SYS_truncate, (uintptr_t)path, (uintptr_t)len, 0);
+}
+
+static inline struct sys_ret sys_getcwd(char* buf, size_t len) {
+    return syscall3(SYS_getcwd, (uintptr_t)buf, len, 0);
+}
+
+static inline struct sys_ret sys_chdir(const char* path) {
+    return syscall3(SYS_chdir, (uintptr_t)path, 0, 0);
+}
+
+static inline struct sys_ret sys_fchdir(int fd) {
+    return syscall3(SYS_fchdir, (uintptr_t)fd, 0, 0);
+}
+
+static inline struct sys_ret sys_mkdirat(int dirfd, const char* path, uint32_t mode) {
+    return syscall3(SYS_mkdirat, (uintptr_t)dirfd, (uintptr_t)path, mode);
+}
+
+static inline struct sys_ret sys_rmdirat(int dirfd, const char* path) {
+    return syscall3(SYS_rmdirat, (uintptr_t)dirfd, (uintptr_t)path, 0);
+}
+
+static inline struct sys_ret sys_unlinkat(int dirfd, const char* path, uint32_t flags) {
+    return syscall3(SYS_unlinkat, (uintptr_t)dirfd, (uintptr_t)path, flags);
+}
+
+static inline struct sys_ret sys_getdents(int fd, void* buf, size_t len) {
+    return syscall3(SYS_getdents, (uintptr_t)fd, (uintptr_t)buf, len);
+}
+
+static inline struct sys_ret sys_renameat(int old_dirfd, const char* old_path, int new_dirfd, const char* new_path) {
+    return syscall4(SYS_renameat, (uintptr_t)old_dirfd, (uintptr_t)old_path, (uintptr_t)new_dirfd, (uintptr_t)new_path);
+}
+
+static inline struct sys_ret sys_linkat(
+    int old_dirfd,
+    const char* old_path,
+    int new_dirfd,
+    const char* new_path,
+    uint32_t flags
+) {
+    return syscall5(
+        SYS_linkat,
+        (uintptr_t)old_dirfd,
+        (uintptr_t)old_path,
+        (uintptr_t)new_dirfd,
+        (uintptr_t)new_path,
+        flags
+    );
+}
+
+static inline struct sys_ret sys_symlinkat(const char* target, int dirfd, const char* path) {
+    return syscall3(SYS_symlinkat, (uintptr_t)target, (uintptr_t)dirfd, (uintptr_t)path);
+}
+
+static inline struct sys_ret sys_readlinkat(int dirfd, const char* path, char* buf, size_t len) {
+    return syscall4(SYS_readlinkat, (uintptr_t)dirfd, (uintptr_t)path, (uintptr_t)buf, len);
+}
+
+static inline struct sys_ret sys_dup(int fd) {
+    return syscall3(SYS_dup, (uintptr_t)fd, 0, 0);
+}
+
+static inline struct sys_ret sys_dup3(int old_fd, int new_fd, uint32_t flags) {
+    return syscall3(SYS_dup3, (uintptr_t)old_fd, (uintptr_t)new_fd, flags);
+}
+
+static inline struct sys_ret sys_umask(uint32_t mask) {
+    return syscall3(SYS_umask, mask, 0, 0);
+}
+
+static inline struct sys_ret sys_pipe(int* fds, uint32_t flags) {
+    return syscall3(SYS_pipe, (uintptr_t)fds, flags, 0);
+}
+
+static inline struct sys_ret sys_ppoll(void* fds, size_t nfds, const void* timeout, const void* sigmask) {
+    return syscall4(SYS_ppoll, (uintptr_t)fds, nfds, (uintptr_t)timeout, (uintptr_t)sigmask);
+}

tests/user/lib/utest.c

@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: ISC
+
+#include "utest.h"
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include "syscall.h"
+
+constexpr size_t LINE_MAX = 240;
+
+extern const struct utest_case __start_utest_cases[];
+extern const struct utest_case __stop_utest_cases[];
+
+size_t strlen(const char* str) {
+    size_t len = 0;
+    while (str[len])
+        len++;
+    return len;
+}
+
+int memcmp(const void* s1, const void* s2, size_t size) {
+    const uint8_t* a = s1;
+    const uint8_t* b = s2;
+    for (size_t i = 0; i < size; i++) {
+        if (a[i] != b[i])
+            return a[i] < b[i] ? -1 : 1;
+    }
+    return 0;
+}
+
+void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t* s = src;
+    for (size_t i = 0; i < n; i++)
+        d[i] = s[i];
+    return dest;
+}
+
+void* memset(void* dest, int value, size_t n) {
+    uint8_t* d = dest;
+    for (size_t i = 0; i < n; i++)
+        d[i] = (uint8_t)value;
+    return dest;
+}
+
+bool utest_streq(const char* a, const char* b) {
+    if (!a || !b)
+        return a == b;
+    while (*a && *a == *b) {
+        a++;
+        b++;
+    }
+    return *a == *b;
+}
+
+struct line {
+    char buf[LINE_MAX];
+    size_t len;
+};
+
+static void put(struct line* line, char c) {
+    if (line->len < sizeof(line->buf) - 1)
+        line->buf[line->len++] = c;
+}
+
+static void put_str(struct line* line, const char* str) {
+    for (; *str; str++)
+        put(line, *str);
+}
+
+static void put_num(struct line* line, unsigned long long value, unsigned base, bool negative, bool prefix) {
+    char digits[24];
+    size_t n = 0;
+    do {
+        digits[n++] = "0123456789abcdef"[value % base];
+        value /= base;
+    } while (value);
+    if (negative)
+        put(line, '-');
+    if (prefix)
+        put_str(line, "0x");
+    while (n)
+        put(line, digits[--n]);
+}
+
+static void format(struct line* line, const char* fmt, va_list args) {
+    for (; *fmt; fmt++) {
+        if (*fmt != '%') {
+            put(line, *fmt);
+            continue;
+        }
+        fmt++;
+        bool alt = false;
+        if (*fmt == '#') {
+            alt = true;
+            fmt++;
+        }
+        int longs = 0;
+        bool size = false;
+        for (;; fmt++) {
+            if (*fmt == 'l')
+                longs++;
+            else if (*fmt == 'z')
+                size = true;
+            else
+                break;
+        }
+
+        switch (*fmt) {
+        case 'd':
+        case 'i': {
+            long long v = size ? (long long)va_arg(args, ptrdiff_t)
+                : longs        ? va_arg(args, long long)
+                               : va_arg(args, int);
+            put_num(line, v < 0 ? -(unsigned long long)v : (unsigned long long)v, 10, v < 0, false);
+            break;
+        }
+        case 'u':
+        case 'x': {
+            unsigned long long v = size ? va_arg(args, size_t)
+                : longs                 ? va_arg(args, unsigned long long)
+                                        : va_arg(args, unsigned int);
+            put_num(line, v, *fmt == 'x' ? 16 : 10, false, alt && *fmt == 'x');
+            break;
+        }
+        case 'p':
+            put_num(line, (uintptr_t)va_arg(args, void*), 16, false, true);
+            break;
+        case 's': {
+            const char* s = va_arg(args, const char*);
+            put_str(line, s ? s : "(null)");
+            break;
+        }
+        case 'c':
+            put(line, (char)va_arg(args, int));
+            break;
+        case '%':
+            put(line, '%');
+            break;
+        default:
+            put(line, '?');
+            break;
+        }
+    }
+}
+
+static void emit(struct line* line) {
+    line->buf[line->len] = 0;
+    sys_syslog(SYSLOG_INFO, line->buf, line->len);
+}
+
+void utest_log(const char* fmt, ...) {
+    struct line line = {};
+    va_list args;
+    va_start(args, fmt);
+    format(&line, fmt, args);
+    va_end(args);
+    emit(&line);
+}
+
+void utest_fail(struct utest* test, const char* file, int line_nr, const char* fmt, ...) {
+    struct line line = {};
+    put_str(&line, "# ");
+    put_str(&line, test->suite);
+    put(&line, '.');
+    put_str(&line, test->name);
+    put_str(&line, ": ");
+    put_str(&line, file);
+    put(&line, ':');
+    put_num(&line, (unsigned)line_nr, 10, false, false);
+    put_str(&line, ": ");
+    va_list args;
+    va_start(args, fmt);
+    format(&line, fmt, args);
+    va_end(args);
+    emit(&line);
+    test->failed = true;
+}
+
+void utest_run_all() {
+    size_t index = 0;
+    for (const struct utest_case* c = __start_utest_cases; c < __stop_utest_cases; c++) {
+        struct utest test = {.suite = c->suite, .name = c->name};
+        c->func(&test);
+
+        index++;
+        if (test.skipped)
+            utest_log("ok %zu %s.%s # SKIP %s", index, c->suite, c->name, test.skip_reason ? test.skip_reason : "");
+        else if (c->xfail_reason)
+            utest_log(
+                "%s %zu %s.%s # TODO %s",
+                test.failed ? "not ok" : "ok",
+                index,
+                c->suite,
+                c->name,
+                c->xfail_reason
+            );
+        else
+            utest_log("%s %zu %s.%s", test.failed ? "not ok" : "ok", index, c->suite, c->name);
+    }
+
+    utest_log("1..%zu", index);
+}
+
+int main() {
+    utest_run_all();
+    for (;;)
+        ;
+}

tests/user/lib/utest.h

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h> // IWYU pragma: keep
+#include <stdint.h> // IWYU pragma: keep
+
+struct utest {
+    const char* suite;
+    const char* name;
+    bool failed;
+    bool skipped;
+    const char* skip_reason;
+};
+
+struct utest_case {
+    const char* suite;
+    const char* name;
+    const char* xfail_reason;
+    void (*func)(struct utest* test);
+};
+
+#define _UTEST(suite_name, case_name, reason) \
+    static void suite_name##_##case_name(struct utest* test); \
+    [[gnu::used, gnu::retain, gnu::section("utest_cases"), gnu::aligned(8)]] \
+    static const struct utest_case suite_name##_##case_name##_case = { \
+        .suite = #suite_name, \
+        .name = #case_name, \
+        .xfail_reason = (reason), \
+        .func = suite_name##_##case_name, \
+    }; \
+    static void suite_name##_##case_name(struct utest* test)
+
+// Expected success
+#define UTEST(suite_name, case_name)               _UTEST(suite_name, case_name, nullptr)
+// Expected failure
+#define UTEST_XFAIL(suite_name, case_name, reason) _UTEST(suite_name, case_name, reason)
+
+[[gnu::format(printf, 1, 2)]]
+void utest_log(const char* fmt, ...);
+
+[[gnu::format(printf, 4, 5)]]
+void utest_fail(struct utest* test, const char* file, int line, const char* fmt, ...);
+
+void utest_run_all();
+
+bool utest_streq(const char* a, const char* b);
+
+#define _UTEST_CHECK(cond, on_fail, ...) \
+    do { \
+        if (!(cond)) { \
+            utest_fail(test, __FILE__, __LINE__, __VA_ARGS__); \
+            on_fail; \
+        } \
+    } while (0)
+
+#define _UTEST_CMP(a, op, b, on_fail) \
+    do { \
+        const typeof(true ? (a) : (b)) _utest_a = (a); \
+        const typeof(true ? (a) : (b)) _utest_b = (b); \
+        _UTEST_CHECK( \
+            _utest_a op _utest_b, \
+            on_fail, \
+            "expected %s %s %s, got %#llx and %#llx", \
+            #a, \
+            #op, \
+            #b, \
+            (unsigned long long)_utest_a, \
+            (unsigned long long)_utest_b \
+        ); \
+    } while (0)
+
+#define EXPECT_TRUE(cond)  _UTEST_CHECK(cond, (void)0, "expected %s", #cond)
+#define EXPECT_FALSE(cond) _UTEST_CHECK(!(cond), (void)0, "expected !(%s)", #cond)
+#define EXPECT_EQ(a, b)    _UTEST_CMP(a, ==, b, (void)0)
+#define EXPECT_NE(a, b)    _UTEST_CMP(a, !=, b, (void)0)
+#define EXPECT_GT(a, b)    _UTEST_CMP(a, >, b, (void)0)
+#define EXPECT_GE(a, b)    _UTEST_CMP(a, >=, b, (void)0)
+#define EXPECT_STREQ(a, b) _UTEST_CHECK(utest_streq((a), (b)), (void)0, "expected %s == \"%s\"", #a, (b))
+
+#define ASSERT_TRUE(cond) _UTEST_CHECK(cond, return, "expected %s", #cond)
+#define ASSERT_EQ(a, b)   _UTEST_CMP(a, ==, b, return)
+#define ASSERT_NE(a, b)   _UTEST_CMP(a, !=, b, return)
+
+#define SKIP(reason) \
+    do { \
+        test->skipped = true; \
+        test->skip_reason = (reason); \
+        return; \
+    } while (0)

tests/user/memory.c

@@ -0,0 +1,137 @@
+// SPDX-License-Identifier: ISC
+
+#include <stdint.h>
+#include <string.h>
+#include <uapi/errno.h>
+#include <uapi/fcntl.h>
+#include <uapi/mman.h>
+#include "lib/syscall.h"
+#include "lib/utest.h"
+
+constexpr size_t PAGE_SIZE = 0x1000;
+constexpr uint32_t PROT_RW = __PROT_READ | __PROT_WRITE;
+constexpr uint32_t MAP_ANON_PRIVATE = __MAP_PRIVATE | __MAP_ANON;
+
+static uint8_t* map_anon(size_t len) {
+    struct sys_ret ret = sys_mmap(nullptr, len, PROT_RW, MAP_ANON_PRIVATE, -1, 0);
+    return ret.error ? nullptr : (uint8_t*)ret.value;
+}
+
+UTEST(mman, anonymous_mapping) {
+    const size_t len = 4 * PAGE_SIZE;
+    struct sys_ret ret = sys_mmap(nullptr, len, __PROT_READ | __PROT_WRITE, __MAP_PRIVATE | __MAP_ANON, -1, 0);
+    ASSERT_EQ(ret.error, 0);
+
+    uint8_t* mem = (uint8_t*)ret.value;
+    EXPECT_EQ((uintptr_t)mem % PAGE_SIZE, 0);
+    for (size_t i = 0; i < len; i += PAGE_SIZE)
+        EXPECT_EQ(mem[i], 0);
+    for (size_t i = 0; i < len; i++)
+        mem[i] = (uint8_t)i;
+    for (size_t i = 0; i < len; i++) {
+        if (mem[i] != (uint8_t)i) {
+            utest_fail(test, __FILE__, __LINE__, "byte %zu reads back as %u", i, mem[i]);
+            break;
+        }
+    }
+
+    EXPECT_EQ(sys_munmap(mem, len).error, 0);
+}
+
+UTEST(mman, rejects_zero_length) {
+    struct sys_ret ret = sys_mmap(nullptr, 0, __PROT_READ, __MAP_PRIVATE | __MAP_ANON, -1, 0);
+    EXPECT_EQ(ret.error, __EINVAL);
+}
+
+UTEST(mman, shared_anonymous_mapping) {
+    struct sys_ret ret = sys_mmap(nullptr, PAGE_SIZE, PROT_RW, __MAP_SHARED | __MAP_ANON, -1, 0);
+    ASSERT_EQ(ret.error, 0);
+    uint8_t* mem = (uint8_t*)ret.value;
+    EXPECT_EQ(mem[0], 0);
+    mem[0] = 0x5a;
+    mem[PAGE_SIZE - 1] = 0xa5;
+    EXPECT_EQ(mem[0], 0x5a);
+    EXPECT_EQ(mem[PAGE_SIZE - 1], 0xa5);
+    EXPECT_EQ(sys_munmap(mem, PAGE_SIZE).error, 0);
+}
+
+UTEST(mman, file_mapping) {
+    struct sys_ret ret = sys_openat(__AT_FDCWD, "/init", __O_RDONLY, 0);
+    ASSERT_EQ(ret.error, 0);
+    const int fd = (int)ret.value;
+
+    ret = sys_mmap(nullptr, PAGE_SIZE, __PROT_READ, __MAP_PRIVATE, fd, 0);
+    EXPECT_EQ(ret.error, 0);
+    if (!ret.error) {
+        const uint8_t elf_header[] = {0x7f, 'E', 'L', 'F'};
+        EXPECT_EQ(memcmp((const void*)ret.value, elf_header, sizeof(elf_header)), 0);
+        EXPECT_EQ(sys_munmap((void*)ret.value, PAGE_SIZE).error, 0);
+    }
+    EXPECT_EQ(sys_close(fd).error, 0);
+}
+
+UTEST(mman, fixed_mapping_replaces) {
+    uint8_t* mem = map_anon(3 * PAGE_SIZE);
+    ASSERT_TRUE(mem != nullptr);
+    memset(mem, 0xcc, 3 * PAGE_SIZE);
+
+    struct sys_ret ret = sys_mmap(mem + PAGE_SIZE, PAGE_SIZE, PROT_RW, MAP_ANON_PRIVATE | __MAP_FIXED, -1, 0);
+    EXPECT_EQ(ret.error, 0);
+    EXPECT_EQ(ret.value, (uintptr_t)(mem + PAGE_SIZE));
+    EXPECT_EQ(mem[0], 0xcc);
+    EXPECT_EQ(mem[PAGE_SIZE], 0);
+    EXPECT_EQ(mem[2 * PAGE_SIZE], 0xcc);
+    EXPECT_EQ(sys_munmap(mem, 3 * PAGE_SIZE).error, 0);
+}
+
+UTEST(mman, partial_unmap) {
+    uint8_t* mem = map_anon(3 * PAGE_SIZE);
+    ASSERT_TRUE(mem != nullptr);
+    EXPECT_EQ(sys_munmap(mem + PAGE_SIZE, PAGE_SIZE).error, 0);
+    mem[0] = 1;
+    mem[2 * PAGE_SIZE] = 2;
+    EXPECT_EQ(mem[0], 1);
+    EXPECT_EQ(mem[2 * PAGE_SIZE], 2);
+    EXPECT_EQ(sys_munmap(mem, 3 * PAGE_SIZE).error, 0);
+}
+
+UTEST(mman, rejects_invalid_arguments) {
+    EXPECT_EQ(sys_mmap(nullptr, PAGE_SIZE, 0x80, MAP_ANON_PRIVATE, -1, 0).error, __EINVAL);
+    EXPECT_EQ(sys_mmap(nullptr, PAGE_SIZE, PROT_RW, __MAP_ANON, -1, 0).error, __EINVAL);
+    EXPECT_EQ(sys_mmap(nullptr, PAGE_SIZE, PROT_RW, MAP_ANON_PRIVATE, -1, 1).error, __EINVAL);
+    EXPECT_EQ(sys_mmap((void*)0x1001, PAGE_SIZE, PROT_RW, MAP_ANON_PRIVATE | __MAP_FIXED, -1, 0).error, __EINVAL);
+    EXPECT_EQ(sys_mmap(nullptr, PAGE_SIZE, __PROT_READ, __MAP_PRIVATE, 1234, 0).error, __EBADF);
+}
+
+UTEST(mman, mprotect) {
+    uint8_t* mem = map_anon(2 * PAGE_SIZE);
+    ASSERT_TRUE(mem != nullptr);
+    mem[0] = 7;
+    EXPECT_EQ(sys_mprotect(mem, PAGE_SIZE, __PROT_READ).error, 0);
+    EXPECT_EQ(mem[0], 7);
+    EXPECT_EQ(sys_mprotect(mem, PAGE_SIZE, PROT_RW).error, 0);
+    mem[0] = 8;
+    EXPECT_EQ(mem[0], 8);
+    EXPECT_EQ(sys_mprotect(mem, PAGE_SIZE, 0x80).error, __EINVAL);
+    EXPECT_EQ(sys_mprotect(mem, 0, __PROT_READ).error, __EINVAL);
+    EXPECT_EQ(sys_munmap(mem, 2 * PAGE_SIZE).error, 0);
+}
+
+UTEST(mman, msync) {
+    uint8_t* mem = map_anon(PAGE_SIZE);
+    ASSERT_TRUE(mem != nullptr);
+    EXPECT_EQ(sys_msync(mem, PAGE_SIZE, __MS_SYNC).error, 0);
+    EXPECT_EQ(sys_msync(mem, PAGE_SIZE, __MS_ASYNC).error, 0);
+    EXPECT_EQ(sys_msync(mem, PAGE_SIZE, __MS_ASYNC | __MS_SYNC).error, __EINVAL);
+    EXPECT_EQ(sys_msync(mem + 1, PAGE_SIZE, __MS_SYNC).error, __EINVAL);
+    EXPECT_EQ(sys_munmap(mem, PAGE_SIZE).error, 0);
+    EXPECT_EQ(sys_msync(mem, PAGE_SIZE, __MS_ASYNC).error, __ENOMEM);
+}
+
+UTEST(mman, madvise) {
+    uint8_t* mem = map_anon(PAGE_SIZE);
+    ASSERT_TRUE(mem != nullptr);
+    EXPECT_EQ(sys_madvise(mem, PAGE_SIZE, 0).error, 0);
+    EXPECT_EQ(sys_madvise(mem + 1, PAGE_SIZE, 0).error, __EINVAL);
+    EXPECT_EQ(sys_munmap(mem, PAGE_SIZE).error, 0);
+}