x86_64: Enable interrupts in syscall

Kconfig

@@ -5,5 +5,8 @@ config KERNEL_FLAVOR
   default "vanilla"
 
 source "arch/$(ARCH)/Kconfig"
-source "core/Kconfig"
+source "boot/Kconfig"
+source "mem/Kconfig"
+source "sched/Kconfig"
 source "dev/Kconfig"
+source "tests/Kconfig"

arch/x86_64/Kconfig

@@ -4,8 +4,10 @@ config X86_64
   bool
   default y
   select ACPI
+  select HAVE_EFISTUB
   select HAVE_MMU
   select HAVE_MMU_GENERIC
+  select HAVE_TLB_SHOOTDOWN_IPI
   help
     64-bit x86 CPU support for AMD and Intel CPUs.
 

arch/x86_64/acpi.c

@@ -0,0 +1,36 @@
+#include <stdckdint.h>
+#include <stdint.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <x86_64/asm.h>
+
+uacpi_status uacpi_kernel_io_map(uacpi_io_addr base, uacpi_size len, uacpi_handle* out_handle) {
+    uintptr_t end;
+    if (ckd_add(&end, base, len) || end > 0x10000)
+        return UACPI_STATUS_INVALID_ARGUMENT;
+    *out_handle = (uacpi_handle)(uintptr_t)base;
+    return UACPI_STATUS_OK;
+}
+
+void uacpi_kernel_io_unmap(uacpi_handle handle) {
+    // Left intentionally blank.
+}
+
+#define IO_READ(width, suffix) \
+    uacpi_status uacpi_kernel_io_read##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width* out_value) { \
+        *out_value = asm_in##suffix((uintptr_t)handle + offset); \
+        return UACPI_STATUS_OK; \
+    }
+
+#define IO_WRITE(width, suffix) \
+    uacpi_status uacpi_kernel_io_write##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width in_value) { \
+        asm_out##suffix((uintptr_t)handle + offset, in_value); \
+        return UACPI_STATUS_OK; \
+    }
+
+IO_READ(8, b)
+IO_READ(16, w)
+IO_READ(32, l)
+IO_WRITE(8, b)
+IO_WRITE(16, w)
+IO_WRITE(32, l)

arch/x86_64/apic.c

@@ -7,6 +7,7 @@
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
 #include <arch/cpu.h>
+#include <arch/timer.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
@@ -95,17 +96,22 @@ void lapic_init(struct local_apic* lapic) {
     clock_spin_ns(10'000'000);
 
     lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
-    lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
-    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
+    lapic_write_reg(lapic, APIC_REG_LVT_TR, mmio_field(APIC_LVT_TR_VECTOR, IDT_TIMER));
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0);
+}
+
+void arch_timer_program(uint64_t deadline_ns) {
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+    const uint64_t now = clock_get_elapsed_ns();
+    const uint64_t delta = MIN(deadline_ns > now ? deadline_ns - now : 0, 1'000'000'000);
+    const uint64_t ticks = delta * lapic->ticks_per_10ms / 10'000'000;
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, MIN(MAX(ticks, 1), 0xFFFF'FFFF));
 }
 
 void lapic_eoi(struct local_apic* lapic) {
     lapic_write_reg(lapic, APIC_REG_EOI, 0);
 }
 
-constexpr uint64_t the_icr =
-    mmio_field(APIC_ICR_VECTOR, 128) | mmio_field(APIC_ICR_DELIVERY_MODE, APIC_DELIVERY_STARTUP);
-
 void lapic_send_ipi(
     struct local_apic* lapic,
     uint32_t target,

arch/x86_64/idt.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/irq.h>
@@ -6,6 +8,7 @@
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
+#include <zinnia/timer.h>
 #include <zinnia/usercopy.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
@@ -98,13 +101,23 @@ void interrupt_handler(struct arch_context* ctx) {
         lapic_eoi(&arch_cpu->lapic);
         pmap_shootdown_handle();
         break;
+    case IDT_TIMER:
+        lapic_eoi(&arch_cpu->lapic);
+        arch_sched_preempt_disable();
+        timer_interrupt();
+        if (arch_sched_preempt_enable() && (cpu->sched.need_resched || cpu->sched.owe_preempt))
+            sched_reschedule(&cpu->sched);
+        break;
     default:
         if (isr < 0x20)
             panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
         assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
         struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
         assert(line, "Unhandled interrupt on ISR %lu", isr);
+        arch_sched_preempt_disable();
         irq_line_raise(line);
+        if (arch_sched_preempt_enable() && cpu->sched.owe_preempt)
+            sched_reschedule(&cpu->sched);
     }
 }
 

arch/x86_64/include/bits/cpu.h

@@ -1,5 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 static inline void arch_cpu_pause() {
     asm volatile("pause");
 }
+
+#define ARCH_CACHE_LINE_SIZE 64

arch/x86_64/include/bits/elf.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/elf.h>

arch/x86_64/include/bits/pmap.h

@@ -1,5 +1,24 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
 
 typedef uint64_t pte_t;
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};

arch/x86_64/include/bits/sched.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/types.h>

arch/x86_64/include/bits/types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/include/x86_64/apic.h

@@ -1,5 +1,10 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
 #include <zinnia/mmio.h>
 #include <stdint.h>
 
@@ -79,3 +84,8 @@ void lapic_send_ipi(
 );
 
 void pic_disable();
+
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic);
+void apic_free_vector(uint8_t vector);
+
+INIT_EXTERN(ioapic_stage);

arch/x86_64/include/x86_64/asm.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/include/x86_64/defs.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
@@ -185,6 +187,7 @@ enum : uint64_t {
     CR4_OSFXSR = 1ull << 9,
     CR4_OSXMMEXCPT = 1ull << 10,
     CR4_UMIP = 1ull << 11,
+    CR4_LA57 = 1ull << 12,
     CR4_VMXE = 1ull << 13,
     CR4_SMXE = 1ull << 14,
     CR4_FSGSBASE = 1ull << 16,

arch/x86_64/include/x86_64/gdt.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/compiler.h>

arch/x86_64/include/x86_64/idt.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/init.h>

arch/x86_64/include/x86_64/percpu.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/irq.h>

arch/x86_64/include/x86_64/pmap.h

@@ -1,19 +1,4 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 #include <zinnia/pmap.h>
-
-enum : pte_t {
-    ARCH_FLAG_PRESENT = 1 << 0,
-    ARCH_FLAG_READ_WRITE = 1 << 1,
-    ARCH_FLAG_USER_MODE = 1 << 2,
-    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
-    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
-    ARCH_FLAG_ACCESSED = 1 << 5,
-    ARCH_FLAG_DIRTY = 1 << 6,
-    ARCH_FLAG_SIZE = 1 << 7,
-    ARCH_FLAG_GLOBAL = 1 << 8,
-    ARCH_FLAG_AVAILABLE = 1 << 9,
-    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
-    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
-
-    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
-};

arch/x86_64/include/x86_64/smp.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/atomic.h>

arch/x86_64/include/x86_64/syscall.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void arch_syscall_stub();

arch/x86_64/include/x86_64/tsc.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void tsc_init();

arch/x86_64/include/x86_64/tss.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/ioapic.c

@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "ioapic: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/percpu.h>
+
+constexpr size_t ISA_IRQS = 16;
+
+constexpr mmio_reg32 IOAPIC_REG_SELECT = {.offset = 0x00};
+constexpr mmio_reg32 IOAPIC_REG_WINDOW = {.offset = 0x10};
+
+constexpr uint32_t IOAPIC_VERSION = 0x01;
+constexpr uint32_t IOAPIC_REDIRECT = 0x10;
+
+constexpr uint64_t IOAPIC_RTE_POLARITY_LOW = 1 << 13;
+constexpr uint64_t IOAPIC_RTE_TRIGGER_LEVEL = 1 << 15;
+constexpr uint64_t IOAPIC_RTE_MASKED = 1 << 16;
+
+struct ioapic_line {
+    struct irq_line line;
+    struct ioapic* ioapic;
+    uint32_t pin;
+    uint32_t lapic;
+    uint8_t vector;
+};
+
+struct ioapic {
+    void* regs;
+    struct ioapic_line** lines;
+    struct spinlock lock;
+    uint32_t gsi_base;
+    uint32_t num_pins;
+};
+
+struct isa_override {
+    uint32_t gsi;
+    uint16_t flags;
+    bool present;
+};
+
+static struct ioapic* ioapics;
+static size_t num_ioapics;
+static struct isa_override isa_overrides[ISA_IRQS];
+static struct spinlock lines_lock;
+
+static uint32_t ioapic_read(struct ioapic* ioapic, uint32_t reg) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    return mmio_reg_read(ioapic->regs, IOAPIC_REG_WINDOW);
+}
+
+static void ioapic_write(struct ioapic* ioapic, uint32_t reg, uint32_t value) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_WINDOW, value);
+}
+
+static void write_rte(struct ioapic* ioapic, uint32_t pin, uint64_t rte) {
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, IOAPIC_RTE_MASKED);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2 + 1, rte >> 32);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, rte);
+}
+
+static void update_mask(struct ioapic_line* line, bool masked) {
+    struct ioapic* ioapic = line->ioapic;
+    spin_lock(&ioapic->lock);
+    uint32_t low = ioapic_read(ioapic, IOAPIC_REDIRECT + line->pin * 2);
+    low = masked ? low | IOAPIC_RTE_MASKED : low & ~IOAPIC_RTE_MASKED;
+    ioapic_write(ioapic, IOAPIC_REDIRECT + line->pin * 2, low);
+    spin_unlock(&ioapic->lock);
+}
+
+static enum irq_mode ioapic_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    struct ioapic_line* line = containerof(self, struct ioapic_line, line);
+    uint64_t rte = line->vector | ((uint64_t)line->lapic << 56);
+    if (polarity == IRQ_POLARITY_LOW)
+        rte |= IOAPIC_RTE_POLARITY_LOW;
+    if (trigger == IRQ_TRIGGER_LEVEL)
+        rte |= IOAPIC_RTE_TRIGGER_LEVEL;
+
+    spin_lock(&line->ioapic->lock);
+    write_rte(line->ioapic, line->pin, rte);
+    spin_unlock(&line->ioapic->lock);
+    return IRQ_MODE_EOI | IRQ_MODE_MASKABLE;
+}
+
+static void ioapic_mask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), true);
+}
+
+static void ioapic_unmask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), false);
+}
+
+static void ioapic_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_get_gsi(uint32_t gsi, struct irq_line** out) {
+    // Find the IOAPIC that handles this GSI.
+    struct ioapic* ioapic = nullptr;
+    for (size_t i = 0; i < num_ioapics; i++) {
+        if (gsi >= ioapics[i].gsi_base && gsi < ioapics[i].gsi_base + ioapics[i].num_pins) {
+            ioapic = &ioapics[i];
+            break;
+        }
+    }
+    if (!ioapic)
+        return ENOENT;
+    const uint32_t pin = gsi - ioapic->gsi_base;
+
+    spin_lock(&lines_lock);
+    struct ioapic_line* line = ioapic->lines[pin];
+    if (line) {
+        spin_unlock(&lines_lock);
+        *out = &line->line;
+        return 0;
+    }
+    spin_unlock(&lines_lock);
+
+    line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+    line->line.set_config = ioapic_set_config;
+    line->line.mask = ioapic_mask;
+    line->line.unmask = ioapic_unmask;
+    line->line.eoi = ioapic_eoi;
+    line->ioapic = ioapic;
+    line->pin = pin;
+
+    errno_t status = apic_alloc_vector(&line->line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
+
+    spin_lock(&lines_lock);
+    if (ioapic->lines[pin]) {
+        spin_unlock(&lines_lock);
+        apic_free_vector(line->vector);
+        mem_free(line);
+        return arch_irq_get_gsi(gsi, out);
+    }
+    ioapic->lines[pin] = line;
+    spin_unlock(&lines_lock);
+
+    *out = &line->line;
+    return 0;
+}
+
+bool arch_irq_isa_override(
+    uint8_t isa_irq,
+    uint32_t* out_gsi,
+    enum irq_trigger_mode* out_trigger,
+    enum irq_polarity* out_polarity
+) {
+    if (isa_irq >= ISA_IRQS || !isa_overrides[isa_irq].present)
+        return false;
+
+    const struct isa_override* iso = &isa_overrides[isa_irq];
+    *out_gsi = iso->gsi;
+    switch (iso->flags & ACPI_MADT_POLARITY_MASK) {
+    case ACPI_MADT_POLARITY_ACTIVE_HIGH:
+        *out_polarity = IRQ_POLARITY_HIGH;
+        break;
+    case ACPI_MADT_POLARITY_ACTIVE_LOW:
+        *out_polarity = IRQ_POLARITY_LOW;
+        break;
+    }
+    switch (iso->flags & ACPI_MADT_TRIGGERING_MASK) {
+    case ACPI_MADT_TRIGGERING_EDGE:
+        *out_trigger = IRQ_TRIGGER_EDGE;
+        break;
+    case ACPI_MADT_TRIGGERING_LEVEL:
+        *out_trigger = IRQ_TRIGGER_LEVEL;
+        break;
+    }
+    return true;
+}
+
+static uacpi_iteration_decision count_ioapics(uacpi_handle user, struct acpi_entry_hdr* hdr) {
+    size_t* count = user;
+    if (hdr->type == ACPI_MADT_ENTRY_TYPE_IOAPIC)
+        (*count)++;
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static uacpi_iteration_decision parse_madt_entry(uacpi_handle, struct acpi_entry_hdr* hdr) {
+    switch (hdr->type) {
+    case ACPI_MADT_ENTRY_TYPE_IOAPIC: {
+        // Map the IOAPIC and mask all of its pins.
+        const struct acpi_madt_ioapic* entry = (const struct acpi_madt_ioapic*)hdr;
+        uintptr_t vaddr = vm_alloc(arch_mem_page_size());
+        if (!vaddr)
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        pmap_map(&kernel_vma.pmap, vaddr, entry->address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+
+        struct ioapic* ioapic = &ioapics[num_ioapics];
+        ioapic->regs = (void*)vaddr;
+        ioapic->gsi_base = entry->gsi_base;
+        ioapic->num_pins = ((ioapic_read(ioapic, IOAPIC_VERSION) >> 16) & 0xFF) + 1;
+        spin_init(&ioapic->lock);
+        ioapic->lines = mem_alloc(ioapic->num_pins * sizeof(*ioapic->lines), 0);
+        if (!ioapic->lines) {
+            pmap_unmap(&kernel_vma.pmap, vaddr);
+            vm_free(vaddr, arch_mem_page_size());
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        }
+
+        for (uint32_t pin = 0; pin < ioapic->num_pins; pin++)
+            write_rte(ioapic, pin, IOAPIC_RTE_MASKED);
+
+        num_ioapics++;
+        log("IOAPIC %u at %#x handles GSIs %u-%u\n",
+            entry->id,
+            entry->address,
+            ioapic->gsi_base,
+            ioapic->gsi_base + ioapic->num_pins - 1);
+        break;
+    }
+    case ACPI_MADT_ENTRY_TYPE_INTERRUPT_SOURCE_OVERRIDE:
+        auto iso = (const struct acpi_madt_interrupt_source_override*)hdr;
+        if (iso->bus == 0 && iso->source < ISA_IRQS) {
+            isa_overrides[iso->source] = (struct isa_override){
+                .gsi = iso->gsi,
+                .flags = iso->flags,
+                .present = true,
+            };
+        }
+        break;
+    }
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static void ioapic_init(struct boot_info*) {
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log_warn("No MADT found\n");
+        return;
+    }
+
+    size_t capacity = 0;
+    uacpi_status status = uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), count_ioapics, &capacity);
+    if (status != UACPI_STATUS_OK)
+        log_warn("Failed to parse the MADT: %s\n", uacpi_status_to_string(status));
+
+    if (capacity) {
+        ioapics = mem_alloc(capacity * sizeof(*ioapics), 0);
+        if (!ioapics) {
+            log_err("Unable to allocate %zu controllers\n", capacity);
+            uacpi_table_unref(&table);
+            return;
+        }
+    }
+
+    uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), parse_madt_entry, nullptr);
+    uacpi_table_unref(&table);
+}
+INIT_TASK(ioapic_stage, "arch.x86_64.ioapic", ioapic_init);
+INIT_DEPENDS(ioapic_stage, acpi_stage, percpu_stage);
+INIT_ENTAILS(ioapic_stage, acpi_namespace_stage);

arch/x86_64/irq.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/percpu.h>
@@ -59,11 +61,7 @@ static void apic_msi_eoi(struct irq_line* self) {
     lapic_eoi(&percpu_ref(arch_data)->lapic);
 }
 
-errno_t arch_irq_alloc_msi(struct msi_line** out) {
-    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
-    if (!line)
-        return ENOMEM;
-
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic) {
     irq_lock();
     struct arch_percpu* arch_cpu = percpu_ref(arch_data);
     size_t slot;
@@ -73,20 +71,39 @@ errno_t arch_irq_alloc_msi(struct msi_line** out) {
     }
     if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
         irq_unlock();
-        mem_free(line);
         return ENOSPC;
     }
 
+    *out_vector = slot + 0x20;
+    *out_lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = line;
+    irq_unlock();
+    return 0;
+}
+
+void apic_free_vector(uint8_t vector) {
+    irq_lock();
+    percpu_ref(arch_data)->irq_lines[vector - 0x20] = nullptr;
+    irq_unlock();
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
     line->msi.line.set_config = apic_msi_set_config;
     line->msi.line.mask = apic_msi_mask;
     line->msi.line.unmask = apic_msi_unmask;
     line->msi.line.eoi = apic_msi_eoi;
     line->msi.msg_addr = apic_msi_msg_addr;
     line->msi.msg_data = apic_msi_msg_data;
-    line->vector = slot + 0x20;
-    line->lapic = arch_cpu->lapic.id;
-    arch_cpu->irq_lines[slot] = &line->msi.line;
-    irq_unlock();
+
+    errno_t status = apic_alloc_vector(&line->msi.line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
 
     *out = &line->msi;
     return 0;

arch/x86_64/mem.c

@@ -1,4 +1,3 @@
-#include <zinnia/types.h>
 #include <arch/mem.h>
 #include <stddef.h>
 #include <stdint.h>

arch/x86_64/meson.build

@@ -10,11 +10,11 @@ kernel_sources += files(
     'gdt.c',
     'hpet.c',
     'idt.c',
+    'ioapic.c',
     'irq.c',
     'mem.c',
     'percpu.c',
     'pmap.c',
-    'pmap.c',
     'random.c',
     'sched.c',
     'serial.c',
@@ -28,3 +28,7 @@ kernel_sources += files(
 if kconfig.get('CONFIG_PCI', 'n') == 'y'
     kernel_sources += files('pci.c')
 endif
+
+if kconfig.get('CONFIG_ACPI', 'n') == 'y'
+    kernel_sources += files('acpi.c')
+endif

arch/x86_64/pmap.c

@@ -11,6 +11,10 @@ void pte_clear(pte_t* pte) {
     *pte = 0;
 }
 
+void pte_write(pte_t* pte, pte_t val) {
+    *pte = val;
+}
+
 void pte_flush(uintptr_t vaddr) {
     asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
 }
@@ -67,12 +71,19 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     return result;
 }
 
-bool pte_is_present(pte_t* pte) {
-    return *pte & ARCH_FLAG_PRESENT;
+pte_t pte_protect(pte_t old, enum pte_flags flags) {
+    pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+    if (flags & PTE_WRITE)
+        result |= ARCH_FLAG_READ_WRITE;
+    if (!(flags & PTE_EXEC))
+        result |= ARCH_FLAG_EXECUTE_DISABLE;
+    return result;
 }
 
-bool pte_is_dir(pte_t* pte) {
-    return *pte & ARCH_FLAG_SIZE;
+bool pte_is_present(pte_t* pte) {
+    return *pte & ARCH_FLAG_PRESENT;
 }
 
 phys_t pte_address(pte_t* pte) {

arch/x86_64/sched.c

@@ -11,6 +11,7 @@
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
@@ -93,6 +94,11 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
     return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
 }
 
+constexpr size_t FPU_FCW_OFFSET = 0;
+constexpr size_t FPU_MXCSR_OFFSET = 24;
+constexpr uint16_t FPU_DEFAULT_FCW = 0x37F;
+constexpr uint32_t FPU_DEFAULT_MXCSR = 0x1F80;
+
 // Calls task_entry with the correct parameters.
 [[__naked]]
 void task_entry_thunk() {
@@ -142,6 +148,11 @@ errno_t arch_task_init(
         context->fpu_region = (void*)fpu_base;
         context->fpu_phys = pages;
 
+        const uint16_t fcw = FPU_DEFAULT_FCW;
+        const uint32_t mxcsr = FPU_DEFAULT_MXCSR;
+        memcpy((uint8_t*)fpu_base + FPU_FCW_OFFSET, &fcw, sizeof(fcw));
+        memcpy((uint8_t*)fpu_base + FPU_MXCSR_OFFSET, &mxcsr, sizeof(mxcsr));
+
         context->ds = asm_read_ds();
         context->es = asm_read_es();
         context->fs = asm_read_fs();

arch/x86_64/serial.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>

arch/x86_64/smp.c

@@ -148,7 +148,7 @@ static void smp_trampoline() {
           [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
           [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
           [cr0] "i"(CR0_PE | CR0_ET),
-          [cr0_pg] "i"(CR0_PG),
+          [cr0_pg] "i"((uint32_t)CR0_PG),
           [cr4_pae] "i"(CR4_PAE),
           [msr_efer] "i"(MSR_EFER),
           [efer_lme] "i"(MSR_EFER_LME),

arch/x86_64/syscall.c

@@ -40,6 +40,7 @@ void arch_syscall_stub() {
         "    push r15\n"
         "    xor rbp, rbp\n"
         "    mov rdi, rsp\n"          // SysV ABI first argument.
+        "    sti\n"
         "    call syscall_dispatch\n" // Call syscall handler.
         "    cli\n"
         "    pop r15\n"

boot/Kconfig

@@ -0,0 +1,19 @@
+config HAVE_EFISTUB
+  bool
+
+menu "Boot loaders"
+
+config BOOT_EFISTUB
+  bool "EFI stub"
+  depends on HAVE_EFISTUB
+  default y
+  help
+    Kernel loader for UEFI systems.
+
+config BOOT_LIMINE
+  bool "Limine"
+  default n
+  help
+    ELF loader with the Limine protocol.
+
+endmenu

boot/efi/efi.h

@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+
+typedef uint64_t efi_status;
+typedef void* efi_handle;
+typedef uint16_t char16;
+
+constexpr efi_status EFI_SUCCESS = 0;
+constexpr efi_status EFI_ERROR = 1ull << 63;
+constexpr efi_status EFI_LOAD_ERROR = EFI_ERROR | 1;
+constexpr efi_status EFI_INVALID_PARAMETER = EFI_ERROR | 2;
+constexpr efi_status EFI_UNSUPPORTED = EFI_ERROR | 3;
+constexpr efi_status EFI_BUFFER_TOO_SMALL = EFI_ERROR | 5;
+constexpr efi_status EFI_OUT_OF_RESOURCES = EFI_ERROR | 9;
+constexpr efi_status EFI_NOT_FOUND = EFI_ERROR | 14;
+
+struct efi_guid {
+    uint32_t data1;
+    uint16_t data2;
+    uint16_t data3;
+    uint8_t data4[8];
+};
+
+enum efi_memory_type : uint32_t {
+    EFI_RESERVED_MEMORY_TYPE,
+    EFI_LOADER_CODE,
+    EFI_LOADER_DATA,
+    EFI_BOOT_SERVICES_CODE,
+    EFI_BOOT_SERVICES_DATA,
+    EFI_RUNTIME_SERVICES_CODE,
+    EFI_RUNTIME_SERVICES_DATA,
+    EFI_CONVENTIONAL_MEMORY,
+    EFI_UNUSABLE_MEMORY,
+    EFI_ACPI_RECLAIM_MEMORY,
+    EFI_ACPI_MEMORY_NVS,
+    EFI_MEMORY_MAPPED_IO,
+    EFI_MEMORY_MAPPED_IO_PORT_SPACE,
+    EFI_PAL_CODE,
+    EFI_PERSISTENT_MEMORY,
+    EFI_UNACCEPTED_MEMORY,
+};
+
+enum efi_allocate_type : uint32_t {
+    EFI_ALLOCATE_ANY_PAGES,
+    EFI_ALLOCATE_MAX_ADDRESS,
+    EFI_ALLOCATE_ADDRESS,
+};
+
+enum efi_locate_search_type : uint32_t {
+    EFI_LOCATE_ALL_HANDLES,
+    EFI_LOCATE_BY_REGISTER_NOTIFY,
+    EFI_LOCATE_BY_PROTOCOL,
+};
+
+struct efi_memory_descriptor {
+    uint32_t type;
+    uint64_t physical_start;
+    uint64_t virtual_start;
+    uint64_t number_of_pages;
+    uint64_t attribute;
+};
+
+struct efi_table_header {
+    uint64_t signature;
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t crc32;
+    uint32_t reserved;
+};
+
+struct efi_simple_text_output_protocol {
+    void* reset;
+    efi_status(EFIAPI* output_string)(struct efi_simple_text_output_protocol* self, const char16* string);
+};
+
+struct efi_boot_services {
+    struct efi_table_header hdr;
+    void* raise_tpl;
+    void* restore_tpl;
+    efi_status(EFIAPI* allocate_pages)(
+        enum efi_allocate_type type,
+        enum efi_memory_type memory_type,
+        size_t pages,
+        uint64_t* memory
+    );
+    efi_status(EFIAPI* free_pages)(uint64_t memory, size_t pages);
+    efi_status(EFIAPI* get_memory_map)(
+        size_t* map_size,
+        struct efi_memory_descriptor* map,
+        size_t* map_key,
+        size_t* descriptor_size,
+        uint32_t* descriptor_version
+    );
+    efi_status(EFIAPI* allocate_pool)(enum efi_memory_type pool_type, size_t size, void** buffer);
+    efi_status(EFIAPI* free_pool)(void* buffer);
+    void* create_event;
+    void* set_timer;
+    void* wait_for_event;
+    void* signal_event;
+    void* close_event;
+    void* check_event;
+    void* install_protocol_interface;
+    void* reinstall_protocol_interface;
+    void* uninstall_protocol_interface;
+    efi_status(EFIAPI* handle_protocol)(efi_handle handle, const struct efi_guid* protocol, void** interface);
+    void* reserved;
+    void* register_protocol_notify;
+    void* locate_handle;
+    void* locate_device_path;
+    void* install_configuration_table;
+    void* load_image;
+    void* start_image;
+    void* exit;
+    void* unload_image;
+    efi_status(EFIAPI* exit_boot_services)(efi_handle image_handle, size_t map_key);
+    void* get_next_monotonic_count;
+    void* stall;
+    void* set_watchdog_timer;
+    void* connect_controller;
+    void* disconnect_controller;
+    void* open_protocol;
+    void* close_protocol;
+    void* open_protocol_information;
+    void* protocols_per_handle;
+    efi_status(EFIAPI* locate_handle_buffer)(
+        enum efi_locate_search_type search_type,
+        const struct efi_guid* protocol,
+        void* search_key,
+        size_t* no_handles,
+        efi_handle** buffer
+    );
+    efi_status(EFIAPI* locate_protocol)(const struct efi_guid* protocol, void* registration, void** interface);
+};
+
+struct efi_configuration_table {
+    struct efi_guid vendor_guid;
+    void* vendor_table;
+};
+
+struct efi_system_table {
+    struct efi_table_header hdr;
+    char16* firmware_vendor;
+    uint32_t firmware_revision;
+    efi_handle console_in_handle;
+    void* con_in;
+    efi_handle console_out_handle;
+    struct efi_simple_text_output_protocol* con_out;
+    efi_handle standard_error_handle;
+    void* std_err;
+    void* runtime_services;
+    struct efi_boot_services* boot_services;
+    size_t number_of_table_entries;
+    struct efi_configuration_table* configuration_table;
+};
+
+struct efi_loaded_image_protocol {
+    uint32_t revision;
+    efi_handle parent_handle;
+    struct efi_system_table* system_table;
+    efi_handle device_handle;
+    void* file_path;
+    void* reserved;
+    uint32_t load_options_size;
+    void* load_options;
+    void* image_base;
+    uint64_t image_size;
+    enum efi_memory_type image_code_type;
+    enum efi_memory_type image_data_type;
+    void* unload;
+};
+
+constexpr uint64_t EFI_FILE_MODE_READ = 1;
+
+struct efi_file_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open)(
+        struct efi_file_protocol* self,
+        struct efi_file_protocol** new_handle,
+        const char16* file_name,
+        uint64_t open_mode,
+        uint64_t attributes
+    );
+    efi_status(EFIAPI* close)(struct efi_file_protocol* self);
+    void* delete;
+    efi_status(EFIAPI* read)(struct efi_file_protocol* self, size_t* buffer_size, void* buffer);
+    void* write;
+    efi_status(EFIAPI* get_position)(struct efi_file_protocol* self, uint64_t* position);
+    efi_status(EFIAPI* set_position)(struct efi_file_protocol* self, uint64_t position);
+};
+
+struct efi_simple_file_system_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open_volume)(struct efi_simple_file_system_protocol* self, struct efi_file_protocol** root);
+};
+
+enum efi_pixel_format : uint32_t {
+    EFI_PIXEL_RGBX_8,
+    EFI_PIXEL_BGRX_8,
+    EFI_PIXEL_BIT_MASK,
+    EFI_PIXEL_BLT_ONLY,
+};
+
+struct efi_pixel_bitmask {
+    uint32_t red;
+    uint32_t green;
+    uint32_t blue;
+    uint32_t reserved;
+};
+
+struct efi_gop_mode_info {
+    uint32_t version;
+    uint32_t horizontal_resolution;
+    uint32_t vertical_resolution;
+    enum efi_pixel_format pixel_format;
+    struct efi_pixel_bitmask pixel_information;
+    uint32_t pixels_per_scan_line;
+};
+
+struct efi_gop_mode {
+    uint32_t max_mode;
+    uint32_t mode;
+    struct efi_gop_mode_info* info;
+    size_t size_of_info;
+    uint64_t frame_buffer_base;
+    size_t frame_buffer_size;
+};
+
+struct efi_graphics_output_protocol {
+    void* query_mode;
+    void* set_mode;
+    void* blt;
+    struct efi_gop_mode* mode;
+};
+
+static const struct efi_guid EFI_LOADED_IMAGE_PROTOCOL_GUID =
+    {0x5b1b'31a1, 0x9562, 0x11d2, {0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID =
+    {0x964e'5b22, 0x6459, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID =
+    {0x9042'a9de, 0x23dc, 0x4a38, {0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a}};
+static const struct efi_guid EFI_ACPI_20_TABLE_GUID =
+    {0x8868'e871, 0xe4f1, 0x11d3, {0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81}};
+static const struct efi_guid EFI_ACPI_10_TABLE_GUID =
+    {0xeb9d'2d30, 0x2d88, 0x11d3, {0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d}};

boot/efi/header.c

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+#include "pe.h"
+
+extern const char __pe_code_size[];
+extern const char __pe_data_sizes[];
+extern const char __pe_entry[];
+extern const char __pe_image_sizes[];
+extern const char __pe_base_relocs[];
+extern const char __pe_text_virtual[];
+extern const char __pe_text_raw[];
+extern const char __pe_reloc_virtual[];
+extern const char __pe_reloc_raw[];
+extern const char __pe_rodata_virtual[];
+extern const char __pe_rodata_raw[];
+extern const char __pe_data_virtual[];
+extern const char __pe_data_raw[];
+
+struct efistub_pe_header {
+    struct pe_dos_header dos;
+    uint32_t signature;
+    struct pe_coff_header coff;
+    struct pe_optional_header64 optional;
+    struct pe_section_header sections[4];
+};
+
+static_assert(offsetof(struct efistub_pe_header, optional) % 8 == 0);
+static_assert(offsetof(struct efistub_pe_header, sections) % 8 == 0);
+
+[[__used, __section(".header")]]
+const struct efistub_pe_header efistub_pe_header = {
+    .dos =
+        {
+            .e_magic = PE_DOS_MAGIC,
+            .e_lfanew = offsetof(struct efistub_pe_header, signature),
+        },
+    .signature = PE_SIGNATURE,
+    .coff =
+        {
+            .machine = EFI_ARCH_PE_MACHINE,
+            .number_of_sections = 4,
+            .size_of_optional_header = sizeof(struct pe_optional_header64),
+            .characteristics = PE_FILE_EXECUTABLE_IMAGE | PE_FILE_LINE_NUMS_STRIPPED | PE_FILE_DEBUG_STRIPPED,
+        },
+    .optional =
+        {
+            .magic_and_code_size = (uintptr_t)__pe_code_size + PE_OPTIONAL_MAGIC_PE32_PLUS,
+            .data_sizes = (uintptr_t)__pe_data_sizes,
+            .entry_and_code_base = (uintptr_t)__pe_entry,
+            .section_alignment = 0x1000,
+            .file_alignment = 0x1000,
+            .image_and_header_sizes = (uintptr_t)__pe_image_sizes,
+            .subsystem = PE_SUBSYSTEM_EFI_APPLICATION,
+            .dll_characteristics = PE_DLL_NX_COMPAT,
+            .number_of_rva_and_sizes = PE_NUM_DATA_DIRECTORIES,
+            .data_directories =
+                {
+                    [PE_DIRECTORY_BASE_RELOC] = {.layout = (uintptr_t)__pe_base_relocs},
+                },
+        },
+    .sections = {
+        {
+            .name = ".text",
+            .virtual_layout = (uintptr_t)__pe_text_virtual,
+            .raw_layout = (uintptr_t)__pe_text_raw,
+            .characteristics = PE_SCN_CNT_CODE | PE_SCN_MEM_EXECUTE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".reloc",
+            .virtual_layout = (uintptr_t)__pe_reloc_virtual,
+            .raw_layout = (uintptr_t)__pe_reloc_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_DISCARDABLE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".rdata",
+            .virtual_layout = (uintptr_t)__pe_rodata_virtual,
+            .raw_layout = (uintptr_t)__pe_rodata_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".data",
+            .virtual_layout = (uintptr_t)__pe_data_virtual,
+            .raw_layout = (uintptr_t)__pe_data_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ | PE_SCN_MEM_WRITE,
+        },
+    },
+};
+
+[[__used, __section(".reloc")]]
+const struct pe_base_reloc_block efistub_base_relocs = {
+    .page_rva = 0,
+    .block_size = sizeof(struct pe_base_reloc_block),
+};

boot/efi/main.c

@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: ISC
+
+#include <string.h>
+#include "stub.h"
+
+constexpr size_t MEM_MAP_SLACK = 64;
+
+struct efi_boot_services* bs;
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    enum efi_memory_type efi_type = EFI_LOADER_DATA;
+    if (usage == PHYS_STATIC)
+        efi_type = MEM_STATIC;
+    else if (usage == PHYS_RECLAIMABLE)
+        efi_type = MEM_RECLAIMABLE;
+
+    const size_t pages = ALIGN_UP(size, PAGE_SIZE) / PAGE_SIZE;
+    if (bs->allocate_pages(EFI_ALLOCATE_ANY_PAGES, efi_type, pages, out))
+        return false;
+    memset((void*)*out, 0, pages * PAGE_SIZE);
+    return true;
+}
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)phys;
+}
+
+static efi_status open_file_on(efi_handle handle, const char16* path, struct efi_file_protocol** out) {
+    struct efi_simple_file_system_protocol* fs;
+    efi_status status = bs->handle_protocol(handle, &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID, (void**)&fs);
+    if (status)
+        return status;
+
+    struct efi_file_protocol* root;
+    status = fs->open_volume(fs, &root);
+    if (status)
+        return status;
+
+    status = root->open(root, out, path, EFI_FILE_MODE_READ, 0);
+    if (status && path[0] == '\\')
+        status = root->open(root, out, path + 1, EFI_FILE_MODE_READ, 0);
+    root->close(root);
+    return status;
+}
+
+static efi_status read_file(struct efi_file_protocol* file, enum phys_mem_usage usage, uint64_t* base, uint64_t* size) {
+    uint64_t total;
+    uint64_t done = 0;
+    efi_status status = file->set_position(file, UINT64_MAX);
+    if (!status)
+        status = file->get_position(file, &total);
+    if (!status)
+        status = file->set_position(file, 0);
+    if (!status && !loader_alloc(usage, total ? total : 1, base))
+        status = EFI_OUT_OF_RESOURCES;
+    while (!status && done < total) {
+        size_t chunk = total - done;
+        status = file->read(file, &chunk, (void*)(*base + done));
+        if (!chunk)
+            break;
+        done += chunk;
+    }
+    file->close(file);
+    *size = done;
+    return status;
+}
+
+static void color_from_mask(uint32_t mask, struct boot_color* out) {
+    if (!mask)
+        return;
+    out->shift = __builtin_ctz(mask);
+    out->size = __builtin_ctzll(~(uint64_t)(mask >> out->shift));
+}
+
+static enum phys_mem_usage convert_mem_type(uint32_t type) {
+    switch (type) {
+    case EFI_LOADER_CODE:
+    case EFI_LOADER_DATA:
+    case EFI_BOOT_SERVICES_CODE:
+    case EFI_BOOT_SERVICES_DATA:
+    case EFI_CONVENTIONAL_MEMORY:
+        return PHYS_USABLE;
+    case EFI_ACPI_RECLAIM_MEMORY:
+    case MEM_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case MEM_STATIC:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+efi_status efi_main(efi_handle image_handle, struct efi_system_table* st) {
+    bs = st->boot_services;
+
+    // Check if we support this configuration.
+    if (!arch_check_cpu())
+        return EFI_UNSUPPORTED;
+
+    struct efi_loaded_image_protocol* image;
+    efi_status status = bs->handle_protocol(image_handle, &EFI_LOADED_IMAGE_PROTOCOL_GUID, (void**)&image);
+    if (status)
+        return status;
+
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, "EFISTUB"))
+        return EFI_LOAD_ERROR;
+    struct boot_info* info = kernel.info;
+
+    // Reserve room for the memory map.
+    size_t map_size = 0;
+    size_t map_key;
+    size_t desc_size;
+    uint32_t desc_version;
+    status = bs->get_memory_map(&map_size, nullptr, &map_key, &desc_size, &desc_version);
+    if (status != EFI_BUFFER_TOO_SMALL)
+        return status;
+
+    const size_t map_capacity = map_size + MEM_MAP_SLACK * desc_size;
+    uint64_t map;
+    if (!loader_alloc(PHYS_USABLE, map_capacity, &map))
+        return EFI_OUT_OF_RESOURCES;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_USABLE, map_capacity / desc_size * sizeof(struct phys_mem), &mem))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Use the load options as the command line.
+    const char16* opts = image->load_options;
+    const size_t opts_len = image->load_options_size / sizeof(char16);
+    size_t cmdline_len = 0;
+    for (size_t i = 0; opts && i < opts_len && opts[i]; i++) {
+        if (opts[i] < 0x20 || opts[i] > 0x7e || cmdline_len + 1 >= sizeof(info->cmdline)) {
+            cmdline_len = 0;
+            break;
+        }
+        info->cmdline[cmdline_len++] = (char)opts[i];
+    }
+    info->cmdline[cmdline_len] = 0;
+
+    // Without load options, fall back to a command line file on the boot device.
+    struct efi_file_protocol* cmdline_file;
+    if (!cmdline_len && !open_file_on(image->device_handle, u"\\EFI\\zinnia\\cmdline", &cmdline_file)) {
+        uint64_t base;
+        uint64_t size;
+        status = read_file(cmdline_file, PHYS_USABLE, &base, &size);
+        if (status)
+            return status;
+
+        const char* text = (const char*)base;
+        while (cmdline_len < size && text[cmdline_len] && text[cmdline_len] != '\n' && text[cmdline_len] != '\r') {
+            if (text[cmdline_len] < 0x20 || text[cmdline_len] > 0x7e)
+                return EFI_INVALID_PARAMETER;
+            if (cmdline_len + 1 >= sizeof(info->cmdline))
+                return EFI_BUFFER_TOO_SMALL;
+            info->cmdline[cmdline_len] = text[cmdline_len];
+            cmdline_len++;
+        }
+        while (cmdline_len && info->cmdline[cmdline_len - 1] == ' ')
+            cmdline_len--;
+        info->cmdline[cmdline_len] = 0;
+    }
+
+    // Load every initrd= file.
+    const char* cmdline = info->cmdline;
+    for (size_t i = 0; cmdline[i];) {
+        if (cmdline[i] == ' ') {
+            i++;
+            continue;
+        }
+
+        const size_t start = i;
+        bool quoted = false;
+        while (cmdline[i] && (quoted || cmdline[i] != ' ')) {
+            if (cmdline[i] == '"')
+                quoted = !quoted;
+            i++;
+        }
+        const char* token = cmdline + start;
+        const size_t len = i - start;
+        if (len <= 7 || memcmp(token, "initrd=", 7))
+            continue;
+
+        const size_t path_len = len - 7;
+        if (path_len >= BOOT_PATH_MAX)
+            return EFI_INVALID_PARAMETER;
+        char path[BOOT_PATH_MAX];
+        char16 wide[BOOT_PATH_MAX];
+        for (size_t j = 0; j < path_len; j++) {
+            path[j] = token[7 + j];
+            wide[j] = path[j] == '/' ? '\\' : (char16)path[j];
+        }
+        path[path_len] = 0;
+        wide[path_len] = 0;
+
+        // Try the boot device first, then all other file systems.
+        struct efi_file_protocol* file;
+        status = open_file_on(image->device_handle, wide, &file);
+        size_t num_handles;
+        efi_handle* handles;
+        if (status &&
+            !bs->locate_handle_buffer(
+                EFI_LOCATE_BY_PROTOCOL,
+                &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID,
+                nullptr,
+                &num_handles,
+                &handles
+            )) {
+            for (size_t j = 0; j < num_handles && status; j++) {
+                if (handles[j] != image->device_handle)
+                    status = open_file_on(handles[j], wide, &file);
+            }
+            bs->free_pool(handles);
+        }
+        if (status)
+            return status;
+
+        uint64_t base;
+        uint64_t size;
+        status = read_file(file, PHYS_STATIC, &base, &size);
+        if (status)
+            return status;
+
+        if (!loader_add_file(&kernel, path, base, size))
+            return EFI_OUT_OF_RESOURCES;
+    }
+
+    // Framebuffer
+    struct efi_graphics_output_protocol* gop;
+    if (!bs->locate_protocol(&EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, nullptr, (void**)&gop) && gop && gop->mode &&
+        gop->mode->info) {
+        const struct efi_gop_mode_info* mode = gop->mode->info;
+        struct boot_framebuffer* fb = &info->fb;
+        switch (mode->pixel_format) {
+        case EFI_PIXEL_RGBX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){0, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){16, 8};
+            break;
+        case EFI_PIXEL_BGRX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){16, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){0, 8};
+            break;
+        case EFI_PIXEL_BIT_MASK: {
+            const struct efi_pixel_bitmask* m = &mode->pixel_information;
+            const uint32_t all = m->red | m->green | m->blue | m->reserved;
+            if (!all)
+                break;
+            fb->bytes_per_pixel = (32 - __builtin_clz(all) + 7) / 8;
+            color_from_mask(m->red, &fb->red);
+            color_from_mask(m->green, &fb->green);
+            color_from_mask(m->blue, &fb->blue);
+            break;
+        }
+        default:
+            break;
+        }
+
+        if (fb->bytes_per_pixel) {
+            fb->base = gop->mode->frame_buffer_base;
+            fb->width = mode->horizontal_resolution;
+            fb->height = mode->vertical_resolution;
+            fb->pitch = mode->pixels_per_scan_line * fb->bytes_per_pixel;
+        }
+    }
+
+    // Find the RSDP.
+    for (size_t i = 0; i < st->number_of_table_entries; i++) {
+        const struct efi_configuration_table* table = &st->configuration_table[i];
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_20_TABLE_GUID, sizeof(struct efi_guid))) {
+            info->rsdp = (uint64_t)table->vendor_table;
+            break;
+        }
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_10_TABLE_GUID, sizeof(struct efi_guid)))
+            info->rsdp = (uint64_t)table->vendor_table;
+    }
+
+    // Map everything up to the highest non-reserved address.
+    map_size = map_capacity;
+    status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+    if (status)
+        return status;
+
+    uint64_t top = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        if (convert_mem_type(desc->type) != PHYS_RESERVED)
+            top = MAX(top, desc->physical_start + desc->number_of_pages * PAGE_SIZE);
+    }
+
+    uint64_t root;
+    if (!loader_page_tables(&kernel, top, (uint64_t)arch_enter_kernel, &root))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Exit boot services.
+    for (size_t attempt = 0;; attempt++) {
+        map_size = map_capacity;
+        status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+        if (status)
+            return status;
+        status = bs->exit_boot_services(image_handle, map_key);
+        if (!status)
+            break;
+        if (attempt == 8)
+            return status;
+    }
+
+    // Hand the final memory map to the kernel.
+    struct phys_mem* mem_map = (struct phys_mem*)mem;
+    size_t num_mem = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        mem_map[num_mem++] = (struct phys_mem){
+            .address = desc->physical_start,
+            .length = desc->number_of_pages * PAGE_SIZE,
+            .usage = convert_mem_type(desc->type),
+        };
+    }
+    loader_set_mem_map(&kernel, mem_map, num_mem);
+
+    arch_enter_kernel(root, kernel.entry);
+}

boot/efi/meson.build

@@ -0,0 +1,42 @@
+efistub_sources = files(
+    'header.c',
+    'main.c',
+    'reloc.c',
+)
+
+efistub_lds = meson.current_source_dir() / 'stub.ld'
+
+subdir(target_machine.cpu_family())
+
+efistub_elf = executable(
+    'efistub.elf',
+    sources: [loader_sources, efistub_sources, loader_kernel],
+    include_directories: [
+        common_includes,
+        loader_includes,
+        include_directories('.', target_machine.cpu_family()),
+    ],
+    c_args: loader_c_args,
+    link_args: [
+        '-nostdlib',
+        '-Wl,-T,' + efistub_lds,
+        '-Wl,--no-dynamic-linker',
+        '-Wl,-z,text',
+        '-Wl,-z,norelro',
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: efistub_lds,
+    pie: true,
+)
+
+zinnia_efi = custom_target(
+    'zinnia.efi',
+    input: efistub_elf,
+    output: 'zinnia.efi',
+    command: [objcopy, '-O', 'binary', '@INPUT@', '@OUTPUT@'],
+    build_by_default: true,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/efi/pe.h

@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint16_t PE_DOS_MAGIC = 0x5a4d;
+constexpr uint32_t PE_SIGNATURE = 0x0000'4550;
+constexpr uint16_t PE_OPTIONAL_MAGIC_PE32_PLUS = 0x020b;
+constexpr uint16_t PE_SUBSYSTEM_EFI_APPLICATION = 10;
+constexpr size_t PE_NUM_DATA_DIRECTORIES = 16;
+constexpr size_t PE_DIRECTORY_BASE_RELOC = 5;
+
+enum pe_file_characteristics : uint16_t {
+    PE_FILE_EXECUTABLE_IMAGE = 0x0002,
+    PE_FILE_LINE_NUMS_STRIPPED = 0x0004,
+    PE_FILE_DEBUG_STRIPPED = 0x0200,
+};
+
+enum pe_dll_characteristics : uint16_t {
+    PE_DLL_NX_COMPAT = 0x0100,
+};
+
+enum pe_section_characteristics : uint32_t {
+    PE_SCN_CNT_CODE = 0x0000'0020,
+    PE_SCN_CNT_INITIALIZED_DATA = 0x0000'0040,
+    PE_SCN_MEM_DISCARDABLE = 0x0200'0000,
+    PE_SCN_MEM_EXECUTE = 0x2000'0000,
+    PE_SCN_MEM_READ = 0x4000'0000,
+    PE_SCN_MEM_WRITE = 0x8000'0000,
+};
+
+struct pe_dos_header {
+    uint16_t e_magic;
+    uint16_t e_cblp;
+    uint16_t e_cp;
+    uint16_t e_crlc;
+    uint16_t e_cparhdr;
+    uint16_t e_minalloc;
+    uint16_t e_maxalloc;
+    uint16_t e_ss;
+    uint16_t e_sp;
+    uint16_t e_csum;
+    uint16_t e_ip;
+    uint16_t e_cs;
+    uint16_t e_lfarlc;
+    uint16_t e_ovno;
+    uint16_t e_res[4];
+    uint16_t e_oemid;
+    uint16_t e_oeminfo;
+    uint16_t e_res2[10];
+    uint32_t e_lfanew;
+};
+
+struct pe_coff_header {
+    uint16_t machine;
+    uint16_t number_of_sections;
+    uint32_t time_date_stamp;
+    uint32_t pointer_to_symbol_table;
+    uint32_t number_of_symbols;
+    uint16_t size_of_optional_header;
+    uint16_t characteristics;
+};
+
+struct pe_data_directory {
+    union {
+        struct {
+            uint32_t virtual_address;
+            uint32_t size;
+        };
+        uint64_t layout;
+    };
+};
+
+struct pe_optional_header64 {
+    union {
+        struct {
+            uint16_t magic;
+            uint8_t major_linker_version;
+            uint8_t minor_linker_version;
+            uint32_t size_of_code;
+        };
+        uint64_t magic_and_code_size;
+    };
+    union {
+        struct {
+            uint32_t size_of_initialized_data;
+            uint32_t size_of_uninitialized_data;
+        };
+        uint64_t data_sizes;
+    };
+    union {
+        struct {
+            uint32_t address_of_entry_point;
+            uint32_t base_of_code;
+        };
+        uint64_t entry_and_code_base;
+    };
+    uint64_t image_base;
+    uint32_t section_alignment;
+    uint32_t file_alignment;
+    uint16_t major_operating_system_version;
+    uint16_t minor_operating_system_version;
+    uint16_t major_image_version;
+    uint16_t minor_image_version;
+    uint16_t major_subsystem_version;
+    uint16_t minor_subsystem_version;
+    uint32_t win32_version_value;
+    union {
+        struct {
+            uint32_t size_of_image;
+            uint32_t size_of_headers;
+        };
+        uint64_t image_and_header_sizes;
+    };
+    uint32_t checksum;
+    uint16_t subsystem;
+    uint16_t dll_characteristics;
+    uint64_t size_of_stack_reserve;
+    uint64_t size_of_stack_commit;
+    uint64_t size_of_heap_reserve;
+    uint64_t size_of_heap_commit;
+    uint32_t loader_flags;
+    uint32_t number_of_rva_and_sizes;
+    struct pe_data_directory data_directories[PE_NUM_DATA_DIRECTORIES];
+};
+
+struct pe_section_header {
+    char name[8];
+    union {
+        struct {
+            uint32_t virtual_size;
+            uint32_t virtual_address;
+        };
+        uint64_t virtual_layout;
+    };
+    union {
+        struct {
+            uint32_t size_of_raw_data;
+            uint32_t pointer_to_raw_data;
+        };
+        uint64_t raw_layout;
+    };
+    uint32_t pointer_to_relocations;
+    uint32_t pointer_to_line_numbers;
+    uint16_t number_of_relocations;
+    uint16_t number_of_line_numbers;
+    uint32_t characteristics;
+};
+
+struct pe_base_reloc_block {
+    uint32_t page_rva;
+    uint32_t block_size;
+    uint16_t entries[2];
+};
+
+static_assert(sizeof(struct pe_dos_header) == 64);
+static_assert(sizeof(struct pe_coff_header) == 20);
+static_assert(sizeof(struct pe_optional_header64) == 240);
+static_assert(offsetof(struct pe_optional_header64, size_of_image) == 56);
+static_assert(offsetof(struct pe_optional_header64, data_directories) == 112);
+static_assert(sizeof(struct pe_section_header) == 40);
+static_assert(sizeof(struct pe_base_reloc_block) == 12);

boot/efi/reloc.c

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+
+efi_status efistub_relocate(uintptr_t base, const elf64_dyn_t* dynamic) {
+    uintptr_t rela = 0;
+    size_t rela_size = 0;
+    size_t rela_ent = sizeof(elf64_rela_t);
+    uintptr_t relr = 0;
+    size_t relr_size = 0;
+
+    for (const elf64_dyn_t* dyn = dynamic; dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = base + dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        case DT_RELR:
+            relr = base + dyn->d_un.d_val;
+            break;
+        case DT_RELRSZ:
+            relr_size = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    for (size_t off = 0; rela && off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = (const elf64_rela_t*)(rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return EFI_LOAD_ERROR;
+        *(uint64_t*)(base + r->r_offset) = base + r->r_addend;
+    }
+
+    uint64_t* where = nullptr;
+    for (size_t off = 0; relr && off + sizeof(uint64_t) <= relr_size; off += sizeof(uint64_t)) {
+        uint64_t entry = *(const uint64_t*)(relr + off);
+        if (!(entry & 1)) {
+            where = (uint64_t*)(base + entry);
+            *where++ += base;
+            continue;
+        }
+        for (size_t i = 0; (entry >>= 1) != 0; i++) {
+            if (entry & 1)
+                where[i] += base;
+        }
+        where += 63;
+    }
+
+    return EFI_SUCCESS;
+}

boot/efi/stub.h

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+#include "loader.h"
+
+constexpr enum efi_memory_type MEM_STATIC = (enum efi_memory_type)0x8000'0000;
+constexpr enum efi_memory_type MEM_RECLAIMABLE = (enum efi_memory_type)0x8000'0001;
+
+extern struct efi_boot_services* bs;

boot/efi/stub.ld

@@ -0,0 +1,100 @@
+ENTRY(_start)
+
+PHDRS
+{
+    text    PT_LOAD    FLAGS(5);
+    rodata  PT_LOAD    FLAGS(4);
+    data    PT_LOAD    FLAGS(6);
+    dynamic PT_DYNAMIC FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0;
+
+    .header : {
+        __image_base = .;
+        KEEP(*(.header))
+    } :text
+
+    . = ALIGN(0x1000);
+    .text : {
+        __text_start = .;
+        *(.text .text.*)
+        . = ALIGN(0x1000);
+        __text_end = .;
+    } :text
+
+    .reloc : {
+        __reloc_start = .;
+        KEEP(*(.reloc))
+        __reloc_data_end = .;
+        . = ALIGN(0x1000);
+        __reloc_end = .;
+    } :rodata
+
+    .rodata : {
+        __rodata_start = .;
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    .hash : { *(.hash) } :rodata
+    .gnu.hash : { *(.gnu.hash) } :rodata
+    .dynsym : { *(.dynsym) } :rodata
+    .dynstr : { *(.dynstr) } :rodata
+    .rela.dyn : { *(.rela.*) } :rodata
+    .relr.dyn : { *(.relr.dyn) } :rodata
+
+    .rodata.end : {
+        . = ALIGN(0x1000);
+        __rodata_end = .;
+    } :rodata
+
+    .dynamic : {
+        __data_start = .;
+        *(.dynamic)
+    } :data :dynamic
+
+    .data : {
+        *(.data .data.*)
+        *(.got .got.*)
+        . = ALIGN(0x1000);
+        __data_raw_end = .;
+    } :data
+
+    .bss (NOLOAD) : {
+        __bss_start = .;
+        *(.bss .bss.*)
+        *(COMMON)
+        . = ALIGN(0x1000);
+        __bss_end = .;
+    } :data
+
+    __image_end = .;
+
+    __pe_code_size = ABSOLUTE((__text_end - __text_start) << 32);
+    __pe_data_sizes = ABSOLUTE(((__reloc_end - __reloc_start) + (__rodata_end - __rodata_start) +
+                                (__data_raw_end - __data_start)) | ((__bss_end - __bss_start) << 32));
+    __pe_entry = ABSOLUTE(_start | (__text_start << 32));
+    __pe_image_sizes = ABSOLUTE(__image_end | (__text_start << 32));
+    __pe_base_relocs = ABSOLUTE(__reloc_start | ((__reloc_data_end - __reloc_start) << 32));
+    __pe_text_virtual = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_text_raw = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_reloc_virtual = ABSOLUTE((__reloc_data_end - __reloc_start) | (__reloc_start << 32));
+    __pe_reloc_raw = ABSOLUTE((__reloc_end - __reloc_start) | (__reloc_start << 32));
+    __pe_rodata_virtual = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_rodata_raw = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_data_virtual = ABSOLUTE((__bss_end - __data_start) | (__data_start << 32));
+    __pe_data_raw = ABSOLUTE((__data_raw_end - __data_start) | (__data_start << 32));
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/efi/x86_64/efi_arch.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+#define EFIAPI __attribute__((ms_abi))
+
+constexpr uint16_t EFI_ARCH_PE_MACHINE = 0x8664;

boot/efi/x86_64/entry.S

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl _start
+    .type _start, @function
+_start:
+    // Align the stack pointer
+    push rbp
+    mov rbp, rsp
+    push rdi
+    push rsi
+    push rbx
+    push r12
+    and rsp, -16
+    cld
+
+    mov rbx, rcx
+    mov r12, rdx
+
+    // Clear .bss
+    lea rdi, [rip + __bss_start]
+    lea rcx, [rip + __bss_end]
+    sub rcx, rdi
+    xor eax, eax
+    rep stosb
+
+    // Self-relocate
+    lea rdi, [rip + __image_base]
+    lea rsi, [rip + _DYNAMIC]
+    call efistub_relocate
+    test rax, rax
+    jnz 1f
+
+    mov rdi, rbx
+    mov rsi, r12
+    call efi_main
+
+1:
+    lea rsp, [rbp - 32]
+    pop r12
+    pop rbx
+    pop rsi
+    pop rdi
+    pop rbp
+    ret
+    .size _start, . - _start

boot/efi/x86_64/meson.build

@@ -0,0 +1 @@
+efistub_sources += files('entry.S')

boot/kernel.ld

@@ -7,6 +7,7 @@ PHDRS {
     dynamic PT_DYNAMIC  FLAGS(6);
     percpu  PT_LOAD     FLAGS(6);
     cmdline 0x60000005;
+    boot    0x60000008;
 }
 
 SECTIONS
@@ -49,6 +50,11 @@ SECTIONS
         KEEP(*(.initgraph.edges))
         __ld_initgraph_edges_end = .;
 
+        . = ALIGN(8);
+        __ld_ktest_start = .;
+        KEEP(*(.ktest))
+        __ld_ktest_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -62,6 +68,10 @@ SECTIONS
         *(.data .data.*)
     } :data
 
+    .boot : {
+        KEEP(*(.boot.info))
+    } :data :boot
+
     .dynsym : {
         __ld_dynsym_start = .;
         KEEP(*(.dynsym*))

boot/limine/entry.c

@@ -1,128 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/cmdline.h>
-#include <zinnia/init.h>
-#include <zinnia/print.h>
-#include <zinnia/utils.h>
-#include <string.h>
-#include "limine.h"
-
-[[__initdata_sorted("limine.0")]]
-static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
-[[__initdata_sorted("limine.1")]]
-static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
-[[__initdata_sorted("limine.2")]]
-static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
-
-#define LIMINE_REQUEST(request, tag, rev) \
-    [[__initdata_sorted("limine.1")]] \
-    static volatile struct limine_##request request = { \
-        .id = tag, \
-        .revision = rev, \
-        .response = nullptr, \
-    }
-
-LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
-LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
-LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
-LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
-LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
-LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
-LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
-LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
-
-[[__initdata]]
-static struct phys_mem mem[128];
-[[__initdata]]
-static struct boot_file files[8];
-
-[[__init]]
-void kernel_entry() {
-    kernel_early_init();
-
-    struct boot_info info = {};
-
-    const struct limine_memmap_response* mm_res = memmap_request.response;
-    const struct limine_hhdm_response* hhdm_res = hhdm_request.response;
-    const struct limine_module_response* module_res = module_request.response;
-    const struct limine_executable_address_response* exec_res = executable_address_request.response;
-    const struct limine_executable_file_response* exec_file_res = executable_file_request.response;
-    const struct limine_rsdp_response* rsdp_res = rsdp_request.response;
-
-    assert(mm_res, "memmap_request returned no response!");
-    assert(hhdm_res, "hhdm_request returned no response!");
-
-    info.cmdline = exec_file_res->executable_file->string;
-    info.kernel_file.data = (phys_t)((uintptr_t)exec_file_res->executable_file->address - hhdm_res->offset);
-    info.kernel_file.length = exec_file_res->executable_file->size;
-
-    info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
-    for (size_t i = 0; i < info.num_mem_maps; i++) {
-        mem[i].address = mm_res->entries[i]->base;
-        mem[i].length = mm_res->entries[i]->length;
-
-        switch (mm_res->entries[i]->type) {
-        case LIMINE_MEMMAP_USABLE:
-            mem[i].usage = PHYS_USABLE;
-            break;
-        case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
-        case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
-            mem[i].usage = PHYS_RECLAIMABLE;
-            break;
-        case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
-            mem[i].usage = PHYS_STATIC;
-            break;
-        default:
-            mem[i].usage = PHYS_RESERVED;
-            break;
-        }
-    }
-
-    if (module_res) {
-        info.files = files;
-        info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
-        for (size_t i = 0; i < info.num_files; i++) {
-            files[i].data = (phys_t)module_res->modules[i]->address - hhdm_res->offset;
-            files[i].length = module_res->modules[i]->size;
-            strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1);
-            files[i].path[sizeof(files[i].path) - 1] = 0;
-        }
-    }
-
-    info.mem_map = mem;
-    info.phys_base = (phys_t)exec_res->physical_base;
-    info.virt_base = (uintptr_t)exec_res->virtual_base;
-    info.hhdm_base = (uintptr_t)hhdm_res->offset;
-
-    const struct limine_framebuffer_response* fb_res = framebuffer_request.response;
-    if (fb_res) {
-        // Only use the first framebuffer.
-        if (fb_res->framebuffer_count >= 1) {
-            const struct limine_framebuffer* fb = framebuffer_request.response->framebuffers[0];
-            info.fb.base = (phys_t)((uintptr_t)fb->address - hhdm_res->offset);
-            info.fb.width = fb->width;
-            info.fb.height = fb->height;
-            info.fb.pitch = fb->pitch;
-            info.fb.bytes_per_pixel = fb->bpp / 8;
-
-            info.fb.red.shift = fb->red_mask_shift;
-            info.fb.red.size = fb->red_mask_size;
-
-            info.fb.green.shift = fb->green_mask_shift;
-            info.fb.green.size = fb->green_mask_size;
-
-            info.fb.blue.shift = fb->blue_mask_shift;
-            info.fb.blue.size = fb->blue_mask_size;
-        }
-    }
-
-#if CONFIG_ACPI
-    if (rsdp_request.response) {
-        info.rsdp = (phys_t)rsdp_request.response->address;
-        if (info.rsdp >= info.hhdm_base)
-            info.rsdp -= info.hhdm_base;
-    }
-#endif
-
-    kernel_main(&info);
-}

boot/limine/loader.ld

@@ -0,0 +1,46 @@
+ENTRY(loader_main)
+
+PHDRS
+{
+    text    PT_LOAD FLAGS(5);
+    rodata  PT_LOAD FLAGS(4);
+    data    PT_LOAD FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0xffffffffc0000000;
+
+    .text : {
+        *(.text .text.*)
+    } :text
+
+    . = ALIGN(0x1000);
+    .rodata : {
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    . = ALIGN(0x1000);
+    .data : {
+        KEEP(*(.limine_requests_start))
+        KEEP(*(.limine_requests))
+        KEEP(*(.limine_requests_end))
+        *(.data .data.*)
+    } :data
+
+    .bss : {
+        *(.bss .bss.*)
+        *(COMMON)
+    } :data
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/limine/main.c

@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <string.h>
+#include "limine.h"
+#include "loader.h"
+
+[[__used, __section(".limine_requests_start")]]
+static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
+[[__used, __section(".limine_requests")]]
+static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
+[[__used, __section(".limine_requests_end")]]
+static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
+
+#define LIMINE_REQUEST(request, tag, rev) \
+    [[__used, __section(".limine_requests")]] \
+    static volatile struct limine_##request request = { \
+        .id = tag, \
+        .revision = rev, \
+        .response = nullptr, \
+    }
+
+LIMINE_REQUEST(bootloader_info_request, LIMINE_BOOTLOADER_INFO_REQUEST_ID, 0);
+LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
+LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
+LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
+LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
+LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
+LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
+
+static uint64_t hhdm_offset;
+static const struct limine_memmap_entry* pool;
+static uint64_t pool_static_end;
+static uint64_t pool_reclaimable_start;
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)(phys + hhdm_offset);
+}
+
+static uint64_t to_phys(const void* ptr) {
+    const uint64_t addr = (uint64_t)ptr;
+    return addr >= hhdm_offset ? addr - hhdm_offset : addr;
+}
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    size = ALIGN_UP(size, PAGE_SIZE);
+    if (pool_reclaimable_start - pool_static_end < size)
+        return false;
+
+    if (usage == PHYS_STATIC) {
+        *out = pool_static_end;
+        pool_static_end += size;
+    } else {
+        pool_reclaimable_start -= size;
+        *out = pool_reclaimable_start;
+    }
+    memset(loader_ptr(*out), 0, size);
+    return true;
+}
+
+static enum phys_mem_usage convert_mem_type(uint64_t type) {
+    switch (type) {
+    case LIMINE_MEMMAP_USABLE:
+        return PHYS_USABLE;
+    case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
+    case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+static bool init_pool(const struct limine_memmap_response* memmap) {
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry->type == LIMINE_MEMMAP_USABLE && (!pool || entry->length > pool->length))
+            pool = entry;
+    }
+    if (!pool)
+        return false;
+
+    pool_static_end = pool->base;
+    pool_reclaimable_start = pool->base + pool->length;
+    return true;
+}
+
+static uint64_t mem_map_top(const struct limine_memmap_response* memmap) {
+    uint64_t top = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (convert_mem_type(entry->type) != PHYS_RESERVED)
+            top = MAX(top, entry->base + entry->length);
+    }
+    return top;
+}
+
+static size_t convert_mem_map(const struct limine_memmap_response* memmap, struct phys_mem* out) {
+    size_t n = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry != pool) {
+            out[n++] = (struct phys_mem){entry->base, entry->length, convert_mem_type(entry->type)};
+            continue;
+        }
+
+        const uint64_t end = entry->base + entry->length;
+        out[n++] = (struct phys_mem){entry->base, pool_static_end - entry->base, PHYS_STATIC};
+        out[n++] = (struct phys_mem){pool_static_end, pool_reclaimable_start - pool_static_end, PHYS_USABLE};
+        out[n++] = (struct phys_mem){pool_reclaimable_start, end - pool_reclaimable_start, PHYS_RECLAIMABLE};
+    }
+    return n;
+}
+
+static void find_framebuffer(struct boot_framebuffer* out) {
+    const struct limine_framebuffer_response* res = framebuffer_request.response;
+    if (!res || !res->framebuffer_count)
+        return;
+
+    const struct limine_framebuffer* fb = res->framebuffers[0];
+    out->base = to_phys(fb->address);
+    out->width = fb->width;
+    out->height = fb->height;
+    out->pitch = fb->pitch;
+    out->bytes_per_pixel = fb->bpp / 8;
+    out->red = (struct boot_color){fb->red_mask_shift, fb->red_mask_size};
+    out->green = (struct boot_color){fb->green_mask_shift, fb->green_mask_size};
+    out->blue = (struct boot_color){fb->blue_mask_shift, fb->blue_mask_size};
+}
+
+static bool load() {
+    const struct limine_memmap_response* memmap = memmap_request.response;
+    const struct limine_executable_address_response* exec = executable_address_request.response;
+    if (!LIMINE_BASE_REVISION_SUPPORTED(limine_revision) || !memmap || !hhdm_request.response || !exec)
+        return false;
+    hhdm_offset = hhdm_request.response->offset;
+
+    if (!arch_check_cpu() || !init_pool(memmap))
+        return false;
+
+    const struct limine_bootloader_info_response* info = bootloader_info_request.response;
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, info ? info->name : "Limine"))
+        return false;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_RECLAIMABLE, (memmap->entry_count + 2) * sizeof(struct phys_mem), &mem))
+        return false;
+
+    const struct limine_executable_file_response* file = executable_file_request.response;
+    if (file && file->executable_file->string) {
+        const char* cmdline = file->executable_file->string;
+        memcpy(kernel.info->cmdline, cmdline, MIN(strlen(cmdline), sizeof(kernel.info->cmdline) - 1));
+    }
+
+    const struct limine_module_response* modules = module_request.response;
+    for (size_t i = 0; modules && i < modules->module_count; i++) {
+        const struct limine_file* module = modules->modules[i];
+        if (!loader_add_file(&kernel, module->path, to_phys(module->address), module->size))
+            return false;
+    }
+
+    find_framebuffer(&kernel.info->fb);
+    if (rsdp_request.response)
+        kernel.info->rsdp = to_phys(rsdp_request.response->address);
+    if (dtb_request.response && dtb_request.response->dtb_ptr)
+        kernel.info->dtb = to_phys(dtb_request.response->dtb_ptr);
+
+    const uint64_t trampoline = (uint64_t)arch_enter_kernel;
+    uint64_t root;
+    if (!loader_page_tables(
+            &kernel,
+            mem_map_top(memmap),
+            exec->physical_base + (trampoline - exec->virtual_base),
+            &root
+        ))
+        return false;
+
+    struct phys_mem* mem_map = loader_ptr(mem);
+    loader_set_mem_map(&kernel, mem_map, convert_mem_map(memmap, mem_map));
+    arch_enter_kernel(root, kernel.entry);
+}
+
+[[noreturn]]
+void loader_main() {
+    load();
+    for (;;)
+        asm volatile("");
+}

boot/limine/meson.build

@@ -1 +1,25 @@
-kernel_sources += files('entry.c')
+limine_lds = meson.current_source_dir() / 'loader.ld'
+
+limine_c_args = [loader_c_args, '-fno-pic']
+if target_machine.cpu_family() == 'x86_64'
+    limine_c_args += '-mcmodel=kernel'
+endif
+
+executable(
+    'zinnia-limine',
+    sources: [loader_sources, files('main.c'), loader_kernel],
+    include_directories: [common_includes, loader_includes],
+    c_args: limine_c_args,
+    link_args: [
+        '-nostdlib',
+        '-static',
+        '-Wl,-T,' + limine_lds,
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: limine_lds,
+    pie: false,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/loader/elf.c

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <string.h>
+#include "loader.h"
+
+static bool in_image(const struct kernel_image* image, uint64_t vaddr, uint64_t len) {
+    return vaddr >= image->virt && len <= image->size && vaddr - image->virt <= image->size - len;
+}
+
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr) {
+    return loader_ptr(image->phys + (vaddr - image->virt));
+}
+
+static bool apply_relocations(const struct kernel_image* image, uint64_t dynamic_vaddr) {
+    if (!in_image(image, dynamic_vaddr, sizeof(elf64_dyn_t)))
+        return false;
+
+    uint64_t rela = 0;
+    uint64_t rela_size = 0;
+    uint64_t rela_ent = sizeof(elf64_rela_t);
+    for (const elf64_dyn_t* dyn = kernel_ptr(image, dynamic_vaddr); dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    if (!rela)
+        return true;
+    if (rela_ent < sizeof(elf64_rela_t) || !in_image(image, rela, rela_size))
+        return false;
+
+    for (uint64_t off = 0; off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = kernel_ptr(image, rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return false;
+        if (!in_image(image, r->r_offset, sizeof(uint64_t)))
+            return false;
+        *(uint64_t*)kernel_ptr(image, r->r_offset) = (uint64_t)r->r_addend;
+    }
+
+    return true;
+}
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out) {
+    elf64_ehdr_t ehdr;
+    if (size < sizeof(ehdr))
+        return false;
+    memcpy(&ehdr, elf, sizeof(ehdr));
+
+    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) || ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS ||
+        ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
+        return false;
+    if (ehdr.e_machine != ELF_ARCH_MACHINE)
+        return false;
+    if (ehdr.e_phentsize != sizeof(elf64_phdr_t) || ehdr.e_phoff > size ||
+        (size - ehdr.e_phoff) / sizeof(elf64_phdr_t) < ehdr.e_phnum)
+        return false;
+
+    const elf64_phdr_t* phdrs = (const elf64_phdr_t*)(elf + ehdr.e_phoff);
+
+    uint64_t lo = UINT64_MAX;
+    uint64_t hi = 0;
+    uint64_t dynamic_vaddr = 0;
+    uint64_t info_vaddr = 0;
+    uint64_t info_size = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type == PT_DYNAMIC)
+            dynamic_vaddr = ph->p_vaddr;
+        if (ph->p_type == PT_BOOT_INFO) {
+            info_vaddr = ph->p_vaddr;
+            info_size = ph->p_memsz;
+        }
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        if (ph->p_filesz > ph->p_memsz || ph->p_offset > size || size - ph->p_offset < ph->p_filesz)
+            return false;
+        if (ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE) < lo)
+            lo = ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE);
+        if (ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE) > hi)
+            hi = ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE);
+    }
+    if (lo >= hi)
+        return false;
+
+    out->virt = lo;
+    out->size = hi - lo;
+    out->entry = ehdr.e_entry;
+    if (!in_image(out, out->entry, 1))
+        return false;
+    if (info_size != sizeof(struct boot_info) || !in_image(out, info_vaddr, info_size))
+        return false;
+
+    if (!loader_alloc(PHYS_STATIC, out->size, &out->phys))
+        return false;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        memcpy(kernel_ptr(out, ph->p_vaddr), elf + ph->p_offset, ph->p_filesz);
+    }
+    out->info = kernel_ptr(out, info_vaddr);
+
+    if (dynamic_vaddr)
+        return apply_relocations(out, dynamic_vaddr);
+    return true;
+}

boot/loader/lib.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+size_t strlen(const char* str) {
+    if (!str)
+        return 0;
+
+    size_t len = 0;
+    while (str[len])
+        len++;
+    return len;
+}
+
+int memcmp(const void* s1, const void* s2, size_t size) {
+    const uint8_t* a = s1;
+    const uint8_t* b = s2;
+    for (size_t i = 0; i < size; i++) {
+        if (a[i] != b[i])
+            return a[i] < b[i] ? -1 : 1;
+    }
+    return 0;
+}
+
+void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t* s = src;
+    for (size_t i = 0; i < n; i++)
+        d[i] = s[i];
+    return dest;
+}
+
+void* memmove(void* dstptr, const void* srcptr, size_t size) {
+    uint8_t* d = dstptr;
+    const uint8_t* s = srcptr;
+    if (d < s) {
+        for (size_t i = 0; i < size; i++)
+            d[i] = s[i];
+    } else {
+        for (size_t i = size; i > 0; i--)
+            d[i - 1] = s[i - 1];
+    }
+    return dstptr;
+}
+
+void* memset(void* dest, int value, size_t n) {
+    uint8_t* d = dest;
+    for (size_t i = 0; i < n; i++)
+        d[i] = (uint8_t)value;
+    return dest;
+}

boot/loader/loader.c

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: ISC
+
+#include "loader.h"
+#include <string.h>
+#include "loader_arch.h"
+
+extern const uint8_t __kernel_start[];
+extern const uint8_t __kernel_end[];
+
+bool loader_load_kernel(struct kernel_image* out, const char* name) {
+    if (!elf_load(__kernel_start, __kernel_end - __kernel_start, out))
+        return false;
+
+    struct boot_info* info = out->info;
+    memcpy(info->loader, name, MIN(strlen(name), sizeof(info->loader) - 1));
+    info->hhdm_base = ARCH_HHDM_BASE;
+    info->phys_base = out->phys;
+    info->virt_base = out->virt;
+    return true;
+}
+
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length) {
+    struct boot_info* info = kernel->info;
+    if (info->num_files == ARRAY_SIZE(info->files))
+        return false;
+
+    struct boot_file* file = &info->files[info->num_files++];
+    file->data = base;
+    file->length = length;
+    const size_t path_len = MIN(strlen(path), sizeof(file->path) - 1);
+    memcpy(file->path, path, path_len);
+    file->path[path_len] = 0;
+    return true;
+}
+
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len) {
+    for (size_t i = 1; i < len; i++) {
+        const struct phys_mem entry = map[i];
+        size_t j = i;
+        while (j > 0 && map[j - 1].address > entry.address) {
+            map[j] = map[j - 1];
+            j--;
+        }
+        map[j] = entry;
+    }
+
+    size_t merged = 0;
+    for (size_t i = 0; i < len; i++) {
+        if (!map[i].length)
+            continue;
+        struct phys_mem* prev = merged ? &map[merged - 1] : nullptr;
+        if (prev && prev->usage == map[i].usage && prev->address + prev->length == map[i].address)
+            prev->length += map[i].length;
+        else
+            map[merged++] = map[i];
+    }
+
+    struct boot_info* info = kernel->info;
+    info->num_mem_maps = MIN(merged, ARRAY_SIZE(info->mem_map));
+    memcpy(info->mem_map, map, info->num_mem_maps * sizeof(*map));
+}
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+) {
+    uint64_t root;
+    if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &root))
+        return false;
+
+    const uint64_t trampoline_virt = ALIGN_DOWN((uint64_t)arch_enter_kernel, PAGE_SIZE);
+    if (!arch_paging_map(root, ARCH_HHDM_BASE, 0, ALIGN_UP(mem_top, ARCH_HUGE_PAGE_SIZE), ARCH_HUGE_PAGE_SIZE) ||
+        !arch_paging_map(root, kernel->virt, kernel->phys, kernel->size, PAGE_SIZE) ||
+        !arch_paging_map(root, trampoline_virt, ALIGN_DOWN(trampoline_phys, PAGE_SIZE), 2 * PAGE_SIZE, PAGE_SIZE))
+        return false;
+
+    *out_root = root;
+    return true;
+}

boot/loader/loader.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/boot.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint64_t PAGE_SIZE = 0x1000;
+
+struct kernel_image {
+    uint64_t phys;
+    uint64_t virt;
+    uint64_t size;
+    uint64_t entry;
+    struct boot_info* info;
+};
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out);
+void* loader_ptr(uint64_t phys);
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out);
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr);
+
+bool loader_load_kernel(struct kernel_image* out, const char* name);
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length);
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len);
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+);
+
+bool arch_check_cpu();
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size);
+
+[[noreturn]]
+void arch_enter_kernel(uint64_t root, uint64_t entry);

boot/loader/x86_64/enter.S

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl arch_enter_kernel
+    .type arch_enter_kernel, @function
+    .balign 16
+arch_enter_kernel:
+    cli
+    mov cr3, rdi
+    jmp rsi
+    .size arch_enter_kernel, . - arch_enter_kernel

boot/loader/x86_64/loader_arch.h

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+constexpr uint64_t ARCH_HHDM_BASE = 0xffff'8000'0000'0000;
+constexpr uint64_t ARCH_HUGE_PAGE_SIZE = 0x20'0000;

boot/loader/x86_64/meson.build

@@ -0,0 +1,11 @@
+loader_sources += files(
+    'enter.S',
+    'paging.c',
+)
+
+loader_c_args += [
+    '-mno-red-zone',
+    '-masm=intel',
+]
+
+loader_objcopy_format = 'elf64-x86-64'

boot/loader/x86_64/paging.c

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/pmap.h>
+#include <x86_64/defs.h>
+#include "loader.h"
+#include "loader_arch.h"
+
+static bool next_table(uint64_t* table, size_t index, uint64_t** out) {
+    if (!(table[index] & ARCH_FLAG_PRESENT)) {
+        uint64_t page;
+        if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &page))
+            return false;
+        table[index] = page | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    }
+    *out = loader_ptr(table[index] & ARCH_PTE_ADDR_MASK);
+    return true;
+}
+
+bool arch_check_cpu() {
+    uint64_t cr4;
+    asm volatile("mov %0, cr4" : "=r"(cr4));
+    if (cr4 & CR4_LA57)
+        return false;
+    return true;
+}
+
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size) {
+    const int leaf_level = page_size == ARCH_HUGE_PAGE_SIZE ? 1 : 0;
+
+    for (uint64_t off = 0; off < size; off += page_size) {
+        const uint64_t va = virt + off;
+        uint64_t* table = loader_ptr(root);
+        for (int level = 3; level > leaf_level; level--) {
+            if (!next_table(table, (va >> (12 + 9 * level)) & 0x1ff, &table))
+                return false;
+        }
+
+        uint64_t entry = (phys + off) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        if (leaf_level == 1)
+            entry |= ARCH_FLAG_SIZE;
+        table[(va >> (12 + 9 * leaf_level)) & 0x1ff] = entry;
+    }
+
+    return true;
+}

boot/meson.build

@@ -1,6 +1,44 @@
-kernel_sources += files(
-    'cmdline.c',
-    'main.c',
+objcopy = find_program('objcopy')
+zinnia_bootdir = zinnia_datadir / 'boot'
+
+loader_sources = files(
+    'loader/elf.c',
+    'loader/lib.c',
+    'loader/loader.c',
 )
 
-subdir('limine')
+loader_c_args = [
+    '-ffreestanding',
+    '-nostdinc',
+    '-fno-stack-protector',
+    '-fno-omit-frame-pointer',
+    '-mgeneral-regs-only',
+]
+
+loader_includes = include_directories('loader', 'loader' / target_machine.cpu_family())
+
+loader_objcopy_format = 'unknown'
+
+subdir('loader' / target_machine.cpu_family())
+
+loader_kernel = custom_target(
+    'loader-kernel.o',
+    input: kernel,
+    output: 'loader-kernel.o',
+    command: [
+        objcopy,
+        '-I', 'binary',
+        '-O', loader_objcopy_format,
+        '--rename-section', '.data=.kernel,alloc,load,readonly,data,contents',
+        '@INPUT@',
+        '@OUTPUT@',
+    ],
+)
+
+if kconfig.get('CONFIG_BOOT_EFISTUB', 'n') == 'y'
+    subdir('efi')
+endif
+
+if kconfig.get('CONFIG_BOOT_LIMINE', 'n') == 'y'
+    subdir('limine')
+endif

core/dpc.c

@@ -1,3 +0,0 @@
-#include <zinnia/dpc.h>
-
-// TODO

core/irq.c

@@ -1,69 +0,0 @@
-#include <zinnia/assert.h>
-#include <zinnia/irq.h>
-#include <zinnia/list.h>
-#include <zinnia/percpu.h>
-#include <zinnia/print.h>
-#include <arch/irq.h>
-
-[[__percpu]]
-struct irq_percpu irq_data;
-
-bool irq_set_state(bool state) {
-    return arch_irq_set_state(state);
-}
-
-bool irq_get_state() {
-    return arch_irq_get_state();
-}
-
-void irq_wait() {
-    arch_irq_wait();
-}
-
-void irq_lock() {
-    bool prev = arch_irq_set_state(false);
-    struct irq_percpu* state = percpu_ref(irq_data);
-    if (state->depth++ == 0)
-        state->saved_state = prev;
-}
-
-void irq_unlock() {
-    struct irq_percpu* state = percpu_ref(irq_data);
-    if (--state->depth == 0 && state->saved_state)
-        arch_irq_set_state(true);
-}
-
-void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
-    irq_lock();
-    line->mode = line->set_config(line, trigger, polarity);
-    irq_unlock();
-}
-
-void irq_line_raise(struct irq_line* line) {
-    assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
-
-    irq_lock();
-    line->is_busy = true;
-
-    bool claimed = false;
-    struct irq_handler* handler;
-    list_foreach(handler, &line->handlers, next) {
-        if (handler->raise(handler) == IRQ_STATUS_HANDLED)
-            claimed = true;
-    }
-
-    if (!claimed)
-        log("irq: Spurious interrupt\n");
-
-    if ((line->mode & IRQ_MODE_EOI) && line->eoi)
-        line->eoi(line);
-
-    line->is_busy = false;
-    irq_unlock();
-}
-
-void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
-    irq_lock();
-    list_insert_head(&line->handlers, handler, next);
-    irq_unlock();
-}

core/lib/meson.build

@@ -1,11 +0,0 @@
-# Freestanding libc functions that are needed in the kernel
-
-kernel_sources += files(
-    'guid.c',
-    'iovec.c',
-    'radix.c',
-    'ringbuf.c',
-    'runtime.c',
-    'stdlib.c',
-    'string.c',
-)

core/mem/vma.c

@@ -1,158 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/errno.h>
-#include <zinnia/mman.h>
-#include <zinnia/pmap.h>
-#include <zinnia/utils.h>
-#include <zinnia/vma.h>
-#include <stdckdint.h>
-
-// Try to create a new, empty virtual address space.
-errno_t vma_new(struct vma** out) {
-    debug_assert(out != nullptr, "Missing out");
-
-    errno_t res;
-    if (!out)
-        return EINVAL;
-
-    struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
-    if (!vmspace)
-        return ENOMEM;
-
-    res = pmap_new_user(&vmspace->pmap, 0);
-    if (res)
-        goto err1;
-
-    tailq_init(&vmspace->mappings);
-
-    *out = vmspace;
-    return 0;
-
-err1:
-    mem_free(vmspace);
-err0:
-    return res;
-}
-
-// Delete a virtual address space.
-void vma_free(struct vma* v) {
-    struct vma_mapping* mapping;
-    while ((mapping = tailq_first(&v->mappings)) != nullptr) {
-        tailq_remove(&v->mappings, mapping, link);
-        vmo_unref(mapping->object);
-        mem_free(mapping);
-    }
-}
-
-errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
-    panic("TODO");
-}
-
-errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
-    panic("TODO");
-}
-
-static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
-    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
-        if (page >= m->start_page && page < m->end_page)
-            return m;
-    }
-    return nullptr;
-}
-
-static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) {
-    if (info->caused_by_write)
-        return prot & PROT_WRITE;
-    if (info->caused_by_fetch)
-        return prot & PROT_EXEC;
-    return prot & (PROT_READ | PROT_WRITE);
-}
-
-static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
-    enum pte_flags pf = 0;
-    if (prot & PROT_READ)
-        pf |= PTE_READ;
-    if (prot & PROT_WRITE)
-        pf |= PTE_WRITE;
-    if (prot & PROT_EXEC)
-        pf |= PTE_EXEC;
-    if (v->pmap.is_user)
-        pf |= PTE_USER;
-    return pf;
-}
-
-bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
-    const size_t page_size = arch_mem_page_size();
-    const size_t faulty_page = info->addr / page_size;
-
-    spin_lock(&v->lock);
-    struct vma_mapping* mapping = find_mapping(v, faulty_page);
-    if (!mapping || !can_access(mapping->prot, info)) {
-        spin_unlock(&v->lock);
-        return false;
-    }
-
-    struct vmo* object = mapping->object;
-    const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
-    const enum prot_flags prot = mapping->prot;
-    vmo_ref(object);
-    spin_unlock(&v->lock);
-
-    struct page* page;
-    if (object->get_page(object, object_page, &page)) {
-        vmo_unref(object);
-        return false;
-    }
-
-    spin_lock(&v->lock);
-    mapping = find_mapping(v, faulty_page);
-    const bool still_valid = mapping && mapping->object == object &&
-        (faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info);
-
-    if (still_valid) {
-        errno_t status =
-            pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
-        assert(status == 0, "Failed to map a demand-loaded page");
-    }
-    spin_unlock(&v->lock);
-
-    vmo_unref(object);
-    return true;
-}
-
-errno_t vma_map(
-    struct vma* vma,
-    struct vmo* vmo,
-    uintptr_t addr,
-    size_t len,
-    enum prot_flags flags,
-    uintptr_t vmo_offset
-) {
-    if (!vma || !vmo)
-        return EINVAL;
-
-    uintptr_t end;
-    if (ckd_add(&end, addr, len))
-        return ENOMEM;
-
-    const size_t page_size = arch_mem_page_size();
-    if (addr % page_size != vmo_offset % page_size)
-        return EINVAL;
-
-    struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
-    if (!mapping)
-        return ENOMEM;
-
-    vmo_ref(vmo);
-    mapping->start_page = addr / page_size;
-    mapping->end_page = ALIGN_UP(end, page_size) / page_size;
-    mapping->offset_page = vmo_offset / page_size;
-    mapping->object = vmo;
-    mapping->prot = flags;
-
-    spin_lock(&vma->lock);
-    tailq_insert_head(&vma->mappings, mapping, link);
-    spin_unlock(&vma->lock);
-
-    return 0;
-}

core/mem/vmo.c

@@ -1,263 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/errno.h>
-#include <zinnia/list.h>
-#include <zinnia/utils.h>
-#include <zinnia/vmo.h>
-
-void vmo_ref(struct vmo* obj) {
-    refcount_inc(&obj->rc);
-}
-
-void vmo_unref(struct vmo* obj) {
-    if (refcount_dec(&obj->rc))
-        obj->destroy(obj);
-}
-
-errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
-
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
-
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_read)
-        *actual_read = progress;
-    return status;
-}
-
-errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
-
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
-
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_written)
-        *actual_written = progress;
-    return status;
-}
-
-errno_t vmo_copy(
-    struct vmo* target,
-    uintptr_t target_offset,
-    struct vmo* src,
-    uintptr_t src_offset,
-    size_t len,
-    size_t* actual_copied
-) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t target_misalign = (progress + target_offset) % page_size;
-        const size_t src_misalign = (progress + src_offset) % page_size;
-        const size_t target_page_index = (progress + target_offset) / page_size;
-        const size_t src_page_index = (progress + src_offset) / page_size;
-
-        const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
-
-        struct page* tpage;
-        status = target->get_page(target, target_page_index, &tpage);
-        if (status != 0)
-            break;
-
-        struct page* spage;
-        status = src->get_page(src, src_page_index, &spage);
-        if (status != 0)
-            break;
-
-        uintptr_t taddr = vm_page_addr(tpage);
-        uintptr_t saddr = vm_page_addr(spage);
-
-        memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_copied)
-        *actual_copied = progress;
-    return status;
-}
-
-static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    // Try to find the requested page in the cache.
-    struct page_list* iter;
-    spin_lock(&paged->lock);
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
-            continue;
-
-        // We found a cached page!
-        *out = iter->value;
-        spin_unlock(&paged->lock);
-        return 0;
-    }
-    spin_unlock(&paged->lock);
-
-    // Get page from out pager if we don't have one already.
-    struct page* new_page;
-    errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
-    if (status)
-        return status;
-
-    struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
-    if (!entry) {
-        paged->source.put_page(&paged->source, offset_idx, new_page);
-        return ENOMEM;
-    }
-
-    entry->offset = offset_idx;
-    entry->value = new_page;
-
-    spin_lock(&paged->lock);
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
-            continue;
-
-        *out = iter->value;
-        spin_unlock(&paged->lock);
-        mem_free(entry);
-        paged->source.put_page(&paged->source, offset_idx, new_page);
-        return 0;
-    }
-
-    // Add page to the object cache.
-    list_insert_head(&paged->cache, entry, next);
-    spin_unlock(&paged->lock);
-
-    *out = new_page;
-    return 0;
-}
-
-static void paged_destroy(struct vmo* vmo) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    struct page_list* iter = list_first(&paged->cache);
-    while (iter) {
-        struct page_list* next = list_next(iter, next);
-        paged->source.put_page(&paged->source, iter->offset, iter->value);
-        mem_free(iter);
-        iter = next;
-    }
-
-    mem_free(paged);
-}
-
-errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
-    struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
-    if (!result)
-        return ENOMEM;
-
-    refcount_init(&result->object.rc);
-    result->source = pager;
-    result->object.get_page = paged_get_page;
-    result->object.destroy = paged_destroy;
-
-    *out = result;
-    return 0;
-}
-
-static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
-    phys_t addr;
-    errno_t status = mem_phys_alloc(1, 0, &addr);
-    if (status != 0)
-        return status;
-    size_t idx = addr / arch_mem_page_size();
-    *out = &vm_pfndb[idx];
-    return 0;
-}
-
-static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
-    return mem_phys_free(vm_page_addr(page), 1);
-}
-
-static const struct pager_ops phys_pager = {
-    .get_page = phys_get_page,
-    .put_page = phys_put_page,
-};
-
-errno_t vmo_new_phys(struct paged_vmo** out) {
-    return vmo_new_paged(phys_pager, out);
-}
-
-void vmo_truncate_paged(struct paged_vmo* paged, size_t length) {
-    const size_t page_size = arch_mem_page_size();
-    const size_t keep_pages = ROUND_UP(length, page_size);
-
-    spin_lock(&paged->lock);
-    struct page_list** pp = &list_first(&paged->cache);
-    while (*pp) {
-        struct page_list* entry = *pp;
-        if (entry->offset >= keep_pages) {
-            *pp = list_next(entry, next);
-            paged->source.put_page(&paged->source, entry->offset, entry->value);
-            mem_free(entry);
-        } else {
-            pp = &list_next(entry, next);
-        }
-    }
-    spin_unlock(&paged->lock);
-}
-
-struct phys_range_vmo {
-    struct vmo object;
-    uintptr_t start;
-    size_t len;
-};
-
-static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
-    if (offset_idx >= p->len / arch_mem_page_size())
-        return EINVAL;
-    *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
-    return 0;
-}
-
-static void phys_range_destroy(struct vmo* vmo) {
-    mem_free(containerof(vmo, struct phys_range_vmo, object));
-}
-
-errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
-    struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
-    if (!vmo)
-        return ENOMEM;
-
-    refcount_init(&vmo->object.rc);
-    vmo->object.get_page = phys_range_get_page;
-    vmo->object.destroy = phys_range_destroy;
-    vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
-    vmo->len = ALIGN_UP(length, arch_mem_page_size());
-
-    *out = &vmo->object;
-    return 0;
-}

core/meson.build

@@ -1,17 +0,0 @@
-kernel_sources += files(
-    'clock.c',
-    'console.c',
-    'initgraph.c',
-    'irq.c',
-    'ksyms.c',
-    'module.c',
-    'panic.c',
-    'percpu.c',
-    'print.c',
-    'random.c',
-    'spin.c',
-)
-
-subdir('lib')
-subdir('mem')
-subdir('sched')

core/sched/core.c

@@ -1 +0,0 @@
-#include <zinnia/sched.h>

core/sched/sched_dumb.c

@@ -1,194 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/atomic.h>
-#include <zinnia/errno.h>
-#include <zinnia/irq.h>
-#include <zinnia/list.h>
-#include <zinnia/percpu.h>
-#include <zinnia/pmap.h>
-#include <zinnia/print.h>
-#include <zinnia/sched.h>
-#include <zinnia/spin.h>
-#include <zinnia/stack.h>
-#include <zinnia/tailq.h>
-#include <zinnia/vma.h>
-#include <arch/sched.h>
-#include <stdint.h>
-
-static void idle_func(uintptr_t) {
-    while (1) {
-        irq_set_state(true);
-        irq_wait();
-    }
-}
-
-void sched_to_user(uintptr_t info_ptr) {
-    struct task_start_info* info = (struct task_start_info*)info_ptr;
-    uintptr_t ip = info->ip;
-    uintptr_t sp = info->sp;
-    uintptr_t arg = info->arg;
-    mem_free(info);
-    arch_sched_jump_to_user(ip, sp, arg);
-}
-
-void sched_to_user_context(uintptr_t ctx) {
-    arch_sched_jump_to_context((struct arch_context*)ctx);
-}
-
-void sched_init(struct scheduler* sched) {
-    struct task* idle_task;
-    assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!");
-    sched->idle_task = idle_task;
-
-    struct task* boot_task = mem_alloc(sizeof(struct task), 0);
-    assert(boot_task != nullptr, "Unable to create boot task!");
-    strncpy(boot_task->name, "boot", sizeof(boot_task->name));
-    boot_task->space = &kernel_vma;
-    sched->current = boot_task;
-
-    tailq_init(&sched->run_queue);
-
-    log("sched: Scheduler initialized\n");
-}
-
-void sched_add_task(struct scheduler* sched, struct task* task) {
-    assert(task != nullptr, "Tried to add NULL to the run queue");
-    task->state = TASK_STATE_READY;
-    tailq_insert_tail(&sched->run_queue, task, next);
-}
-
-static struct task* next_task(struct scheduler* sched) {
-    struct task* t = tailq_first(&sched->run_queue);
-
-    if (t == nullptr)
-        return sched->idle_task;
-
-    tailq_remove(&sched->run_queue, t, next);
-    t->state = TASK_STATE_RUNNING;
-
-    return t;
-}
-
-static void finish_switch(struct task* prev) {
-    if (prev->state == TASK_STATE_DEAD)
-        task_destroy(prev);
-}
-
-static void do_reschedule(struct scheduler* sched) {
-    struct task* from = sched->current;
-    struct task* to = next_task(sched);
-
-    // We'd end up on the same task, skip switching altogether.
-    if (from == to) {
-        irq_unlock();
-        return;
-    }
-
-    sched->current = to;
-    pmap_set(&to->space->pmap);
-
-    struct percpu* cpu = percpu_get();
-    from->user_stack = cpu->user_stack;
-    cpu->kernel_stack = kernel_stack_top(&to->kernel_stack);
-    cpu->user_stack = to->user_stack;
-
-    struct irq_percpu irq_state = *percpu_ref(irq_data);
-    struct task* prev = arch_sched_switch(from, to);
-    *percpu_ref(irq_data) = irq_state;
-    finish_switch(prev);
-    irq_unlock();
-}
-
-void sched_reschedule(struct scheduler* sched) {
-    irq_lock();
-    if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD)
-        sched_add_task(sched, sched->current);
-    do_reschedule(sched);
-}
-
-void sched_yield(struct scheduler* sched) {
-    irq_lock();
-    do_reschedule(sched);
-}
-
-void sched_block(struct scheduler* sched) {
-    irq_lock();
-    if (atomic_exchange(&sched->current->wake_pending, false, ATOMIC_ACQ_REL)) {
-        irq_unlock();
-        return;
-    }
-    sched->current->state = TASK_STATE_BLOCKED;
-    do_reschedule(sched);
-}
-
-void sched_wake(struct task* task) {
-    irq_lock();
-    if (task->state == TASK_STATE_BLOCKED)
-        sched_add_task(&percpu_get()->sched, task);
-    else
-        atomic_store(&task->wake_pending, true, ATOMIC_RELEASE);
-    irq_unlock();
-}
-
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
-    if (!out)
-        return EINVAL;
-
-    struct task* new_task = mem_alloc(sizeof(struct task), 0);
-    if (new_task == nullptr)
-        return ENOMEM;
-
-    static atomic_size_t last_tid = 0;
-    new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE);
-    new_task->space = space;
-    strncpy(new_task->name, name, sizeof(new_task->name));
-
-    // Allocate a kernel stack.
-    errno_t s = stack_init(&new_task->kernel_stack, KERNEL_STACK_SIZE);
-    if (s != 0) {
-        mem_free(new_task);
-        return s;
-    }
-    new_task->user_stack = 0;
-
-    // Initialize the arch-dependent parts.
-    s = arch_task_init(&new_task->context, entry, arg0, kernel_stack_top(&new_task->kernel_stack), true);
-    if (s != 0) {
-        stack_destroy(&new_task->kernel_stack);
-        mem_free(new_task);
-        return s;
-    }
-
-    *out = new_task;
-    return 0;
-}
-
-void task_destroy(struct task* task) {
-    arch_task_fini(&task->context);
-    stack_destroy(&task->kernel_stack);
-    mem_free(task);
-}
-
-void task_entry(task_fn_t entry, uintptr_t arg0, struct task* prev) {
-    finish_switch(prev);
-
-    *percpu_ref(irq_data) = (struct irq_percpu){
-        .depth = 1,
-        .saved_state = true,
-    };
-    irq_unlock();
-
-    entry(arg0);
-
-    struct scheduler* sched = &percpu_get()->sched;
-    sched->current->state = TASK_STATE_DEAD;
-    sched_yield(sched);
-    panic("unreachable");
-}
-
-[[__init]]
-static void sched_stage_init(struct boot_info*) {
-    sched_init(&percpu_get()->sched);
-}
-INIT_TASK(sched_stage, "core.sched", sched_stage_init);
-INIT_DEPENDS(sched_stage, percpu_stage);

core/wait.c

@@ -1,17 +0,0 @@
-#include <zinnia/waitq.h>
-
-void waitq_init(struct waitq* wq) {
-    tailq_init(&wq->waiters);
-}
-
-void waitq_wait(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_one(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_all(struct waitq* wq) {
-    // TODO
-}

dev/acpi/include/uacpi_types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdarg.h>

dev/acpi/include/zinnia/acpi.h

@@ -1,13 +1,22 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/errno.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>
 #include <zinnia/types.h>
+#include <zinnia/workqueue.h>
 #include <stdint.h>
 
 INIT_EXTERN(acpi_stage);
+INIT_EXTERN(acpi_namespace_stage);
 
 extern phys_t acpi_rsdp_addr;
+extern struct workqueue* acpi_workqueue;
+
+errno_t acpi_reboot();
+errno_t acpi_power_off();
 
 struct acpi_match {
     const char* hid;

dev/acpi/init.c

@@ -1,16 +1,25 @@
+// SPDX-License-Identifier: ISC
+
 #define pr_fmt(fmt) "acpi: " fmt
 
 #include <zinnia/acpi.h>
 #include <zinnia/alloc.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/irq.h>
+#include <zinnia/pci.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
 #include <zinnia/types.h>
+#include <zinnia/workqueue.h>
+#include <uacpi/event.h>
+#include <uacpi/sleep.h>
 #include <uacpi/status.h>
 #include <uacpi/tables.h>
 #include <uacpi/uacpi.h>
 
 phys_t acpi_rsdp_addr = 0;
+struct workqueue* acpi_workqueue;
 
 static char table_buf[0x1000];
 
@@ -43,3 +52,66 @@ static bool acpi_stage_init(struct boot_info* info) {
 INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
 INIT_DEPENDS(acpi_stage, mem_stage);
 INIT_ENTAILS(acpi_stage, percpu_stage);
+
+static bool namespace_done;
+
+[[__init]]
+static void acpi_namespace_init(struct boot_info*) {
+    errno_t error = workqueue_create("acpi", SCHED_PRIO_WORK_HIGH, &acpi_workqueue);
+    if (error) {
+        log_err("Failed to create the ACPI workqueue (%i)\n", error);
+        return;
+    }
+
+    uacpi_status status = uacpi_initialize(0);
+    if (status != UACPI_STATUS_OK) {
+        log_err("Initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_load();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace loading failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_initialize();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_finalize_gpe_initialization();
+    if (status != UACPI_STATUS_OK)
+        log_warn("GPE initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+
+    namespace_done = true;
+}
+INIT_TASK(acpi_namespace_stage, "dev.acpi.namespace", acpi_namespace_init);
+INIT_DEPENDS(acpi_namespace_stage, acpi_stage, workqueue_stage);
+#if CONFIG_PCI
+INIT_DEPENDS(acpi_namespace_stage, pci_stage);
+#endif
+
+errno_t acpi_reboot() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_reboot();
+    log_err("Failed to reset the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}
+
+errno_t acpi_power_off() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_prepare_for_sleep_state(UACPI_SLEEP_STATE_S5);
+    if (status == UACPI_STATUS_OK) {
+        irq_lock();
+        status = uacpi_enter_sleep_state(UACPI_SLEEP_STATE_S5);
+        irq_unlock();
+    }
+    log_err("Failed to power off the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}

dev/acpi/meson.build

@@ -15,6 +15,5 @@ kernel_includes += include_directories('include')
 
 kernel_c_args += [
     '-DUACPI_FORMATTED_LOGGING',
-    '-DUACPI_BAREBONES_MODE', # TODO: Remove
     '-DUACPI_OVERRIDE_TYPES',
 ]

dev/acpi/uacpi_glue.c

@@ -1,13 +1,24 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/acpi.h>
 #include <zinnia/alloc.h>
+#include <zinnia/clock.h>
 #include <zinnia/console.h>
 #include <zinnia/init.h>
+#include <zinnia/irq.h>
 #include <zinnia/mmio.h>
+#include <zinnia/pci.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/semaphore.h>
+#include <zinnia/spin.h>
 #include <zinnia/utils.h>
 #include <zinnia/virt.h>
 #include <zinnia/vma.h>
+#include <zinnia/workqueue.h>
+#include <arch/irq.h>
 #include <arch/mem.h>
 #include <stdarg.h>
 #include <uacpi/kernel_api.h>
@@ -71,3 +82,250 @@ void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
 void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
     log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
 }
+
+uacpi_status uacpi_kernel_pci_device_open(uacpi_pci_address address, uacpi_handle* out_handle) {
+#if CONFIG_PCI
+    struct pci_address addr = {
+        .domain = address.segment,
+        .bus = address.bus,
+        .device = address.device,
+        .function = address.function,
+    };
+    struct pci_access* access = pci_access_find(&addr);
+    if (!access)
+        return UACPI_STATUS_NOT_FOUND;
+
+    struct pci_device* dev = mem_alloc(sizeof(*dev), 0);
+    if (!dev)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    dev->address = addr;
+    dev->access = access;
+    *out_handle = dev;
+    return UACPI_STATUS_OK;
+#else
+    return UACPI_STATUS_NOT_FOUND;
+#endif
+}
+
+void uacpi_kernel_pci_device_close(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+#if CONFIG_PCI
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        *value = pci_read##width(device, offset); \
+        return UACPI_STATUS_OK; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        pci_write##width(device, offset, value); \
+        return UACPI_STATUS_OK; \
+    }
+#else
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#endif
+
+PCI_READ(8)
+PCI_READ(16)
+PCI_READ(32)
+PCI_WRITE(8)
+PCI_WRITE(16)
+PCI_WRITE(32)
+
+void* uacpi_kernel_alloc(uacpi_size size) {
+    return mem_alloc(size, ALLOC_NOZERO);
+}
+
+void uacpi_kernel_free(void* mem) {
+    if (mem)
+        mem_free(mem);
+}
+
+uacpi_u64 uacpi_kernel_get_nanoseconds_since_boot() {
+    return clock_get_elapsed_ns();
+}
+
+void uacpi_kernel_stall(uacpi_u8 usec) {
+    clock_spin_ns(usec * 1000ull);
+}
+
+void uacpi_kernel_sleep(uacpi_u64 msec) {
+    sched_sleep_until(clock_get_elapsed_ns() + msec * 1'000'000);
+}
+
+uacpi_handle uacpi_kernel_create_semaphore(uacpi_u32 initial_units) {
+    sem_t* sem = mem_alloc(sizeof(*sem), 0);
+    if (sem)
+        sem_init(sem, initial_units);
+    return sem;
+}
+
+void uacpi_kernel_free_semaphore(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_status uacpi_kernel_wait_for_semaphore(uacpi_handle handle, uacpi_u16 timeout) {
+    sem_t* sem = handle;
+    if (timeout == 0)
+        return sem_trywait(sem) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+    if (timeout == 0xFFFF) {
+        sem_wait(sem);
+        return UACPI_STATUS_OK;
+    }
+
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout * 1'000'000ull;
+    return sem_timedwait(sem, &deadline) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+}
+
+void uacpi_kernel_signal_semaphore(uacpi_handle handle) {
+    sem_post(handle);
+}
+
+uacpi_thread_id uacpi_kernel_get_thread_id() {
+    return percpu_get()->sched.current;
+}
+
+uacpi_interrupt_state uacpi_kernel_disable_interrupts() {
+    irq_lock();
+    return 0;
+}
+
+void uacpi_kernel_restore_interrupts(uacpi_interrupt_state) {
+    irq_unlock();
+}
+
+uacpi_status uacpi_kernel_handle_firmware_request(uacpi_firmware_request* req) {
+    switch (req->type) {
+    case UACPI_FIRMWARE_REQUEST_TYPE_BREAKPOINT:
+        log_warn("acpi: Ignoring AML breakpoint\n");
+        break;
+    case UACPI_FIRMWARE_REQUEST_TYPE_FATAL:
+        log_err(
+            "acpi: Fatal firmware error: type %u, code %#x, arg %#llx\n",
+            req->fatal.type,
+            req->fatal.code,
+            (unsigned long long)req->fatal.arg
+        );
+        break;
+    }
+    return UACPI_STATUS_OK;
+}
+
+struct acpi_irq {
+    struct irq_handler handler;
+    uacpi_interrupt_handler func;
+    uacpi_handle ctx;
+};
+
+static enum irq_status acpi_irq_raise(struct irq_handler* self) {
+    struct acpi_irq* irq = containerof(self, struct acpi_irq, handler);
+    return irq->func(irq->ctx) == UACPI_INTERRUPT_HANDLED ? IRQ_STATUS_HANDLED : IRQ_STATUS_IGNORED;
+}
+
+uacpi_status uacpi_kernel_install_interrupt_handler(
+    uacpi_u32 irq,
+    uacpi_interrupt_handler handler,
+    uacpi_handle ctx,
+    uacpi_handle* out_irq_handle
+) {
+    uint32_t gsi = irq;
+    enum irq_trigger_mode trigger = IRQ_TRIGGER_LEVEL;
+    enum irq_polarity polarity = IRQ_POLARITY_LOW;
+    if (irq <= UINT8_MAX)
+        arch_irq_isa_override(irq, &gsi, &trigger, &polarity);
+
+    struct irq_line* line;
+    errno_t status = arch_irq_get_gsi(gsi, &line);
+    if (status) {
+        log_err("acpi: Unable to get GSI %u for IRQ %u (%i)\n", gsi, irq, status);
+        return UACPI_STATUS_UNIMPLEMENTED;
+    }
+
+    struct acpi_irq* acpi_irq = mem_alloc(sizeof(*acpi_irq), 0);
+    if (!acpi_irq)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    acpi_irq->handler.raise = acpi_irq_raise;
+    acpi_irq->func = handler;
+    acpi_irq->ctx = ctx;
+
+    irq_line_attach(line, &acpi_irq->handler);
+    irq_line_program(line, trigger, polarity);
+    log("acpi: Installed handler for IRQ %u on GSI %u\n", irq, gsi);
+
+    *out_irq_handle = acpi_irq;
+    return UACPI_STATUS_OK;
+}
+
+uacpi_status uacpi_kernel_uninstall_interrupt_handler(uacpi_interrupt_handler handler, uacpi_handle irq_handle) {
+    struct acpi_irq* acpi_irq = irq_handle;
+    struct irq_line* line = acpi_irq->handler.line;
+    irq_line_detach(line, &acpi_irq->handler);
+    if (list_is_empty(&line->handlers) && line->mask)
+        line->mask(line);
+    mem_free(acpi_irq);
+    return UACPI_STATUS_OK;
+}
+
+uacpi_handle uacpi_kernel_create_spinlock() {
+    struct spinlock* lock = mem_alloc(sizeof(*lock), 0);
+    if (lock)
+        spin_init(lock);
+    return lock;
+}
+
+void uacpi_kernel_free_spinlock(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_cpu_flags uacpi_kernel_lock_spinlock(uacpi_handle handle) {
+    spin_lock(handle);
+    return 0;
+}
+
+void uacpi_kernel_unlock_spinlock(uacpi_handle handle, uacpi_cpu_flags) {
+    spin_unlock(handle);
+}
+
+struct acpi_work {
+    struct work work;
+    uacpi_work_handler handler;
+    uacpi_handle ctx;
+};
+
+static void acpi_work_fn(struct work* work) {
+    struct acpi_work* item = containerof(work, struct acpi_work, work);
+    const uacpi_work_handler handler = item->handler;
+    const uacpi_handle ctx = item->ctx;
+    handler(ctx);
+}
+
+uacpi_handle uacpi_kernel_create_work_item() {
+    struct acpi_work* item = mem_alloc(sizeof(*item), 0);
+    if (item)
+        work_init(&item->work, acpi_work_fn);
+    return item;
+}
+
+void uacpi_kernel_free_work_item(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+void uacpi_kernel_schedule_work(uacpi_work_type, uacpi_handle work_item, uacpi_work_handler handler, uacpi_handle ctx) {
+    struct acpi_work* item = work_item;
+    item->handler = handler;
+    item->ctx = ctx;
+    work_queue(acpi_workqueue, &item->work);
+}
+
+uacpi_status uacpi_kernel_wait_for_work_completion() {
+    workqueue_flush(acpi_workqueue);
+    return UACPI_STATUS_OK;
+}

dev/block/bio.c

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/utils.h>
+
+struct bio* bio_alloc(enum bio_op op, size_t num_segs) {
+    struct bio* bio = mem_alloc(sizeof(struct bio) + num_segs * sizeof(struct bio_seg), 0);
+    if (!bio)
+        return nullptr;
+    bio->op = op;
+    bio->num_segs = num_segs;
+    event_init(&bio->event);
+    atomic_init(&bio->pending, 0);
+    atomic_init(&bio->short_offset, UINT64_MAX);
+    atomic_init(&bio->child_status, 0);
+    return bio;
+}
+
+void bio_free(struct bio* bio) {
+    mem_free(bio);
+}
+
+static void record_short(struct bio* parent, uint64_t offset, errno_t status) {
+    uint64_t current = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    while (offset < current) {
+        if (atomic_compare_exchange_weak(&parent->short_offset, &current, offset, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            break;
+    }
+    if (status) {
+        errno_t expected = 0;
+        atomic_compare_exchange(&parent->child_status, &expected, status, ATOMIC_RELAXED, ATOMIC_RELAXED);
+    }
+}
+
+static void parent_put(struct bio* parent) {
+    if (atomic_fetch_sub(&parent->pending, 1, ATOMIC_ACQ_REL) != 1)
+        return;
+    const uint64_t offset = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    const size_t done = offset == UINT64_MAX ? parent->num_lbas : offset;
+    const errno_t status = atomic_load(&parent->child_status, ATOMIC_RELAXED);
+    bio_complete(parent, done == 0 ? status : 0, done);
+}
+
+void bio_complete(struct bio* bio, errno_t status, size_t done_lbas) {
+    bio->status = status;
+    bio->done_lbas = done_lbas;
+
+    struct bio* parent = bio->parent;
+    if (parent) {
+        if (status || done_lbas < bio->num_lbas)
+            record_short(parent, bio->parent_offset + done_lbas, status);
+        bio_free(bio);
+        parent_put(parent);
+        return;
+    }
+
+    if (bio->end)
+        bio->end(bio);
+    else
+        event_set(&bio->event);
+}
+
+errno_t bio_wait(struct bio* bio, size_t* out_lbas) {
+    event_wait(&bio->event);
+    if (out_lbas)
+        *out_lbas = bio->done_lbas;
+    return bio->status;
+}
+
+// Submits one bio and completes early if it failed.
+static void submit_one(struct blockdev* dev, struct bio* bio) {
+    errno_t status = dev->ops->submit(dev, bio);
+    if (status)
+        bio_complete(bio, status, 0);
+}
+
+void bio_submit(struct blockdev* dev, struct bio* bio) {
+    bio->dev = dev;
+    if (bio->op == BIO_FLUSH) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    if (bio->op == BIO_READ) {
+        if (bio->lba >= dev->lba_count) {
+            bio_complete(bio, 0, 0);
+            return;
+        }
+        bio->num_lbas = MIN(bio->num_lbas, dev->lba_count - bio->lba);
+    }
+    if (bio->num_lbas == 0) {
+        bio_complete(bio, 0, 0);
+        return;
+    }
+
+    const size_t lba_size = dev->lba_size;
+    const size_t lbas_per_page = MAX(arch_mem_page_size() / lba_size, 1);
+    size_t max_lbas = dev->limits.max_lbas ? dev->limits.max_lbas : SIZE_MAX;
+    if (max_lbas > lbas_per_page)
+        max_lbas -= max_lbas % lbas_per_page;
+    const size_t max_segs = dev->limits.max_segs ? dev->limits.max_segs : SIZE_MAX;
+
+    if (bio->num_lbas <= max_lbas && bio->num_segs <= max_segs) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    atomic_store(&bio->pending, 1, ATOMIC_RELAXED);
+    atomic_store(&bio->short_offset, UINT64_MAX, ATOMIC_RELAXED);
+    atomic_store(&bio->child_status, 0, ATOMIC_RELAXED);
+
+    size_t seg = 0;
+    size_t seg_off = 0;
+    uint64_t consumed = 0;
+    while (consumed < bio->num_lbas) {
+        const size_t cap = MIN(max_segs, bio->num_segs - seg);
+        struct bio* child = cap ? bio_alloc(bio->op, cap) : nullptr;
+        if (!child) {
+            record_short(bio, consumed, cap ? ENOMEM : EINVAL);
+            break;
+        }
+
+        size_t count = 0;
+        size_t lbas = 0;
+        while (lbas < max_lbas && count < cap && consumed + lbas < bio->num_lbas) {
+            const struct bio_seg* src = &bio->segs[seg];
+            const size_t take =
+                MIN(MIN(src->len - seg_off, (max_lbas - lbas) * lba_size),
+                    (bio->num_lbas - consumed - lbas) * lba_size);
+            if (take < lba_size)
+                break;
+            child->segs[count++] = (struct bio_seg){.addr = src->addr + seg_off, .len = take};
+            lbas += take / lba_size;
+            seg_off += take;
+            if (seg_off == src->len) {
+                seg++;
+                seg_off = 0;
+            }
+        }
+
+        if (lbas == 0) {
+            bio_free(child);
+            record_short(bio, consumed, EINVAL);
+            break;
+        }
+
+        child->num_segs = count;
+        child->num_lbas = lbas;
+        child->lba = bio->lba + consumed;
+        child->parent = bio;
+        child->parent_offset = consumed;
+        child->dev = dev;
+        consumed += lbas;
+        atomic_fetch_add(&bio->pending, 1, ATOMIC_RELAXED);
+        submit_one(dev, child);
+    }
+
+    parent_put(bio);
+}

dev/block/blockdev.c

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "block: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/tailq.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vec.h>
+#include <zinnia/vma.h>
+#include <string.h>
+#include <uapi/block.h>
+
+static size_t lbas_per_page(struct blockdev* dev) {
+    return arch_mem_page_size() / dev->lba_size;
+}
+
+static errno_t cache_get_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t page_size = arch_mem_page_size();
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+
+    if (lba >= dev->lba_count) {
+        *out_valid = 0;
+        return 0;
+    }
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_READ, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+
+    const size_t valid = done / per_page + (done % per_page != 0);
+    if (done % per_page != 0) {
+        const size_t tail = (done % per_page) * dev->lba_size;
+        memset((uint8_t*)HHDM_PTR(vm_page_addr(pages[valid - 1])) + tail, 0, page_size - tail);
+    }
+    *out_valid = valid;
+    return valid == 0 && num_lbas > 0 ? EIO : 0;
+}
+
+static errno_t cache_put_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+    if (lba >= dev->lba_count)
+        return 0;
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_WRITE, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+    return done < num_lbas ? EIO : 0;
+}
+
+static void cache_destroy(struct vmo* vmo) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    if (dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static const struct vmo_pager_ops cache_pager = {
+    .get_pages = cache_get_pages,
+    .put_pages = cache_put_pages,
+    .max_cluster = 64,
+    .destroy = cache_destroy,
+};
+
+void blockdev_init(
+    struct blockdev* dev,
+    const struct blockdev_ops* ops,
+    uint32_t major,
+    uint32_t minor,
+    size_t lba_size,
+    uint64_t lba_count
+) {
+    vmo_init(&dev->cache, &cache_pager);
+    dev->ops = ops;
+    dev->dev = dev_make(major, minor);
+    dev->lba_size = lba_size;
+    dev->lba_count = lba_count;
+    atomic_init(&dev->claimed, false);
+}
+
+void blockdev_ref(struct blockdev* dev) {
+    vmo_ref(&dev->cache);
+}
+
+void blockdev_unref(struct blockdev* dev) {
+    vmo_unref(&dev->cache);
+}
+
+errno_t blockdev_rw(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    const struct bio_seg* segs,
+    size_t num_segs,
+    size_t* out_lbas
+) {
+    struct bio* bio = bio_alloc(op, num_segs);
+    if (!bio)
+        return ENOMEM;
+    memcpy(bio->segs, segs, num_segs * sizeof(struct bio_seg));
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_rw_pages(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    struct page** pages,
+    size_t count,
+    size_t* out_lbas
+) {
+    const size_t page_size = arch_mem_page_size();
+    size_t bytes = num_lbas * dev->lba_size;
+    if (bytes > count * page_size)
+        return EINVAL;
+
+    struct bio* bio = bio_alloc(op, count);
+    if (!bio)
+        return ENOMEM;
+
+    size_t num_segs = 0;
+    for (size_t i = 0; i < count && bytes > 0; i++) {
+        const size_t len = MIN(bytes, page_size);
+        bio->segs[num_segs++] = (struct bio_seg){.addr = vm_page_addr(pages[i]), .len = len};
+        bytes -= len;
+    }
+    bio->num_segs = num_segs;
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_flush(struct blockdev* dev) {
+    struct bio* bio = bio_alloc(BIO_FLUSH, 0);
+    if (!bio)
+        return ENOMEM;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, nullptr);
+    bio_free(bio);
+    return status;
+}
+
+static struct blockdev* file_dev(struct vfs_file* file) {
+    return file->priv;
+}
+
+static uint64_t dev_size(struct blockdev* dev) {
+    return dev->lba_count * dev->lba_size;
+}
+
+static errno_t blockdev_file_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct blockdev* dev = file_dev(file);
+    *out_read = 0;
+    if (offset >= dev_size(dev))
+        return 0;
+
+    size_t progress;
+    errno_t status =
+        vmo_read_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_read = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct blockdev* dev = file_dev(file);
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (offset >= dev_size(dev))
+        return ENOSPC;
+
+    size_t progress;
+    errno_t status =
+        vmo_write_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_written = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    struct blockdev* dev = file_dev(file);
+    switch (request) {
+    case __BLKGETSIZE64: {
+        const uint64_t size = dev_size(dev);
+        if (!usercopy_write((__user void*)arg, &size, sizeof(size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKSSZGET: {
+        const unsigned int lba_size = dev->lba_size;
+        if (!usercopy_write((__user void*)arg, &lba_size, sizeof(lba_size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKRRPART: {
+        errno_t status = blockdev_rescan(dev);
+        if (!status)
+            *out_result = 0;
+        return status;
+    }
+    default:
+        if (dev->ops->ioctl)
+            return dev->ops->ioctl(dev, file, request, arg, out_result);
+        return ENOTTY;
+    }
+}
+
+static errno_t blockdev_file_sync(struct vfs_file* file, bool data_only) {
+    struct blockdev* dev = file_dev(file);
+    errno_t status = vmo_sync(&dev->cache, 0, UINT64_MAX);
+    if (status)
+        return status;
+    return blockdev_flush(dev);
+}
+
+static errno_t blockdev_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    errno_t status =
+        vma_map(vma, &file_dev(file)->cache, addr, len, prot, (flags & MAP_SHARED) ? VMA_SHARED : 0, offset);
+    if (status)
+        return status;
+    *out_addr = addr;
+    return 0;
+}
+
+static void blockdev_file_release(struct vfs_file* file) {
+    blockdev_unref(file_dev(file));
+}
+
+static const struct vfs_file_ops blockdev_file_ops = {
+    .read = blockdev_file_read,
+    .write = blockdev_file_write,
+    .ioctl = blockdev_file_ioctl,
+    .sync = blockdev_file_sync,
+    .mmap = blockdev_file_mmap,
+    .release = blockdev_file_release,
+};
+
+static errno_t blockdev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    file->priv = dev;
+    file->ops = &blockdev_file_ops;
+    return 0;
+}
+
+static void blockdev_node_destroy(struct vfs_node* node) {
+    blockdev_unref(node->priv);
+}
+
+static const struct vfs_node_ops blockdev_node_ops = {
+    .destroy = blockdev_node_destroy,
+    .device = {
+        .open = blockdev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_BLOCKDEV,
+        .ops = &blockdev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    blockdev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        blockdev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out) {
+    if (node->type != VNODE_BLOCKDEV || node->ops != &blockdev_node_ops)
+        return ENOTBLK;
+
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    *out = dev;
+    return 0;
+}
+
+errno_t blockdev_claim(struct blockdev* dev) {
+    bool expected = false;
+    if (!atomic_compare_exchange(&dev->claimed, &expected, true, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return EBUSY;
+    return 0;
+}
+
+void blockdev_release(struct blockdev* dev) {
+    atomic_store(&dev->claimed, false, ATOMIC_RELEASE);
+}
+
+struct disk_node {
+    char name[64];
+    struct blockdev* dev;
+};
+
+struct disk {
+    char name[32];
+    struct blockdev* dev;
+    vec(struct disk_node) nodes;
+    tailq_link(struct disk) link;
+};
+
+static tailq_head(struct disk) disks;
+static mutex_t disks_lock;
+
+static void add_node(struct disk* disk, const char* name, struct blockdev* dev) {
+    struct disk_node node = {.dev = dev};
+    ksnprintf(node.name, sizeof(node.name), "%s", name);
+    if (vec_push(&disk->nodes, node) && dev)
+        blockdev_unref(dev);
+}
+
+static void scan_partitions(struct disk* disk) {
+    struct gpt_partition* parts;
+    size_t count;
+    if (gpt_scan(disk->dev, &parts, &count))
+        return;
+
+    for (size_t i = 0; i < count; i++) {
+        const struct gpt_partition* part = &parts[i];
+        if (part->end_lba < part->start_lba || part->end_lba >= disk->dev->lba_count) {
+            log_warn("Ignoring partition %zu of \"%s\" with invalid range\n", i + 1, disk->name);
+            continue;
+        }
+
+        char part_name[48];
+        ksnprintf(part_name, sizeof(part_name), "%sp%zu", disk->name, i + 1);
+
+        struct blockdev* dev;
+        errno_t status = partition_new(disk->dev, part->start_lba, part->end_lba - part->start_lba + 1, &dev);
+        if (status) {
+            log_warn("Unable to create partition \"%s\" (%i)\n", part_name, status);
+            continue;
+        }
+
+        char path[64];
+        ksnprintf(path, sizeof(path), "block/%s", part_name);
+        status = devtmpfs_add_blockdev(path, dev, 0660);
+        if (status) {
+            blockdev_unref(dev);
+            log_warn("Unable to register \"%s\" (%i)\n", path, status);
+            continue;
+        }
+        add_node(disk, path, dev);
+
+        char type_str[GUID_STR_LEN];
+        char uuid_str[GUID_STR_LEN];
+        guid_format(&part->type_guid, type_str);
+        guid_format(&part->unique_guid, uuid_str);
+
+        ksnprintf(path, sizeof(path), "block/parttype-%s", type_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+        else if (status == EEXIST)
+            log("Partition type %s already claimed, no \"%s\" link for \"%s\"\n", type_str, path, part_name);
+
+        ksnprintf(path, sizeof(path), "block/partuuid-%s", uuid_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+
+        log("Partition %zu: \"%s\" Type: %s UUID: %s\n", i + 1, part_name, type_str, uuid_str);
+    }
+
+    mem_free(parts);
+}
+
+errno_t blockdev_register(const char* name, struct blockdev* dev) {
+    struct disk* disk = mem_alloc(sizeof(struct disk), 0);
+    if (!disk)
+        return ENOMEM;
+    ksnprintf(disk->name, sizeof(disk->name), "%s", name);
+    vec_init(&disk->nodes);
+
+    char path[64];
+    ksnprintf(path, sizeof(path), "block/%s", name);
+    errno_t status = devtmpfs_add_blockdev(path, dev, 0660);
+    if (status) {
+        mem_free(disk);
+        return status;
+    }
+
+    blockdev_ref(dev);
+    disk->dev = dev;
+    log("Registered block device \"%s\"\n", name);
+
+    mutex_lock(&disks_lock);
+    tailq_insert_tail(&disks, disk, link);
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_rescan(struct blockdev* dev) {
+    mutex_lock(&disks_lock);
+    struct disk* disk;
+    for (disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        if (disk->dev == dev)
+            break;
+    }
+    if (!disk) {
+        mutex_unlock(&disks_lock);
+        return ENOTTY;
+    }
+
+    for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+        struct disk_node* node = &vec_elem_get(&disk->nodes, i);
+        errno_t status = devtmpfs_remove(node->name);
+        if (status)
+            log_warn("Unable to remove stale node \"%s\" (%i)\n", node->name, status);
+        if (node->dev)
+            blockdev_unref(node->dev);
+    }
+    disk->nodes.length = 0;
+
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_sync_all() {
+    errno_t result = 0;
+    mutex_lock(&disks_lock);
+    for (struct disk* disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+            struct blockdev* part = vec_elem_get(&disk->nodes, i).dev;
+            errno_t status = part ? vmo_sync(&part->cache, 0, UINT64_MAX) : 0;
+            if (status && !result)
+                result = status;
+        }
+        errno_t status = vmo_sync(&disk->dev->cache, 0, UINT64_MAX);
+        if (!status)
+            status = blockdev_flush(disk->dev);
+        if (status && !result)
+            result = status;
+    }
+    mutex_unlock(&disks_lock);
+    return result;
+}
+
+static void block_stage_init(struct boot_info* info) {
+    tailq_init(&disks);
+    mutex_init(&disks_lock);
+    errno_t status = devtmpfs_mkdir("block", 0755);
+    if (status && status != EEXIST)
+        log_warn("Unable to create /dev/block (%i)\n", status);
+}
+
+INIT_TASK(block_stage, "dev.block", block_stage_init);
+INIT_DEPENDS(block_stage, devtmpfs_stage);

dev/block/gpt.c

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+struct [[__packed]] gpt_header {
+    char signature[8];
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t header_crc32;
+    uint32_t reserved;
+    uint64_t my_lba;
+    uint64_t alternate_lba;
+    uint64_t first_usable_lba;
+    uint64_t last_usable_lba;
+    struct guid disk_guid;
+    uint64_t partition_entry_lba;
+    uint32_t num_partition_entries;
+    uint32_t partition_entry_size;
+    uint32_t partition_entry_array_crc32;
+};
+
+struct [[__packed]] gpt_entry {
+    struct guid type_guid;
+    struct guid unique_guid;
+    uint64_t start_lba;
+    uint64_t end_lba;
+    uint64_t attributes;
+    uint16_t name[36];
+};
+
+constexpr size_t GPT_MAX_ENTRIES = 1024;
+
+errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count) {
+    struct gpt_header header;
+    size_t read;
+    errno_t status = vmo_read(&dev->cache, dev->lba_size, &header, sizeof(header), &read);
+    if (status)
+        return status;
+    if (read != sizeof(header) || memcmp(header.signature, "EFI PART", 8) != 0)
+        return ENODATA;
+    if (header.partition_entry_size < sizeof(struct gpt_entry) || header.num_partition_entries > GPT_MAX_ENTRIES)
+        return EINVAL;
+
+    struct gpt_partition* parts = mem_alloc(MAX(header.num_partition_entries, 1) * sizeof(struct gpt_partition), 0);
+    if (!parts)
+        return ENOMEM;
+
+    size_t count = 0;
+    const uint64_t base = header.partition_entry_lba * dev->lba_size;
+    for (size_t i = 0; i < header.num_partition_entries; i++) {
+        struct gpt_entry entry;
+        status = vmo_read(&dev->cache, base + i * header.partition_entry_size, &entry, sizeof(entry), &read);
+        if (status || read != sizeof(entry))
+            break;
+        if (guid_is_zero(&entry.type_guid))
+            continue;
+
+        struct gpt_partition* part = &parts[count++];
+        part->type_guid = entry.type_guid;
+        part->unique_guid = entry.unique_guid;
+        part->start_lba = entry.start_lba;
+        part->end_lba = entry.end_lba;
+    }
+
+    *out = parts;
+    *out_count = count;
+    return 0;
+}

dev/block/meson.build

@@ -1,3 +1,11 @@
-if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
+kernel_sources += files(
+    'bio.c',
+    'blockdev.c',
+    'gpt.c',
+    'partition.c',
+    'ram.c',
+)
+
+if kconfig.get('CONFIG_NVME', 'n') != 'n'
     subdir('nvme')
 endif

dev/block/nvme/Kconfig

@@ -1,13 +1,4 @@
-menu "NVMe Support"
-
-config NVME_CORE
-  bool
-  default y
-
-config NVME_PCI
-  bool "NVMe via PCI transport"
+config NVME
+  bool "NVMe block devices"
   default y
   depends on PCI
-  depends on NVME_CORE
-
-endmenu

dev/block/nvme/command.c

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/mmio.h>
+#include "nvme.h"
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, cmd->do_write ? NVME_CMD_WRITE : NVME_CMD_READ) |
+        mmio_field(NVME_SQE_CID, cmd->cid) | mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_RW_SLBA_LOW, (uint32_t)cmd->start_lba);
+
+    uint32_t cdw11 = mmio_field(NVME_RW_SLBA_HIGH, (uint32_t)(cmd->start_lba >> 32));
+
+    uint32_t cdw12 = mmio_field(NVME_RW_NLB, (uint16_t)(cmd->num_lbas - 1));
+    uint32_t cdw13 = 0;
+    uint32_t cdw14 = 0;
+    uint32_t cdw15 = 0;
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->prp1);
+    mmio_reg_write(sqe, NVME_SQE_DPTR1, cmd->prp2);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+    mmio_reg_write(sqe, NVME_SQE_CDW12, cdw12);
+    mmio_reg_write(sqe, NVME_SQE_CDW13, cdw13);
+    mmio_reg_write(sqe, NVME_SQE_CDW14, cdw14);
+    mmio_reg_write(sqe, NVME_SQE_CDW15, cdw15);
+}
+
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_CMD_FLUSH) | mmio_field(NVME_SQE_CID, cmd->cid);
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+}
+
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_CQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_CQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_CQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_CQ_IV, cmd->irq_vector) |
+        mmio_field(NVME_CREATE_CQ_IEN, cmd->irqs_enabled ? 1 : 0) |
+        mmio_field(NVME_CREATE_CQ_PC, 1); // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->cq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_SQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_SQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_SQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_SQ_CQID, (uint16_t)cmd->cqid) |
+        mmio_field(NVME_CREATE_SQ_QPRIO, 0) | // Priority is the highest.
+        mmio_field(NVME_CREATE_SQ_PC, 1);     // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->sq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_IDENTIFY) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_IDENTIFY_CNTID, cmd->controller_id) | mmio_field(NVME_IDENTIFY_CNS, cmd->cns);
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->buffer);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+}

dev/block/nvme/controller.c

@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t ADMIN_QUEUE_SIZE = 32;
+constexpr size_t IO_QUEUE_SIZE_MAX = 1024;
+constexpr size_t IDENTIFY_SIZE = 4096;
+constexpr size_t NS_LIST_LEN = 1024;
+
+constexpr mmio_reg8 ID_MDTS = {77, MMIO_LE};
+constexpr mmio_reg32 ID_NN = {516, MMIO_LE};
+constexpr mmio_reg64 ID_NSZE = {0, MMIO_LE};
+constexpr mmio_reg8 ID_FLBAS = {26, MMIO_LE};
+constexpr mmio_field8 ID_LBAF_IDX = {ID_FLBAS, 0, 4};
+constexpr mmio_reg32 ID_LBAF0 = {128, MMIO_LE};
+constexpr mmio_field32 ID_LBADS = {ID_LBAF0, 16, 8};
+
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl) {
+    const size_t cap = 512 * arch_mem_page_size();
+    if (ctrl->mdts != 0)
+        return MIN(ctrl->mdts, cap);
+    return cap;
+}
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out) {
+    // Read controller version.
+    uint32_t vs = mmio_reg_read(regs, NVME_REG_VS);
+    log("NVMe controller version %u.%u.%u\n",
+        mmio_field_get(NVME_VS_MJR, vs),
+        mmio_field_get(NVME_VS_MNR, vs),
+        mmio_field_get(NVME_VS_TER, vs));
+
+    uint64_t cap = mmio_reg_read(regs, NVME_REG_CAP);
+
+    // Check if our host page size is supported.
+    const size_t mps_max = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMAX, cap) + 12);
+    const size_t mps_min = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMIN, cap) + 12);
+    const size_t page_size = arch_mem_page_size();
+    if (page_size < mps_min || page_size > mps_max) {
+        log_err("Host page size is not supported on this NVMe!\n");
+        return ENOTSUP;
+    }
+
+    struct nvme_ctrl* ctrl = mem_alloc(sizeof(*ctrl), 0);
+    if (!ctrl)
+        return ENOMEM;
+
+    ctrl->regs = regs;
+    ctrl->dma_tag = dma_tag;
+    ctrl->doorbell_stride = (size_t)4 << mmio_field_get(NVME_CAP_DSTRD, cap);
+    ctrl->mps_min = mps_min;
+    ctrl->mps = __builtin_ctzl(page_size) - 12;
+    mutex_init(&ctrl->admin_lock);
+    spin_init(&ctrl->io_lock);
+    atomic_init(&ctrl->ns_count, 0);
+
+    *out = ctrl;
+    return 0;
+}
+
+// Sets the CC.EN flag.
+static errno_t set_status(struct nvme_ctrl* ctrl, bool enable) {
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+    // CAP.TO is in 500ms units.
+    const uint64_t timeout = MAX((uint64_t)mmio_field_get(NVME_CAP_TO, cap), 1) * 500'000'000;
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout;
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &= ~mmio_field(NVME_CC_EN, UINT32_MAX);
+    cc |= mmio_field(NVME_CC_EN, enable ? 1 : 0);
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    while (true) {
+        uint32_t csts = mmio_reg_read(ctrl->regs, NVME_REG_CSTS);
+
+        if (mmio_field_get(NVME_CSTS_CFS, csts) != 0)
+            return EIO;
+
+        bool ready = mmio_field_get(NVME_CSTS_RDY, csts) != 0;
+        if (ready == enable)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    return 0;
+}
+
+static errno_t admin_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe, struct nvme_completion* out) {
+    errno_t status = nvme_queue_submit_cmd(queue, sqe);
+    if (status)
+        return status;
+    return nvme_queue_next_completion(queue, out);
+}
+
+static errno_t identify(struct nvme_queue* queue, const struct nvme_identify_cmd* cmd) {
+    struct nvme_sqe sqe = {};
+    nvme_identify_cmd_write(cmd, &sqe);
+    struct nvme_completion comp;
+    errno_t status = admin_cmd(queue, &sqe, &comp);
+    if (status)
+        return status;
+    if (!nvme_status_is_success(comp.status)) {
+        log_err("Identification failed with status %x\n", comp.status);
+        return EIO;
+    }
+    return 0;
+}
+
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector) {
+    // Disable the controller.
+    errno_t status = set_status(ctrl, false);
+    if (status)
+        return status;
+
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+
+    const size_t queue_depth = MIN((size_t)mmio_field_get(NVME_CAP_MQES, cap) + 1, IO_QUEUE_SIZE_MAX);
+
+    // Create an admin queue first so we can create more queues.
+    struct nvme_queue* admin_queue;
+    status = nvme_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 0, ADMIN_QUEUE_SIZE, &admin_queue);
+    if (status)
+        return status;
+
+    // Set the admin queue sizes.
+    uint32_t aqa = 0;
+    aqa |= mmio_field(NVME_AQA_ACQS, admin_queue->depth - 1);
+    aqa |= mmio_field(NVME_AQA_ASQS, admin_queue->depth - 1);
+    mmio_reg_write(ctrl->regs, NVME_REG_AQA, aqa);
+
+    // Set the addresses to the admin submission and completion queue.
+    mmio_reg_write(ctrl->regs, NVME_REG_ASQ, admin_queue->sq.addr);
+    mmio_reg_write(ctrl->regs, NVME_REG_ACQ, admin_queue->cq.addr);
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &=
+        ~(mmio_field(NVME_CC_MPS, UINT32_MAX) | mmio_field(NVME_CC_IOCQES, UINT32_MAX) |
+          mmio_field(NVME_CC_IOSQES, UINT32_MAX));
+
+    cc |= mmio_field(NVME_CC_MPS, ctrl->mps);
+    cc |= mmio_field(NVME_CC_IOCQES, __builtin_ctzl(sizeof(struct nvme_cqe)));
+    cc |= mmio_field(NVME_CC_IOSQES, __builtin_ctzl(sizeof(struct nvme_sqe)));
+
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    // Re-enable the controller.
+    status = set_status(ctrl, true);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_io_queue* io_queue;
+    status =
+        nvme_io_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 1, queue_depth, !irqs_enabled, &io_queue);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_sqe sqe = {};
+    struct nvme_completion comp;
+    nvme_create_cq_cmd_write(
+        &(struct nvme_create_cq_cmd){
+            .cq_addr = io_queue->cq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .irqs_enabled = irqs_enabled,
+            .irq_vector = irqs_enabled ? io_irq_vector : 0,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    sqe = (struct nvme_sqe){};
+    nvme_create_sq_cmd_write(
+        &(struct nvme_create_sq_cmd){
+            .sq_addr = io_queue->sq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .cqid = io_queue->queue_id,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* old_admin = ctrl->admin_queue;
+    ctrl->admin_queue = admin_queue;
+    mutex_unlock(&ctrl->admin_lock);
+    if (old_admin)
+        nvme_queue_free(old_admin);
+
+    spin_lock(&ctrl->io_lock);
+    ctrl->io_queue = io_queue;
+    spin_unlock(&ctrl->io_lock);
+
+    return 0;
+
+fail_io:
+    nvme_io_queue_free(io_queue);
+fail_admin:
+    nvme_queue_free(admin_queue);
+    return status;
+}
+
+static void trim_ascii_end(char* str, size_t len) {
+    while (len > 0 &&
+           (str[len - 1] == ' ' || str[len - 1] == '\t' || str[len - 1] == '\n' || str[len - 1] == '\r' ||
+            str[len - 1] == '\f'))
+        len--;
+    str[len] = '\0';
+}
+
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl) {
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status)
+        return ENOMEM;
+
+    mutex_lock(&ctrl->admin_lock);
+    if (!ctrl->admin_queue) {
+        status = ENXIO;
+        goto out;
+    }
+
+    status = identify(
+        ctrl->admin_queue,
+        &(struct nvme_identify_cmd){
+            .buffer = identify_buffer.addr,
+            .controller_id = 0, // TODO: Use a real number here
+            .cns = 1,
+            .nsid = 0,
+        }
+    );
+    // Check if the identification was successful.
+    if (status)
+        goto out;
+
+    dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+    // Read the model strings.
+    const uint8_t* view = identify_buffer.virt;
+    memcpy(ctrl->serial, view + 4, sizeof(ctrl->serial) - 1);
+    memcpy(ctrl->model, view + 24, sizeof(ctrl->model) - 1);
+    memcpy(ctrl->revision, view + 64, sizeof(ctrl->revision) - 1);
+    trim_ascii_end(ctrl->serial, sizeof(ctrl->serial) - 1);
+    trim_ascii_end(ctrl->model, sizeof(ctrl->model) - 1);
+    trim_ascii_end(ctrl->revision, sizeof(ctrl->revision) - 1);
+    log("Model \"%s\"\n", ctrl->model);
+
+    atomic_store(&ctrl->ns_count, mmio_reg_read(identify_buffer.virt, ID_NN), ATOMIC_RELEASE);
+
+    uint8_t mdts = mmio_reg_read(identify_buffer.virt, ID_MDTS);
+    if (mdts != 0 && mdts < sizeof(size_t) * 8)
+        ctrl->mdts = ((size_t)1 << mdts) * ctrl->mps_min;
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    return status;
+}
+
+// Scans all existing namespaces.
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out) {
+    vec_init(out);
+
+    uint32_t* ns_list = mem_alloc(NS_LIST_LEN * sizeof(uint32_t), 0);
+    if (!ns_list)
+        return ENOMEM;
+
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status) {
+        mem_free(ns_list);
+        return ENOMEM;
+    }
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* aq = ctrl->admin_queue;
+    if (!aq) {
+        status = ENXIO;
+        goto out;
+    }
+
+    uint32_t nsid = 0;
+    while (true) {
+        const struct nvme_identify_cmd admin_ident_cmd = {
+            .buffer = identify_buffer.addr,
+            .controller_id = 0,
+            .cns = 2,
+            .nsid = nsid,
+        };
+        status = identify(aq, &admin_ident_cmd);
+
+        if (status)
+            goto out;
+
+        dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+        // Parse the identifier list until we hit an namespace value of 0.
+        size_t ns_list_len = 0;
+        const uint32_t* raw_list = identify_buffer.virt;
+        while (ns_list_len < NS_LIST_LEN && raw_list[ns_list_len] != 0) {
+            ns_list[ns_list_len] = raw_list[ns_list_len];
+            ns_list_len++;
+        }
+
+        for (size_t i = 0; i < ns_list_len; i++) {
+            const uint32_t id = ns_list[i];
+            const struct nvme_identify_cmd ns_ident_cmd = {
+                .buffer = identify_buffer.addr,
+                .controller_id = 0,
+                .cns = 0,
+                .nsid = id,
+            };
+            status = identify(aq, &ns_ident_cmd);
+
+            if (status)
+                goto out;
+
+            dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+            const uint64_t lba_count = mmio_reg_read(identify_buffer.virt, ID_NSZE);
+
+            // To get the block size of a namespace, we need to read a descriptor (see LBAF0-15).
+            // Get the format descriptor index used by this namespace.
+            const uint8_t lba_format_idx = mmio_field_get(ID_LBAF_IDX, mmio_reg_read(identify_buffer.virt, ID_FLBAS));
+
+            const mmio_reg32 lbaf = {ID_LBAF0.offset + lba_format_idx * sizeof(uint32_t), MMIO_LE};
+            const uint32_t lba_format = mmio_reg_read(identify_buffer.virt, lbaf);
+
+            uint8_t lba_shift = mmio_field_get(ID_LBADS, lba_format);
+            // If LBADS doesn't contain a non-zero value, we need to assume a default. Use 512 bytes here.
+            if (lba_shift == 0)
+                lba_shift = 9;
+
+            struct nvme_ns* ns;
+            status = nvme_ns_new(ctrl, id, lba_shift, lba_count, nvme_ctrl_max_transfer_bytes(ctrl), &ns);
+            if (status)
+                goto out;
+            status = vec_push(out, ns);
+            if (status) {
+                blockdev_unref(&ns->blk);
+                goto out;
+            }
+        }
+
+        // If there is no 0 in the buffer, there's more entries than can fit in `identify_buffer`.
+        if (ns_list_len < NS_LIST_LEN)
+            break;
+        nsid = ns_list[ns_list_len - 1];
+    }
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    mem_free(ns_list);
+    if (status) {
+        vec_foreach(out, i) {
+            blockdev_unref(&vec_elem_get(out, i)->blk);
+        }
+        vec_destroy(out);
+    }
+    return status;
+}
+
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl) {
+    spin_lock(&ctrl->io_lock);
+    struct nvme_io_queue* queue = ctrl->io_queue;
+    spin_unlock(&ctrl->io_lock);
+    return queue;
+}

dev/block/nvme/core.c

@@ -1,41 +0,0 @@
-#define pr_fmt(fmt) "nvme: " fmt
-
-#include <zinnia/fdt.h>
-#include <zinnia/module.h>
-#include <zinnia/pci.h>
-#include <zinnia/print.h>
-#include <zinnia/utils.h>
-#include <config.h>
-
-#if CONFIG_NVME_PCI
-#include "nvme_pci.h"
-#endif
-
-struct nvme_dev {
-#if CONFIG_NVME_PCI
-    struct pci_device pci_dev;
-#endif
-};
-
-errno_t nvme_init(struct module* mod) {
-    errno_t status = 0;
-
-#if CONFIG_NVME_PCI
-    status = pci_driver_register(&nvme_driver);
-#endif
-
-    return status;
-}
-
-errno_t nvme_exit(struct module* mod) {
-    // TODO: Unregister
-    return 0;
-}
-
-MODULE_ENTRY(nvme_init);
-MODULE_EXIT(nvme_exit);
-
-MODULE_DESC("NVMe Block Devices");
-MODULE_AUTHOR("Marvin Friedrich");
-MODULE_LICENSE("ISC");
-MODULE_VERSION("1.0");

dev/block/nvme/meson.build

@@ -1,12 +1,13 @@
-nvme_sources = files('core.c')
-
-if kconfig.get('CONFIG_NVME_PCI', 'n') == 'y'
-    nvme_sources += files('pci.c')
-endif
-
 modules += {
     'nvme': {
-        'modular': kconfig.get('CONFIG_NVME_CORE') == 'm',
-        'sources': nvme_sources,
+        'modular': kconfig.get('CONFIG_NVME') == 'm',
+        'sources': files(
+            'command.c',
+            'controller.c',
+            'namespace.c',
+            'pci.c',
+            'prp.c',
+            'queue.c',
+        ),
     },
 }

dev/block/nvme/namespace.c

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <stdckdint.h>
+#include "nvme.h"
+
+static errno_t nvme_ns_submit(struct blockdev* blk, struct bio* bio) {
+    struct nvme_ns* ns = containerof(blk, struct nvme_ns, blk);
+
+    if (bio->op != BIO_FLUSH) {
+        uint64_t end_lba;
+        if (ckd_add(&end_lba, bio->lba, (uint64_t)bio->num_lbas)) {
+            bio_complete(bio, EOVERFLOW, 0);
+            return 0;
+        }
+        if (end_lba > ns->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+    }
+
+    struct nvme_io_queue* queue = nvme_ctrl_io_queue(ns->ctrl);
+    if (!queue) {
+        bio_complete(bio, EIO, 0);
+        return 0;
+    }
+
+    size_t bytes = 0;
+    for (size_t i = 0; i < bio->num_segs; i++)
+        bytes += bio->segs[i].len;
+
+    _Atomic(bool) done = false;
+    const bool poll = queue->poll;
+
+    uint16_t cid = nvme_io_queue_submit(
+        queue,
+        bio,
+        (struct nvme_rw_cmd){
+            .prp1 = 0,
+            .prp2 = 0,
+            .cid = 0,
+            .do_write = bio->op == BIO_WRITE,
+            .start_lba = bio->lba,
+            .num_lbas = bio->num_lbas,
+            .bytes = bytes,
+            .control = 0,
+            .ds_mgmt = 0,
+            .ref_tag = 0,
+            .app_tag = 0,
+            .app_mask = 0,
+            .nsid = ns->nsid,
+        },
+        poll ? &done : nullptr
+    );
+
+    // If we're not interrupt-driven, poll the IO completion queue.
+    if (poll) {
+        const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
+        while (!atomic_load(&done, ATOMIC_ACQUIRE)) {
+            nvme_io_queue_drain(queue);
+            if (atomic_load(&done, ATOMIC_ACQUIRE))
+                break;
+            if (clock_get_elapsed_ns() >= deadline && nvme_io_queue_abandon(queue, cid, bio)) {
+                bio_complete(bio, EIO, 0);
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+static void nvme_ns_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct nvme_ns, blk));
+}
+
+static const struct blockdev_ops nvme_ns_ops = {
+    .submit = nvme_ns_submit,
+    .destroy = nvme_ns_destroy,
+};
+
+errno_t nvme_ns_new(
+    struct nvme_ctrl* ctrl,
+    uint32_t nsid,
+    uint8_t lba_shift,
+    uint64_t lba_count,
+    size_t max_transfer_bytes,
+    struct nvme_ns** out
+) {
+    log("Namespace %u: LBA size %zu bytes, %zu MBs total\n",
+        nsid,
+        (size_t)1 << lba_shift,
+        (size_t)(lba_count << lba_shift) / 1024 / 1024);
+
+    struct nvme_ns* ns = mem_alloc(sizeof(*ns), 0);
+    if (!ns)
+        return ENOMEM;
+
+    blockdev_init(&ns->blk, &nvme_ns_ops, NVME_MAJOR, nsid, (size_t)1 << lba_shift, lba_count);
+    ns->blk.limits = (struct blockdev_limits){
+        .max_lbas = MAX(max_transfer_bytes >> lba_shift, 1),
+        .max_segs = 512,
+    };
+    ns->ctrl = ctrl;
+    ns->nsid = nsid;
+    ns->lba_shift = lba_shift;
+    ns->lba_count = lba_count;
+    ns->max_transfer_bytes = max_transfer_bytes;
+
+    *out = ns;
+    return 0;
+}

dev/block/nvme/nvme.h

@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/dma.h>
+#include <zinnia/errno.h>
+#include <zinnia/event.h>
+#include <zinnia/mutex.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "spec.h"
+
+constexpr uint32_t NVME_MAJOR = 159;
+constexpr uint64_t NVME_COMPLETION_TIMEOUT_NS = 30'000'000'000;
+
+// An entry in the completion queue.
+struct nvme_completion {
+    uint32_t result;
+    uint16_t sq_head;
+    uint16_t sq_id;
+    uint16_t cmd_id;
+    bool phase_tag;
+    uint16_t status;
+};
+
+static inline bool nvme_status_is_success(uint16_t status) {
+    return status == 0;
+}
+
+struct nvme_rw_cmd {
+    uint64_t prp1;
+    uint64_t prp2;
+    uint16_t cid;
+    bool do_write;
+    uint64_t start_lba;
+    size_t num_lbas;
+    size_t bytes;
+    uint16_t control;
+    uint32_t ds_mgmt;
+    uint32_t ref_tag;
+    uint16_t app_tag;
+    uint16_t app_mask;
+    uint32_t nsid;
+};
+
+struct nvme_flush_cmd {
+    uint16_t cid;
+    uint32_t nsid;
+};
+
+struct nvme_create_cq_cmd {
+    dma_addr_t cq_addr;
+    size_t depth;
+    size_t qid;
+    bool irqs_enabled;
+    uint16_t irq_vector;
+};
+
+struct nvme_create_sq_cmd {
+    dma_addr_t sq_addr;
+    size_t depth;
+    size_t qid;
+    size_t cqid;
+};
+
+struct nvme_identify_cmd {
+    dma_addr_t buffer;
+    uint16_t controller_id;
+    uint8_t cns;
+    uint32_t nsid;
+};
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe);
+
+struct nvme_queue {
+    // Amount of queue entries.
+    size_t depth;
+    size_t doorbells_offset;
+    size_t doorbell_stride;
+    void* regs;
+    // Physical buffer for the completion queue.
+    struct dma_buffer cq;
+    // The index of the current completion queue entry.
+    size_t cq_head;
+    // Determines whether a completion queue entry is new.
+    uint8_t cq_phase;
+    // Physical buffer for the submission queue.
+    struct dma_buffer sq;
+    // The index of the current submission queue entry.
+    size_t sq_tail;
+};
+
+errno_t nvme_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    struct nvme_queue** out
+);
+void nvme_queue_free(struct nvme_queue* queue);
+errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe);
+errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out);
+
+struct nvme_inflight {
+    bool active;
+    struct bio* bio;
+    size_t lbas;
+    struct dma_map* map;
+    struct dma_buffer prps;
+    _Atomic(bool)* done;
+};
+
+struct nvme_io_queue {
+    size_t queue_id;
+    size_t depth;
+    size_t doorbells_offset;
+    size_t doorbell_stride;
+    void* regs;
+    bool poll;
+    struct dma_tag* tag;
+    struct dma_buffer cq;
+    struct dma_buffer sq;
+    struct spinlock sq_lock;
+    size_t sq_tail;
+    struct spinlock cq_lock;
+    size_t cq_head;
+    uint8_t cq_phase;
+    struct spinlock cmds_lock;
+    struct nvme_inflight* slots;
+    uint16_t* free;
+    size_t num_free;
+    struct event slots_free;
+};
+
+errno_t nvme_io_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    bool poll,
+    struct nvme_io_queue** out
+);
+void nvme_io_queue_free(struct nvme_io_queue* queue);
+uint16_t nvme_io_queue_submit(
+    struct nvme_io_queue* queue,
+    struct bio* bio,
+    struct nvme_rw_cmd cmd,
+    _Atomic(bool)* done
+);
+bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio);
+size_t nvme_io_queue_drain(struct nvme_io_queue* queue);
+
+errno_t nvme_build_prps(
+    struct dma_tag* tag,
+    struct bio* bio,
+    struct nvme_inflight* slot,
+    uint64_t* prp1,
+    uint64_t* prp2
+);
+void nvme_release_prps(struct nvme_inflight* slot);
+
+struct nvme_ns;
+
+struct nvme_ctrl {
+    void* regs;
+    struct dma_tag* dma_tag;
+    size_t doorbell_stride; // The amount of bytes between two doorbells.
+    size_t mps_min;
+    uint8_t mps;
+
+    char serial[20 + 1];  // Serial number of the controller.
+    char model[40 + 1];   // Model name of the controller.
+    char revision[8 + 1]; // Revision of the controller's firmware.
+
+    struct nvme_queue* admin_queue; // Queue to submit admin commands to.
+    mutex_t admin_lock;
+
+    struct nvme_io_queue* io_queue; // Queue to submit IO commands to.
+    struct spinlock io_lock;
+
+    _Atomic(uint32_t) ns_count; // Amount of namespaces.
+    size_t mdts;                // Maximum data transfer size.
+};
+
+typedef vec(struct nvme_ns*) nvme_ns_vec_t;
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out);
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl);
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector);
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl);
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out);
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl);
+
+struct nvme_ns {
+    struct blockdev blk;
+    struct nvme_ctrl* ctrl;
+    uint32_t nsid;
+    uint8_t lba_shift;
+    uint64_t lba_count;
+    size_t max_transfer_bytes;
+};
+
+errno_t nvme_ns_new(
+    struct nvme_ctrl* ctrl,
+    uint32_t nsid,
+    uint8_t lba_shift,
+    uint64_t lba_count,
+    size_t max_transfer_bytes,
+    struct nvme_ns** out
+);

dev/block/nvme/nvme_pci.h

@@ -1,3 +0,0 @@
-#pragma once
-
-extern struct pci_driver nvme_driver;

dev/block/nvme/pci.c

@@ -1,11 +1,134 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/module.h>
 #include <zinnia/pci.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+#include "nvme.h"
+
+static _Atomic(size_t) nvme_counter;
+
+struct nvme_irq_handler {
+    struct irq_handler handler;
+    struct nvme_ctrl* controller;
+};
+
+static void nvme_irq_thread(struct irq_handler* self) {
+    struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler);
+    struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller);
+    if (queue)
+        nvme_io_queue_drain(queue);
+}
+
+static errno_t map_bar(struct pci_bar* bar, void** out) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t len = ALIGN_UP(bar->size, page_size);
+    uintptr_t vaddr = vm_alloc(len);
+    if (!vaddr)
+        return ENOMEM;
+
+    for (size_t i = 0; i < len; i += page_size) {
+        errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, bar->address + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status)
+            return status;
+    }
+
+    *out = (void*)vaddr;
+    return 0;
+}
 
 static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
-    pci_log(dev, "Detected controller \n");
+    pci_log(dev, "Probing NVMe device\n");
 
-    return 0;
+    // Enable MMIO decoding and DMA. We only support MSI-X or polling, so keep legacy INTx disabled.
+    uint16_t cmd = pci_read16(dev, PCI_COMMAND);
+    pci_write16(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEM | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE);
+
+    struct irq_line* irq_line = nullptr;
+    if (pci_setup_msix(dev, &irq_line)) {
+        pci_log_warn(dev, "NVMe MSI-X setup failed, falling back to polling completions\n");
+        irq_line = nullptr;
+    }
+
+    struct pci_bar bar;
+    errno_t status = pci_bar_get(dev, 0, &bar);
+    if (status)
+        return ENXIO;
+    if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
+        return ENXIO;
+
+    void* regs;
+    status = map_bar(&bar, &regs);
+    if (status)
+        return status;
+
+    struct nvme_ctrl* controller;
+    status = nvme_ctrl_new_pci(regs, dev->dma_tag, &controller);
+    if (status) {
+        pci_log_err(dev, "Failed to probe controller: %i\n", status);
+        return ENODEV;
+    }
+
+    if (irq_line) {
+        struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0);
+        if (!handler)
+            return ENOMEM;
+        handler->handler.thread = nvme_irq_thread;
+        handler->controller = controller;
+
+        char name[32];
+        ksnprintf(name, sizeof(name), "nvme-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
+        status = irq_line_attach_task(irq_line, &handler->handler, name);
+        if (status) {
+            mem_free(handler);
+            return status;
+        }
+        if (irq_line->unmask)
+            irq_line->unmask(irq_line);
+    }
+
+    // Reset the controller to initialize all queues and other structures.
+    status = nvme_ctrl_reset(controller, irq_line != nullptr, 0);
+    if (status) {
+        pci_log_err(dev, "Failed to reset controller: %i\n", status);
+        return ENODEV;
+    }
+
+    status = nvme_ctrl_identify(controller);
+    if (status) {
+        pci_log_err(dev, "Failed to identify controller: %i\n", status);
+        return ENODEV;
+    }
+
+    nvme_ns_vec_t namespaces;
+    status = nvme_ctrl_scan_namespaces(controller, &namespaces);
+    if (status) {
+        pci_log_err(dev, "Failed to identify controller: %i\n", status);
+        return ENODEV;
+    }
+
+    const size_t nvme_id = atomic_fetch_add(&nvme_counter, 1, ATOMIC_SEQ_CST);
+    vec_foreach(&namespaces, i) {
+        struct nvme_ns* ns = vec_elem_get(&namespaces, i);
+        char path[32];
+        ksnprintf(path, sizeof(path), "nvme%zun%u", nvme_id, ns->nsid);
+        status = blockdev_register(path, &ns->blk);
+        if (status)
+            break;
+    }
+    vec_destroy(&namespaces);
+
+    return status;
 }
 
 static const struct pci_match nvme_match[] = {
@@ -15,9 +138,26 @@ static const struct pci_match nvme_match[] = {
 
 MATCH_TABLE("pci", nvme_match);
 
-struct pci_driver nvme_driver = {
-    .name = "nvme_pci",
+static struct pci_driver nvme_driver = {
+    .name = "nvme",
     .match = nvme_match,
     .num_matches = ARRAY_SIZE(nvme_match),
     .probe = nvme_pci_probe,
 };
+
+errno_t nvme_init(struct module* mod) {
+    return pci_driver_register(&nvme_driver);
+}
+
+errno_t nvme_exit(struct module* mod) {
+    // TODO: Unregister
+    return 0;
+}
+
+MODULE_ENTRY(nvme_init);
+MODULE_EXIT(nvme_exit);
+
+MODULE_DESC("NVMe Block Devices");
+MODULE_AUTHOR("Marvin Friedrich");
+MODULE_LICENSE("ISC");
+MODULE_VERSION("1.0");

dev/block/nvme/prp.c

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include "nvme.h"
+
+static size_t count_pages(struct dma_map* map, size_t page_size) {
+    size_t count = 0;
+    for (size_t i = 0; i < map->num_segs; i++) {
+        const struct dma_seg* seg = &map->segs[i];
+        if (seg->len == 0)
+            continue;
+        const dma_addr_t end = seg->addr + seg->len;
+        for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size)
+            count++;
+    }
+    return count;
+}
+
+static void fill_pages(struct dma_map* map, size_t page_size, uint64_t* out) {
+    bool first_page = true;
+    size_t idx = 0;
+    for (size_t i = 0; i < map->num_segs; i++) {
+        const struct dma_seg* seg = &map->segs[i];
+        if (seg->len == 0)
+            continue;
+        const dma_addr_t end = seg->addr + seg->len;
+        for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size) {
+            if (first_page)
+                first_page = false;
+            else
+                out[idx++] = page;
+        }
+    }
+}
+
+errno_t nvme_build_prps(
+    struct dma_tag* tag,
+    struct bio* bio,
+    struct nvme_inflight* slot,
+    uint64_t* prp1,
+    uint64_t* prp2
+) {
+    const size_t page_size = arch_mem_page_size();
+    slot->map = nullptr;
+    slot->prps = (struct dma_buffer){};
+
+    if (bio->num_segs == 0)
+        return EIO;
+
+    errno_t status = dma_map_create(tag, SIZE_MAX, bio->num_segs, SIZE_MAX, 0, &slot->map);
+    if (status)
+        return status;
+
+    status = dma_map_load_bio(slot->map, bio);
+    if (status)
+        goto fail;
+    if (slot->map->num_segs == 0) {
+        status = EIO;
+        goto fail;
+    }
+
+    dma_map_sync(slot->map, 0, slot->map->mapped_len, bio->op == BIO_WRITE ? DMA_SYNC_PREWRITE : DMA_SYNC_PREREAD);
+
+    *prp1 = slot->map->segs[0].addr;
+
+    // Enumerate the base of every page the transfer touches in order.
+    const size_t pages = count_pages(slot->map, page_size);
+    const size_t rest_pages = pages ? pages - 1 : 0;
+
+    if (rest_pages == 0) {
+        *prp2 = 0;
+        return 0;
+    }
+
+    if (rest_pages == 1) {
+        uint64_t second;
+        fill_pages(slot->map, page_size, &second);
+        *prp2 = second;
+        return 0;
+    }
+
+    // We don't want to chain across pages.
+    const size_t entries_per_page = page_size / sizeof(uint64_t);
+    if (rest_pages > entries_per_page) {
+        status = ENOMEM;
+        goto fail;
+    }
+
+    status = dma_buffer_alloc(tag, page_size, &slot->prps);
+    if (status)
+        goto fail;
+
+    fill_pages(slot->map, page_size, slot->prps.virt);
+    dma_buffer_sync(&slot->prps, 0, rest_pages * sizeof(uint64_t), DMA_SYNC_PREWRITE);
+
+    *prp2 = slot->prps.addr;
+    return 0;
+
+fail:
+    nvme_release_prps(slot);
+    return status;
+}
+
+void nvme_release_prps(struct nvme_inflight* slot) {
+    if (slot->map) {
+        dma_map_sync(
+            slot->map,
+            0,
+            slot->map->mapped_len,
+            slot->bio && slot->bio->op == BIO_WRITE ? DMA_SYNC_POSTWRITE : DMA_SYNC_POSTREAD
+        );
+        dma_map_unload(slot->map);
+        dma_map_destroy(slot->map);
+        slot->map = nullptr;
+    }
+    dma_buffer_free(&slot->prps);
+}

dev/block/nvme/queue.c

@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t DOORBELL_OFFSET = 0x1000;
+
+static void ring_tail(void* regs, size_t doorbells_offset, size_t tail) {
+    mmio_reg_write(regs, ((mmio_reg32){doorbells_offset, MMIO_LE}), (uint32_t)tail);
+}
+
+static void ring_head(void* regs, size_t doorbells_offset, size_t doorbell_stride, size_t head) {
+    mmio_reg_write(regs, ((mmio_reg32){doorbells_offset + doorbell_stride, MMIO_LE}), (uint32_t)head);
+}
+
+// Creates a new submission and completion queue pair.
+errno_t nvme_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    struct nvme_queue** out
+) {
+    const size_t align = 0x1000;
+    const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
+    const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
+
+    struct nvme_queue* queue = mem_alloc(sizeof(*queue), 0);
+    if (!queue)
+        return ENOMEM;
+
+    // Allocate memory the completion queue.
+    errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
+    if (status)
+        goto fail;
+
+    // Allocate memory for the submission queue.
+    status = dma_buffer_alloc(tag, sq_size, &queue->sq);
+    if (status)
+        goto fail;
+
+    // Calculate the offset of the doorbell registers. The stride is already precomputed here.
+    queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
+    queue->doorbell_stride = doorbell_stride;
+    queue->depth = depth;
+    queue->regs = regs;
+    queue->cq_head = 0;
+    queue->cq_phase = 1; // When the controller is enabled, the first phase is 1.
+    queue->sq_tail = 0;
+
+    *out = queue;
+    return 0;
+
+fail:
+    nvme_queue_free(queue);
+    return status;
+}
+
+void nvme_queue_free(struct nvme_queue* queue) {
+    dma_buffer_free(&queue->cq);
+    dma_buffer_free(&queue->sq);
+    mem_free(queue);
+}
+
+// Submits a command to this queue.
+errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe) {
+    struct nvme_sqe* entries = queue->sq.virt;
+    memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
+    dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
+
+    queue->sq_tail++;
+    if (queue->sq_tail == queue->depth)
+        queue->sq_tail = 0;
+
+    // Notify the controller of the new tail index.
+    ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
+
+    return 0;
+}
+
+// Reads the next completion entry from the queue.
+errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out) {
+    struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
+
+    // Wait until the phase for this entry has changed.
+    uint32_t dw3;
+    const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
+    while (true) {
+        dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
+        dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
+
+        // The controller will flip the phase bit of the current entry when writing.
+        if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) == queue->cq_phase)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    // Then, read the rest of the completion queue entry.
+    uint32_t dw0 = mmio_reg_read(entry, NVME_CQE_DW0);
+    uint32_t dw2 = mmio_reg_read(entry, NVME_CQE_DW2);
+
+    *out = (struct nvme_completion){
+        .result = dw0,
+        .sq_head = mmio_field_get(NVME_CQE_SQ_HEAD, dw2),
+        .sq_id = mmio_field_get(NVME_CQE_SQ_IDENT, dw2),
+        .cmd_id = mmio_field_get(NVME_CQE_CID, dw3),
+        .status = mmio_field_get(NVME_CQE_STATUS, dw3),
+        .phase_tag = mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != 0,
+    };
+
+    queue->cq_head++;
+    if (queue->cq_head == queue->depth) {
+        queue->cq_head = 0;
+        queue->cq_phase ^= 1; // Flip the phase bit.
+    }
+
+    // Notify the controller of the new head index.
+    ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
+
+    return 0;
+}
+
+errno_t nvme_io_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    bool poll,
+    struct nvme_io_queue** out
+) {
+    const size_t align = 0x1000;
+    const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
+    const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
+
+    struct nvme_io_queue* queue = mem_alloc(sizeof(*queue), 0);
+    if (!queue)
+        return ENOMEM;
+
+    errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
+    if (status)
+        goto fail;
+
+    status = dma_buffer_alloc(tag, sq_size, &queue->sq);
+    if (status)
+        goto fail;
+
+    const size_t slots = depth - 1;
+    queue->slots = mem_alloc(slots * sizeof(*queue->slots), 0);
+    queue->free = mem_alloc(slots * sizeof(*queue->free), 0);
+    if (!queue->slots || !queue->free) {
+        status = ENOMEM;
+        goto fail;
+    }
+    for (size_t i = 0; i < slots; i++)
+        queue->free[i] = slots - 1 - i;
+    queue->num_free = slots;
+
+    queue->queue_id = queue_id;
+    queue->depth = depth;
+    queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
+    queue->doorbell_stride = doorbell_stride;
+    queue->regs = regs;
+    queue->poll = poll;
+    queue->tag = tag;
+    queue->sq_tail = 0;
+    queue->cq_head = 0;
+    queue->cq_phase = 1;
+    spin_init(&queue->sq_lock);
+    spin_init(&queue->cq_lock);
+    spin_init(&queue->cmds_lock);
+    event_init(&queue->slots_free);
+
+    *out = queue;
+    return 0;
+
+fail:
+    nvme_io_queue_free(queue);
+    return status;
+}
+
+void nvme_io_queue_free(struct nvme_io_queue* queue) {
+    dma_buffer_free(&queue->cq);
+    dma_buffer_free(&queue->sq);
+    mem_free(queue->slots);
+    mem_free(queue->free);
+    mem_free(queue);
+}
+
+static uint16_t acquire_slot(struct nvme_io_queue* queue) {
+    while (true) {
+        spin_lock(&queue->cmds_lock);
+        if (queue->num_free) {
+            uint16_t cid = queue->free[--queue->num_free];
+            spin_unlock(&queue->cmds_lock);
+            return cid;
+        }
+        spin_unlock(&queue->cmds_lock);
+
+        if (queue->poll) {
+            nvme_io_queue_drain(queue);
+            continue;
+        }
+
+        nvme_io_queue_drain(queue);
+        event_reset(&queue->slots_free);
+        spin_lock(&queue->cmds_lock);
+        bool empty = queue->num_free == 0;
+        spin_unlock(&queue->cmds_lock);
+        if (empty)
+            event_wait(&queue->slots_free);
+    }
+}
+
+static void release_slot(struct nvme_io_queue* queue, uint16_t cid) {
+    spin_lock(&queue->cmds_lock);
+    queue->free[queue->num_free++] = cid;
+    spin_unlock(&queue->cmds_lock);
+}
+
+static void write_sqe(struct nvme_io_queue* queue, const struct nvme_sqe* sqe) {
+    spin_lock(&queue->sq_lock);
+    struct nvme_sqe* entries = queue->sq.virt;
+    memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
+    dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
+    queue->sq_tail++;
+    if (queue->sq_tail == queue->depth)
+        queue->sq_tail = 0;
+    ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
+    spin_unlock(&queue->sq_lock);
+}
+
+uint16_t nvme_io_queue_submit(
+    struct nvme_io_queue* queue,
+    struct bio* bio,
+    struct nvme_rw_cmd cmd,
+    _Atomic(bool)* done
+) {
+    const uint16_t cid = acquire_slot(queue);
+    struct nvme_inflight* slot = &queue->slots[cid];
+
+    struct nvme_sqe sqe = {};
+    if (bio->op == BIO_FLUSH) {
+        slot->map = nullptr;
+        slot->prps = (struct dma_buffer){};
+        nvme_flush_cmd_write(&(struct nvme_flush_cmd){.cid = cid, .nsid = cmd.nsid}, &sqe);
+    } else {
+        errno_t status = nvme_build_prps(queue->tag, bio, slot, &cmd.prp1, &cmd.prp2);
+        if (status) {
+            release_slot(queue, cid);
+            bio_complete(bio, EIO, 0);
+            if (done)
+                atomic_store(done, true, ATOMIC_RELEASE);
+            return cid;
+        }
+        cmd.cid = cid;
+        nvme_rw_cmd_write(&cmd, &sqe);
+    }
+
+    spin_lock(&queue->cmds_lock);
+    slot->active = true;
+    slot->bio = bio;
+    slot->lbas = bio->num_lbas;
+    slot->done = done;
+    spin_unlock(&queue->cmds_lock);
+
+    write_sqe(queue, &sqe);
+    return cid;
+}
+
+bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio) {
+    spin_lock(&queue->cmds_lock);
+    struct nvme_inflight* slot = &queue->slots[cid];
+    bool abandoned = slot->active && slot->bio == bio;
+    if (abandoned) {
+        slot->bio = nullptr;
+        slot->done = nullptr;
+    }
+    spin_unlock(&queue->cmds_lock);
+    return abandoned;
+}
+
+size_t nvme_io_queue_drain(struct nvme_io_queue* queue) {
+    spin_lock(&queue->cq_lock);
+    size_t completed = 0;
+
+    while (true) {
+        struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
+        dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
+        uint32_t dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
+        if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != queue->cq_phase)
+            break;
+
+        uint16_t cid = mmio_field_get(NVME_CQE_CID, dw3);
+        uint16_t status = mmio_field_get(NVME_CQE_STATUS, dw3);
+
+        queue->cq_head++;
+        if (queue->cq_head == queue->depth) {
+            queue->cq_head = 0;
+            queue->cq_phase ^= 1;
+        }
+
+        struct nvme_inflight inflight = {};
+        spin_lock(&queue->cmds_lock);
+        if (cid < queue->depth - 1 && queue->slots[cid].active) {
+            inflight = queue->slots[cid];
+            queue->slots[cid].active = false;
+            queue->free[queue->num_free++] = cid;
+        }
+        spin_unlock(&queue->cmds_lock);
+
+        if (inflight.active) {
+            nvme_release_prps(&inflight);
+            if (inflight.bio) {
+                if (nvme_status_is_success(status))
+                    bio_complete(inflight.bio, 0, inflight.lbas);
+                else
+                    bio_complete(inflight.bio, EIO, 0);
+            }
+            if (inflight.done)
+                atomic_store(inflight.done, true, ATOMIC_RELEASE);
+        }
+
+        completed++;
+    }
+
+    if (completed > 0)
+        ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
+
+    spin_unlock(&queue->cq_lock);
+
+    if (completed > 0)
+        event_set(&queue->slots_free);
+
+    return completed;
+}

dev/block/nvme/spec.h

@@ -1 +1,152 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
+
+#include <zinnia/compiler.h>
+#include <zinnia/mmio.h>
+#include <stddef.h>
+#include <stdint.h>
+
+// Submission queue entry.
+struct [[__packed]] nvme_sqe {
+    uint32_t cdw0;
+
+    uint32_t nsid; // Namespace Identifier
+    uint32_t cdw2;
+    uint32_t cdw3;
+
+    uint64_t mptr; // Metadata Pointer
+
+    uint64_t dptr0; // Data Pointer 0
+    uint64_t dptr1; // Data Pointer 1
+
+    uint32_t cdw10;
+    uint32_t cdw11;
+    uint32_t cdw12;
+    uint32_t cdw13;
+    uint32_t cdw14;
+    uint32_t cdw15;
+};
+static_assert(sizeof(struct nvme_sqe) == 0x40);
+
+constexpr mmio_reg32 NVME_SQE_CDW0 = {offsetof(struct nvme_sqe, cdw0), MMIO_LE};
+constexpr mmio_field32 NVME_SQE_CID = {NVME_SQE_CDW0, 16, 16}; // Command Identifier
+constexpr mmio_field32 NVME_SQE_PSDT = {NVME_SQE_CDW0, 14, 2}; // PRP or SGL for Data Transfer
+constexpr mmio_field32 NVME_SQE_FUSE = {NVME_SQE_CDW0, 8, 2};  // Fused Operation
+constexpr mmio_field32 NVME_SQE_OPC = {NVME_SQE_CDW0, 0, 8};   // Opcode
+
+constexpr mmio_reg32 NVME_SQE_NSID = {offsetof(struct nvme_sqe, nsid), MMIO_LE};   // Namespace Identifier
+constexpr mmio_reg64 NVME_SQE_MPTR = {offsetof(struct nvme_sqe, mptr), MMIO_LE};   // Metadata Pointer
+constexpr mmio_reg64 NVME_SQE_DPTR0 = {offsetof(struct nvme_sqe, dptr0), MMIO_LE}; // Data Pointer 0
+constexpr mmio_reg64 NVME_SQE_DPTR1 = {offsetof(struct nvme_sqe, dptr1), MMIO_LE}; // Data Pointer 1
+
+constexpr mmio_reg32 NVME_SQE_CDW10 = {offsetof(struct nvme_sqe, cdw10), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW11 = {offsetof(struct nvme_sqe, cdw11), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW12 = {offsetof(struct nvme_sqe, cdw12), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW13 = {offsetof(struct nvme_sqe, cdw13), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW14 = {offsetof(struct nvme_sqe, cdw14), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW15 = {offsetof(struct nvme_sqe, cdw15), MMIO_LE};
+
+constexpr uint8_t NVME_CREATE_CQ_OPCODE = 0x05;
+constexpr mmio_field32 NVME_CREATE_CQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
+constexpr mmio_field32 NVME_CREATE_CQ_QID = {NVME_SQE_CDW10, 0, 16};    // Queue Identifier
+constexpr mmio_field32 NVME_CREATE_CQ_IV = {NVME_SQE_CDW11, 16, 16};    // Interrupt Vector
+constexpr mmio_field32 NVME_CREATE_CQ_IEN = {NVME_SQE_CDW11, 1, 1};     // Interrupts Enabled
+constexpr mmio_field32 NVME_CREATE_CQ_PC = {NVME_SQE_CDW11, 0, 1};      // Physically Contiguous
+
+constexpr uint8_t NVME_CREATE_SQ_OPCODE = 0x01;
+constexpr mmio_field32 NVME_CREATE_SQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
+constexpr mmio_field32 NVME_CREATE_SQ_QID = {NVME_SQE_CDW10, 0, 16};    // Queue Identifier
+constexpr mmio_field32 NVME_CREATE_SQ_CQID = {NVME_SQE_CDW11, 16, 16};  // Completion Queue Identifier
+constexpr mmio_field32 NVME_CREATE_SQ_QPRIO = {NVME_SQE_CDW11, 1, 2};   // Queue Priority
+constexpr mmio_field32 NVME_CREATE_SQ_PC = {NVME_SQE_CDW11, 0, 1};      // Physically Contiguous
+
+constexpr mmio_field32 NVME_IDENTIFY_CNTID = {NVME_SQE_CDW10, 16, 16}; // Controller Identifier
+constexpr mmio_field32 NVME_IDENTIFY_CNS = {NVME_SQE_CDW10, 0, 8};     // Controller or Namespace Structure
+
+constexpr mmio_field32 NVME_RW_SLBA_LOW = {NVME_SQE_CDW10, 0, 32};  // Starting LBA Lower Bits
+constexpr mmio_field32 NVME_RW_SLBA_HIGH = {NVME_SQE_CDW11, 0, 32}; // Starting LBA Higher Bits
+constexpr mmio_field32 NVME_RW_LR = {NVME_SQE_CDW12, 31, 1};        // Limited Retry
+constexpr mmio_field32 NVME_RW_FUA = {NVME_SQE_CDW12, 30, 1};       // Force Unit Access
+constexpr mmio_field32 NVME_RW_PRINFO = {NVME_SQE_CDW12, 26, 4};    // Protection Information Field
+constexpr mmio_field32 NVME_RW_NLB = {NVME_SQE_CDW12, 0, 16};       // Number of Logical Blocks
+
+struct [[__packed]] nvme_cqe {
+    uint32_t dw0;
+    uint32_t dw1;
+    uint32_t dw2;
+    uint32_t dw3;
+};
+static_assert(sizeof(struct nvme_cqe) == 0x10);
+
+constexpr mmio_reg32 NVME_CQE_DW0 = {offsetof(struct nvme_cqe, dw0), MMIO_LE};
+constexpr mmio_reg32 NVME_CQE_DW2 = {offsetof(struct nvme_cqe, dw2), MMIO_LE};
+constexpr mmio_field32 NVME_CQE_SQ_IDENT = {NVME_CQE_DW2, 16, 16};
+constexpr mmio_field32 NVME_CQE_SQ_HEAD = {NVME_CQE_DW2, 0, 16};
+constexpr mmio_reg32 NVME_CQE_DW3 = {offsetof(struct nvme_cqe, dw3), MMIO_LE};
+constexpr mmio_field32 NVME_CQE_STATUS = {NVME_CQE_DW3, 17, 15};
+constexpr mmio_field32 NVME_CQE_PHASE_TAG = {NVME_CQE_DW3, 16, 1};
+constexpr mmio_field32 NVME_CQE_CID = {NVME_CQE_DW3, 0, 16};
+
+constexpr mmio_reg64 NVME_REG_CAP = {0x00, MMIO_LE};            // Controller Capabilities
+constexpr mmio_field64 NVME_CAP_MPSMAX = {NVME_REG_CAP, 52, 4}; // Memory Page Size Maximum
+constexpr mmio_field64 NVME_CAP_MPSMIN = {NVME_REG_CAP, 48, 4}; // Memory Page Size Minimum
+constexpr mmio_field64 NVME_CAP_BPS = {NVME_REG_CAP, 45, 1};    // Boot Partition Support
+constexpr mmio_field64 NVME_CAP_CSS = {NVME_REG_CAP, 37, 8};    // Command Sets Supported
+constexpr mmio_field64 NVME_CAP_NSSRS = {NVME_REG_CAP, 36, 1};  // NVM Subsystem Reset Supported
+constexpr mmio_field64 NVME_CAP_DSTRD = {NVME_REG_CAP, 32, 4};  // Doorbell Stride
+constexpr mmio_field64 NVME_CAP_TO = {NVME_REG_CAP, 24, 8};     // Timeout
+constexpr mmio_field64 NVME_CAP_AMS = {NVME_REG_CAP, 17, 2};    // Arbitration Mechanism Supported
+constexpr mmio_field64 NVME_CAP_CQR = {NVME_REG_CAP, 16, 1};    // Contiguous Queues Required
+constexpr mmio_field64 NVME_CAP_MQES = {NVME_REG_CAP, 0, 16};   // Maximum Queue Entries Supported
+
+constexpr mmio_reg32 NVME_REG_VS = {0x08, MMIO_LE};         // Version
+constexpr mmio_field32 NVME_VS_MJR = {NVME_REG_VS, 16, 16}; // Major Version Number
+constexpr mmio_field32 NVME_VS_MNR = {NVME_REG_VS, 8, 8};   // Minor Version Number
+constexpr mmio_field32 NVME_VS_TER = {NVME_REG_VS, 0, 8};   // Tertiary Version Number
+
+constexpr mmio_reg32 NVME_REG_INTMS = {0x0C, MMIO_LE}; // Interrupt Mask Set
+constexpr mmio_reg32 NVME_REG_INTMC = {0x10, MMIO_LE}; // Interrupt Mask Clear
+
+constexpr mmio_reg32 NVME_REG_CC = {0x14, MMIO_LE};           // Controller Configuration
+constexpr mmio_field32 NVME_CC_IOCQES = {NVME_REG_CC, 20, 4}; // I/O Completion Queue Entry Size
+constexpr mmio_field32 NVME_CC_IOSQES = {NVME_REG_CC, 16, 4}; // I/O Submission Queue Entry Size
+constexpr mmio_field32 NVME_CC_SHN = {NVME_REG_CC, 14, 2};    // Shutdown Notification
+constexpr mmio_field32 NVME_CC_AMS = {NVME_REG_CC, 11, 3};    // Arbitration Mechanism Selected
+constexpr mmio_field32 NVME_CC_MPS = {NVME_REG_CC, 7, 4};     // Memory Page Size
+constexpr mmio_field32 NVME_CC_CSS = {NVME_REG_CC, 4, 3};     // I/O Command Set Selected
+constexpr mmio_field32 NVME_CC_EN = {NVME_REG_CC, 0, 1};      // Enable
+
+constexpr mmio_reg32 NVME_REG_CSTS = {0x1C, MMIO_LE};           // Controller Status
+constexpr mmio_field32 NVME_CSTS_PP = {NVME_REG_CSTS, 5, 1};    // Processing Paused
+constexpr mmio_field32 NVME_CSTS_NSSRO = {NVME_REG_CSTS, 4, 1}; // NVM Subsystem Reset Occurred
+constexpr mmio_field32 NVME_CSTS_SHST = {NVME_REG_CSTS, 2, 2};  // Shutdown Status
+constexpr mmio_field32 NVME_CSTS_CFS = {NVME_REG_CSTS, 1, 1};   // Controller Fatal Status
+constexpr mmio_field32 NVME_CSTS_RDY = {NVME_REG_CSTS, 0, 1};   // Ready
+
+constexpr mmio_reg32 NVME_REG_AQA = {0x24, MMIO_LE}; // Admin Queue Attributes
+
+constexpr mmio_field32 NVME_AQA_ACQS = {NVME_REG_AQA, 0, 12};  // Admin Completion Queue Size
+constexpr mmio_field32 NVME_AQA_ASQS = {NVME_REG_AQA, 16, 12}; // Admin Submission Queue Size
+
+constexpr mmio_reg64 NVME_REG_ASQ = {0x28, MMIO_LE}; // Admin Submission Queue
+constexpr mmio_reg64 NVME_REG_ACQ = {0x30, MMIO_LE}; // Admin Completion Queue
+
+// Generic Command Set
+enum nvme_cmd : uint8_t {
+    NVME_CMD_FLUSH = 0x00,
+    NVME_CMD_WRITE = 0x01,
+    NVME_CMD_READ = 0x02,
+};
+
+// Admin Command Set
+enum nvme_admin_cmd : uint8_t {
+    NVME_ADMIN_DELETE_SQ = 0x00,
+    NVME_ADMIN_CREATE_SQ = 0x01,
+    NVME_ADMIN_DELETE_CQ = 0x04,
+    NVME_ADMIN_CREATE_CQ = 0x05,
+    NVME_ADMIN_IDENTIFY = 0x06,
+    NVME_ADMIN_ABORT = 0x08,
+    NVME_ADMIN_SET_FEATURES = 0x09,
+    NVME_ADMIN_GET_FEATURES = 0x0A,
+};

dev/block/partition.c

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <stdint.h>
+
+struct partition {
+    struct blockdev blk;
+    struct blockdev* parent;
+    uint64_t start;
+};
+
+static _Atomic(uint32_t) next_minor;
+
+static errno_t partition_submit(struct blockdev* blk, struct bio* bio) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    struct blockdev* parent = part->parent;
+
+    if (bio->op != BIO_FLUSH) {
+        if (bio->lba >= blk->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+        if (bio->op == BIO_WRITE && bio->num_lbas > blk->lba_count - bio->lba) {
+            bio_complete(bio, ENOSPC, 0);
+            return 0;
+        }
+        bio->lba += part->start;
+    }
+    return parent->ops->submit(parent, bio);
+}
+
+static errno_t partition_ioctl(
+    struct blockdev* blk,
+    struct vfs_file* file,
+    uintptr_t request,
+    uintptr_t arg,
+    uintptr_t* out
+) {
+    struct blockdev* parent = containerof(blk, struct partition, blk)->parent;
+    if (!parent->ops->ioctl)
+        return ENOTTY;
+    return parent->ops->ioctl(parent, file, request, arg, out);
+}
+
+static void partition_destroy(struct blockdev* blk) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    blockdev_unref(part->parent);
+    mem_free(part);
+}
+
+static const struct blockdev_ops partition_ops = {
+    .submit = partition_submit,
+    .ioctl = partition_ioctl,
+    .destroy = partition_destroy,
+};
+
+errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out) {
+    struct partition* part = mem_alloc(sizeof(struct partition), 0);
+    if (!part)
+        return ENOMEM;
+
+    blockdev_init(
+        &part->blk,
+        &partition_ops,
+        259,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        parent->lba_size,
+        lba_count
+    );
+    part->blk.limits = parent->limits;
+    blockdev_ref(parent);
+    part->parent = parent;
+    part->start = start_lba;
+    *out = &part->blk;
+    return 0;
+}

dev/block/ram.c

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+constexpr uint32_t RAM_MAJOR = 1;
+constexpr size_t RAM_LBA_SIZE = 512;
+
+struct ramdisk {
+    struct blockdev blk;
+    phys_t base;
+};
+
+static atomic_uint next_minor;
+
+static errno_t ramdisk_submit(struct blockdev* blk, struct bio* bio) {
+    struct ramdisk* ram = containerof(blk, struct ramdisk, blk);
+
+    if (bio->op == BIO_FLUSH) {
+        bio_complete(bio, 0, 0);
+        return 0;
+    }
+    if (bio->lba >= blk->lba_count || bio->num_lbas > blk->lba_count - bio->lba) {
+        bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+        return 0;
+    }
+
+    uint64_t lba = bio->lba;
+    for (size_t i = 0; i < bio->num_segs; i++) {
+        const struct bio_seg* seg = &bio->segs[i];
+        void* store = HHDM_PTR(ram->base + lba * RAM_LBA_SIZE);
+        void* buffer = HHDM_PTR(seg->addr);
+        if (bio->op == BIO_READ)
+            memcpy(buffer, store, seg->len);
+        else
+            memcpy(store, buffer, seg->len);
+        lba += seg->len / RAM_LBA_SIZE;
+    }
+
+    bio_complete(bio, 0, bio->num_lbas);
+    return 0;
+}
+
+static void ramdisk_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct ramdisk, blk));
+}
+
+static const struct blockdev_ops ramdisk_ops = {
+    .submit = ramdisk_submit,
+    .destroy = ramdisk_destroy,
+};
+
+errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out) {
+    struct ramdisk* ram = mem_alloc(sizeof(struct ramdisk), 0);
+    if (!ram)
+        return ENOMEM;
+
+    blockdev_init(
+        &ram->blk,
+        &ramdisk_ops,
+        RAM_MAJOR,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        RAM_LBA_SIZE,
+        length / RAM_LBA_SIZE
+    );
+    ram->base = base;
+    *out = &ram->blk;
+    return 0;
+}

dev/char/chardev.c

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+
+void chardev_init(
+    struct chardev* dev,
+    const struct chardev_ops* ops,
+    const struct vfs_file_ops* file_ops,
+    uint32_t major,
+    uint32_t minor
+) {
+    refcount_init(&dev->rc);
+    dev->ops = ops;
+    dev->file_ops = file_ops;
+    dev->dev = dev_make(major, minor);
+}
+
+void chardev_ref(struct chardev* dev) {
+    refcount_inc(&dev->rc);
+}
+
+void chardev_unref(struct chardev* dev) {
+    if (refcount_dec(&dev->rc) && dev->ops && dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static errno_t chardev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct chardev* dev = node->priv;
+    file->ops = dev->file_ops;
+    if (dev->ops && dev->ops->open)
+        return dev->ops->open(dev, file);
+    return 0;
+}
+
+static void chardev_node_destroy(struct vfs_node* node) {
+    chardev_unref(node->priv);
+}
+
+static const struct vfs_node_ops chardev_node_ops = {
+    .destroy = chardev_node_destroy,
+    .device = {
+        .open = chardev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_chardev(const char* name, struct chardev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_CHARDEV,
+        .ops = &chardev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    chardev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        chardev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t devtmpfs_mkdir(const char* name, mode_t mode) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mkdir(root, root, name, mode, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}
+
+errno_t devtmpfs_remove(const char* name) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_unlink(root, root, name, ident_kernel());
+    vfs_path_drop(root);
+    if (status)
+        return status;
+    devctl_notify_destroy(name);
+    return 0;
+}
+
+errno_t devtmpfs_symlink(const char* name, const char* target) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_symlink(root, root, name, target, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}

dev/char/devctl.c

@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/mutex.h>
+#include <zinnia/poll.h>
+#include <zinnia/tailq.h>
+#include <string.h>
+
+constexpr size_t DEVCTL_QUEUE_CAPACITY = 256;
+constexpr size_t DEVCTL_BACKLOG_CAPACITY = 256;
+
+struct devctl_line {
+    tailq_link(struct devctl_line) link;
+    size_t len;
+    char data[];
+};
+
+struct devctl_queue {
+    tailq_head(struct devctl_line) lines;
+    size_t count;
+};
+
+struct devctl_reader {
+    tailq_link(struct devctl_reader) link;
+    struct devctl_queue queue;
+};
+
+static mutex_t devctl_lock;
+static tailq_head(struct devctl_reader) devctl_readers;
+static struct devctl_queue devctl_backlog;
+static bool devctl_had_reader;
+static struct poll_event devctl_event;
+static atomic_bool devctl_ready;
+static struct chardev devctl_dev;
+
+static void queue_init(struct devctl_queue* queue) {
+    tailq_init(&queue->lines);
+    queue->count = 0;
+}
+
+static struct devctl_line* queue_pop(struct devctl_queue* queue) {
+    struct devctl_line* line = tailq_first(&queue->lines);
+    if (line) {
+        tailq_remove(&queue->lines, line, link);
+        queue->count--;
+    }
+    return line;
+}
+
+static void queue_push(struct devctl_queue* queue, struct devctl_line* line, size_t capacity) {
+    if (queue->count >= capacity)
+        mem_free(queue_pop(queue));
+    tailq_insert_tail(&queue->lines, line, link);
+    queue->count++;
+}
+
+static void queue_clear(struct devctl_queue* queue) {
+    struct devctl_line* line;
+    while ((line = queue_pop(queue)))
+        mem_free(line);
+}
+
+static struct devctl_line* line_new(const char* data, size_t len) {
+    struct devctl_line* line = mem_alloc(sizeof(*line) + len, 0);
+    if (!line)
+        return nullptr;
+    line->len = len;
+    memcpy(line->data, data, len);
+    return line;
+}
+
+static void devctl_notify(const char* type, const char* name) {
+    if (!atomic_load(&devctl_ready, ATOMIC_ACQUIRE))
+        return;
+
+    while (*name == '/')
+        name++;
+
+    const char prefix[] = "!system=DEVFS subsystem=CDEV type=";
+    const char middle[] = " cdev=";
+    const size_t prefix_len = sizeof(prefix) - 1;
+    const size_t type_len = strlen(type);
+    const size_t middle_len = sizeof(middle) - 1;
+    const size_t name_len = strlen(name);
+    const size_t len = prefix_len + type_len + middle_len + name_len + 1;
+
+    char* buf = mem_alloc(len, 0);
+    if (!buf)
+        return;
+    char* pos = buf;
+    memcpy(pos, prefix, prefix_len);
+    pos += prefix_len;
+    memcpy(pos, type, type_len);
+    pos += type_len;
+    memcpy(pos, middle, middle_len);
+    pos += middle_len;
+    memcpy(pos, name, name_len);
+    pos += name_len;
+    *pos = '\n';
+
+    mutex_lock(&devctl_lock);
+    if (tailq_empty(&devctl_readers)) {
+        if (!devctl_had_reader) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&devctl_backlog, line, DEVCTL_BACKLOG_CAPACITY);
+        }
+    } else {
+        for (struct devctl_reader* reader = tailq_first(&devctl_readers); reader; reader = tailq_next(reader, link)) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    mem_free(buf);
+
+    poll_event_notify(&devctl_event);
+}
+
+void devctl_notify_create(const char* name) {
+    devctl_notify("CREATE", name);
+}
+
+void devctl_notify_destroy(const char* name) {
+    devctl_notify("DESTROY", name);
+}
+
+static bool file_nonblocking(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_NONBLOCK;
+}
+
+static errno_t drain_lines(struct devctl_queue* queue, iovec_iter_t* buffer, size_t* out_total) {
+    size_t total = 0;
+    errno_t status = 0;
+    struct devctl_line* line;
+    while ((line = tailq_first(&queue->lines))) {
+        if (line->len > iovec_iter_len(buffer)) {
+            if (total == 0)
+                status = EINVAL;
+            break;
+        }
+        queue_pop(queue);
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, line->data, line->len, &copied);
+        mem_free(line);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_total = total;
+    return status;
+}
+
+static errno_t devctl_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct devctl_reader* reader = file->priv;
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &devctl_event);
+
+    errno_t status = 0;
+    size_t read = 0;
+    mutex_lock(&devctl_lock);
+    while (true) {
+        if (reader->queue.count) {
+            status = drain_lines(&reader->queue, buffer, &read);
+            break;
+        }
+        if (file_nonblocking(file)) {
+            status = EAGAIN;
+            break;
+        }
+        if (poll_table_sleep(&table, &devctl_lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    poll_table_fini(&table);
+
+    *out_read = read;
+    return status;
+}
+
+static errno_t devctl_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    return EBADF;
+}
+
+static enum poll_flags devctl_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    struct devctl_reader* reader = file->priv;
+
+    if (events & (POLLIN | POLLRDNORM))
+        poll_wait(table, &devctl_event);
+
+    enum poll_flags revents = 0;
+    mutex_lock(&devctl_lock);
+    if (reader->queue.count)
+        revents |= POLLIN | POLLRDNORM;
+    mutex_unlock(&devctl_lock);
+    return revents;
+}
+
+static void devctl_release(struct vfs_file* file) {
+    struct devctl_reader* reader = file->priv;
+
+    mutex_lock(&devctl_lock);
+    tailq_remove(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    queue_clear(&reader->queue);
+    mem_free(reader);
+}
+
+static const struct vfs_file_ops devctl_file_ops = {
+    .read = devctl_read,
+    .write = devctl_write,
+    .poll = devctl_poll,
+    .release = devctl_release,
+};
+
+static errno_t devctl_open(struct chardev* dev, struct vfs_file* file) {
+    struct devctl_reader* reader = mem_alloc(sizeof(*reader), 0);
+    if (!reader)
+        return ENOMEM;
+    queue_init(&reader->queue);
+
+    mutex_lock(&devctl_lock);
+    if (!devctl_had_reader) {
+        devctl_had_reader = true;
+        struct devctl_line* line;
+        while ((line = queue_pop(&devctl_backlog)))
+            queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+    }
+    tailq_insert_tail(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    file->priv = reader;
+    vfs_stream_open(file);
+    return 0;
+}
+
+static const struct chardev_ops devctl_dev_ops = {.open = devctl_open};
+
+static void devctl_stage_init(struct boot_info* info) {
+    mutex_init(&devctl_lock);
+    tailq_init(&devctl_readers);
+    queue_init(&devctl_backlog);
+    poll_event_init(&devctl_event);
+    atomic_store(&devctl_ready, true, ATOMIC_RELEASE);
+
+    chardev_init(&devctl_dev, &devctl_dev_ops, &devctl_file_ops, 5, 3);
+    errno_t status = devtmpfs_add_chardev("devctl", &devctl_dev, 0600);
+    assert(status == 0, "Unable to create /dev/devctl (%i)", status);
+}
+
+INIT_TASK(devctl_stage, "dev.devctl", devctl_stage_init);
+INIT_DEPENDS(devctl_stage, devtmpfs_stage);

dev/char/memfiles.c

@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/chardev.h>
+#include <zinnia/random.h>
+#include <zinnia/utils.h>
+
+constexpr size_t MEMFILE_CHUNK = 64;
+
+static const uint8_t zero_chunk[MEMFILE_CHUNK];
+
+static errno_t memfile_discard(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = iovec_iter_len(buffer);
+    return 0;
+}
+
+static errno_t memfile_read_zero(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, zero_chunk, MIN(iovec_iter_len(buffer), MEMFILE_CHUNK), &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static errno_t null_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    *out_read = 0;
+    return 0;
+}
+
+static const struct vfs_file_ops null_file_ops = {
+    .read = null_read,
+    .write = memfile_discard,
+};
+
+static const struct vfs_file_ops zero_file_ops = {
+    .read = memfile_read_zero,
+    .write = memfile_discard,
+};
+
+static errno_t full_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = 0;
+    return ENOSPC;
+}
+
+static const struct vfs_file_ops full_file_ops = {
+    .read = memfile_read_zero,
+    .write = full_write,
+};
+
+static errno_t random_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    uint8_t chunk[MEMFILE_CHUNK];
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        const size_t len = MIN(iovec_iter_len(buffer), sizeof(chunk));
+
+        // TODO: This function sucks and is probably not useful for cryptography.
+        random_fill(chunk, len);
+
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, chunk, len, &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static const struct vfs_file_ops random_file_ops = {
+    .read = random_read,
+    .write = memfile_discard,
+};
+
+static struct chardev null_dev;
+static struct chardev zero_dev;
+static struct chardev full_dev;
+static struct chardev random_dev;
+static struct chardev urandom_dev;
+
+static void memfiles_add(const char* name, struct chardev* dev, const struct vfs_file_ops* ops, uint32_t minor) {
+    chardev_init(dev, nullptr, ops, 1, minor);
+    errno_t status = devtmpfs_add_chardev(name, dev, 0666);
+    assert(status == 0, "Unable to create /dev/%s (%i)", name, status);
+}
+
+static void memfiles_stage_init(struct boot_info* info) {
+    memfiles_add("null", &null_dev, &null_file_ops, 3);
+    memfiles_add("zero", &zero_dev, &zero_file_ops, 5);
+    memfiles_add("full", &full_dev, &full_file_ops, 7);
+    memfiles_add("random", &random_dev, &random_file_ops, 8);
+    memfiles_add("urandom", &urandom_dev, &random_file_ops, 9);
+}
+
+INIT_TASK(memfiles_stage, "dev.memfiles", memfiles_stage_init);
+INIT_DEPENDS(memfiles_stage, devtmpfs_stage, random_stage);

dev/char/meson.build

@@ -1,3 +1,5 @@
 kernel_sources += files(
+    'chardev.c',
+    'devctl.c',
     'memfiles.c',
 )

dev/common/memfiles.c

@@ -1 +0,0 @@
-

dev/dt/driver.c

@@ -0,0 +1,96 @@
+#define pr_fmt(fmt) "fdt: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/fdt.h>
+#include <zinnia/init.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <string.h>
+
+static struct fdt tree;
+static bool tree_valid = false;
+
+static struct spinlock driver_lock = {};
+static vec(struct fdt_driver*) drivers = {};
+
+struct fdt* fdt_get() {
+    return tree_valid ? &tree : nullptr;
+}
+
+static const struct fdt_match* fdt_match_table(
+    const struct fdt_match* table,
+    size_t num_matches,
+    struct fdt_node node
+) {
+    for (size_t i = 0; i < num_matches; i++) {
+        if (fdt_node_is_compatible(node, table[i].compatible))
+            return &table[i];
+    }
+    return nullptr;
+}
+
+errno_t fdt_driver_register(struct fdt_driver* driver) {
+    if (driver->match == nullptr || driver->num_matches == 0) {
+        log_warn("Driver \"%s\" does not define any matches, ignoring\n", driver->name);
+        return 0;
+    }
+
+    if (!tree_valid)
+        return ENODEV;
+
+    spin_lock(&driver_lock);
+    vec_foreach(&drivers, i) {
+        if (strcmp(vec_elem_get(&drivers, i)->name, driver->name) == 0) {
+            spin_unlock(&driver_lock);
+            log_warn("Driver \"%s\" is already registered\n", driver->name);
+            return EEXIST;
+        }
+    }
+    errno_t status = vec_push(&drivers, driver);
+    spin_unlock(&driver_lock);
+    if (status != 0)
+        return status;
+
+    log("Registered platform driver \"%s\" with %zu match%s\n",
+        driver->name,
+        driver->num_matches,
+        driver->num_matches == 1 ? "" : "es");
+    for (size_t i = 0; i < driver->num_matches; i++)
+        log("  [%zu] = \"%s\"\n", i, driver->match[i].compatible);
+
+    for (struct fdt_node node = fdt_root(&tree); node.tree; node = fdt_next_node(node)) {
+        const struct fdt_match* match = fdt_match_table(driver->match, driver->num_matches, node);
+        if (match == nullptr)
+            continue;
+
+        log("Matched driver \"%s\" on rule [%zu]\n", driver->name, (size_t)(match - driver->match));
+
+        status = driver->probe(node, match);
+        if (status != 0)
+            log_warn("%s failed to probe: %i\n", driver->name, status);
+    }
+
+    return 0;
+}
+
+[[__init]]
+static bool fdt_stage_init(struct boot_info* info) {
+    if (!info->dtb)
+        return false;
+
+    const void* blob = HHDM_PTR(info->dtb);
+    if (!fdt_init(&tree, blob, info->dtb_size)) {
+        log_warn("Unable to parse the device tree blob\n");
+        return false;
+    }
+    tree_valid = true;
+
+    struct fdt_prop model = fdt_property(fdt_root(&tree), "model");
+    if (model.tree && model.length)
+        log("Running on \"%.*s\"\n", (int)model.length, (const char*)model.data);
+
+    return true;
+}
+INIT_TASK(fdt_stage, "dev.dt", fdt_stage_init);
+INIT_DEPENDS(fdt_stage, mem_stage);

dev/dt/fdt.c

@@ -212,24 +212,70 @@ bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value) {
     return true;
 }
 
-struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
-    if (!tree || !tree->blob || !compatible)
-        return (struct fdt_node){};
+struct fdt_node fdt_next_node(struct fdt_node node) {
     struct token token;
-    for (size_t offset = 0; read_token(tree, offset, &token); offset = token.next) {
-        if (token.kind != FDT_BEGIN_NODE)
-            continue;
-        struct fdt_node node = {tree, offset};
-        struct fdt_prop property = fdt_property(node, "compatible");
-        const char* data = property.data;
-        for (size_t i = 0; i < property.length;) {
-            size_t length = strnlen(data + i, property.length - i);
-            if (length == property.length - i)
-                break;
-            if (!strcmp(data + i, compatible))
-                return node;
-            i += length + 1;
-        }
+    if (!node.tree || !read_token(node.tree, node.offset, &token))
+        return (struct fdt_node){};
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
+        if (token.kind == FDT_BEGIN_NODE)
+            return (struct fdt_node){node.tree, offset};
+        if (token.kind == FDT_END)
+            break;
+    }
+    return (struct fdt_node){};
+}
+
+bool fdt_node_is_compatible(struct fdt_node node, const char* compatible) {
+    struct fdt_prop property = fdt_property(node, "compatible");
+    const char* data = property.data;
+    for (size_t i = 0; i < property.length;) {
+        size_t length = strnlen(data + i, property.length - i);
+        if (length == property.length - i)
+            break;
+        if (!strcmp(data + i, compatible))
+            return true;
+        i += length + 1;
+    }
+    return false;
+}
+
+static uint64_t read_cells(const uint8_t* data, uint32_t cells) {
+    uint64_t value = 0;
+    for (uint32_t i = 0; i < cells; i++)
+        value = value << 32 | fdt_read_u32(data + i * 4);
+    return value;
+}
+
+bool fdt_node_reg(struct fdt_node node, size_t index, phys_t* addr, size_t* size) {
+    if (!node.tree)
+        return false;
+    struct fdt_node root = fdt_root(node.tree);
+    uint32_t addr_cells = 2;
+    uint32_t size_cells = 2;
+    fdt_property_u32(root, "#address-cells", &addr_cells);
+    fdt_property_u32(root, "#size-cells", &size_cells);
+    if (addr_cells > 2 || size_cells > 2)
+        return false;
+
+    struct fdt_prop reg = fdt_property(node, "reg");
+    const size_t stride = (addr_cells + size_cells) * 4;
+    if (!reg.tree || index >= reg.length / stride)
+        return false;
+
+    const uint8_t* entry = (const uint8_t*)reg.data + index * stride;
+    if (addr)
+        *addr = read_cells(entry, addr_cells);
+    if (size)
+        *size = read_cells(entry + addr_cells * 4, size_cells);
+    return true;
+}
+
+struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
+    if (!tree || !tree->blob || !compatible)
+        return (struct fdt_node){};
+    for (struct fdt_node node = fdt_root(tree); node.tree; node = fdt_next_node(node)) {
+        if (fdt_node_is_compatible(node, compatible))
+            return node;
     }
     return (struct fdt_node){};
 }

dev/dt/meson.build

@@ -1 +1,4 @@
-kernel_sources += files('fdt.c')
+kernel_sources += files(
+    'driver.c',
+    'fdt.c',
+)

dev/meson.build

@@ -1,5 +1,5 @@
-subdir('common')
 subdir('block')
+subdir('char')
 subdir('dt')
 subdir('video')
 

dev/pci/core.c

@@ -1,12 +1,21 @@
+// SPDX-License-Identifier: ISC
+
 #define pr_fmt(fmt) "pci: " fmt
 
 #include <zinnia/alloc.h>
 #include <zinnia/init.h>
+#include <zinnia/mmio.h>
 #include <zinnia/module.h>
 #include <zinnia/pci.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/spin.h>
+#include <zinnia/utils.h>
 #include <zinnia/vec.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
 #include <string.h>
 
 static struct spinlock access_lock = {};
@@ -168,6 +177,119 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) {
     return 0;
 }
 
+constexpr uint32_t PCI_MSI_CTRL_ENABLE = 1 << 16;
+constexpr uint32_t PCI_MSI_CTRL_MME = 0x7 << 20;
+constexpr uint32_t PCI_MSI_CTRL_64BIT = 1 << 23;
+constexpr uint32_t PCI_MSIX_CTRL_FUNCTION_MASK = 1 << 30;
+constexpr uint32_t PCI_MSIX_CTRL_ENABLE = 1u << 31;
+
+static void msix_set_state(struct pci_device* dev, uint8_t cap, bool status) {
+    uint32_t ctrl = pci_read32(dev, cap);
+    ctrl &= ~(PCI_MSIX_CTRL_ENABLE | PCI_MSIX_CTRL_FUNCTION_MASK);
+    if (status)
+        ctrl |= PCI_MSIX_CTRL_ENABLE;
+    pci_write32(dev, cap, ctrl);
+}
+
+errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out) {
+    // message address + data + vector control
+    constexpr size_t ENTRY_SIZE = 8 + 4 + 4;
+
+    const uint8_t cap = pci_cap_find(dev, PCI_CAP_MSIX);
+    if (cap == 0)
+        return ENODEV;
+
+    const uint32_t table = pci_read32(dev, cap + 4);
+    const uint8_t bir = table & 0x7;
+    const size_t table_offset = table & ~0x7;
+
+    pci_log(dev, "Using MSI-X for this device, table at BAR%hhu + %#zx\n", bir, table_offset);
+
+    // Map the first MSI-X table entry (16 bytes).
+    struct pci_bar bar;
+    errno_t status = pci_bar_get(dev, bir, &bar);
+    if (status)
+        return ENODEV;
+    if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
+        return EINVAL;
+
+    const size_t page_size = arch_mem_page_size();
+    const phys_t table_phys = bar.address + table_offset;
+    const phys_t page_base = ALIGN_DOWN(table_phys, page_size);
+    const size_t page_offset = table_phys - page_base;
+    const size_t map_len = ALIGN_UP(page_offset + ENTRY_SIZE, page_size);
+
+    uintptr_t vaddr = vm_alloc(map_len);
+    if (!vaddr)
+        return ENOMEM;
+    for (size_t i = 0; i < map_len; i += page_size) {
+        status = pmap_map(&kernel_vma.pmap, vaddr + i, page_base + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status)
+            return status;
+    }
+    void* entry = (void*)(vaddr + page_offset);
+
+    struct msi_line* msi;
+    status = arch_irq_alloc_msi(&msi);
+    if (status)
+        return status;
+    irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
+
+    // Write entry 0.
+    const uint64_t msg_addr = msi->msg_addr(msi);
+    const uint32_t msg_data = msi->msg_data(msi);
+    mmio_write32(entry + 0x00, (uint32_t)msg_addr);         // Address lo
+    mmio_write32(entry + 0x04, (uint32_t)(msg_addr >> 32)); // Address hi
+    mmio_write32(entry + 0x08, msg_data);                   // Data
+    mmio_write32(entry + 0x0C, 0);                          // Vector control
+
+    // Enable MSI-X on the device.
+    msix_set_state(dev, cap, true);
+
+    *out = &msi->line;
+    return 0;
+}
+
+errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out) {
+    uint8_t cap = pci_cap_find(dev, PCI_CAP_MSI);
+    if (cap == 0)
+        return ENODEV;
+
+    pci_log(dev, "Using MSI for this device\n");
+
+    struct msi_line* msi;
+    errno_t status = arch_irq_alloc_msi(&msi);
+    if (status)
+        return status;
+    irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
+
+    const uint64_t addr = msi->msg_addr(msi);
+    const uint16_t data = msi->msg_data(msi);
+
+    uint32_t ctrl = pci_read32(dev, cap);
+    pci_write32(dev, cap + 0x04, (uint32_t)addr);
+    uint16_t data_off = 0x08;
+    if (ctrl & PCI_MSI_CTRL_64BIT) {
+        pci_write32(dev, cap + 0x08, (uint32_t)(addr >> 32));
+        data_off = 0x0C;
+    }
+    pci_write32(dev, cap + data_off, data);
+
+    ctrl &= ~PCI_MSI_CTRL_MME;
+    ctrl |= PCI_MSI_CTRL_ENABLE;
+    pci_write32(dev, cap, ctrl);
+
+    *out = &msi->line;
+    return 0;
+}
+
+errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out) {
+    errno_t status = pci_setup_msix(dev, out);
+    if (status == ENODEV)
+        return pci_setup_msi(dev, out);
+    return status;
+}
+
 static void pci_device_name(const struct pci_device* dev, char buf[static 18]) {
     ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
 }
@@ -190,6 +312,7 @@ static bool scan_function(struct pci_access* access, struct pci_address addr) {
 
     dev->address = addr;
     dev->access = access;
+    dev->dma_tag = &dma_default_tag;
     dev->id.vendor_id = vendor_id;
     dev->id.device_id = device_id;
     dev->class.prog_if = pci_read8(dev, PCI_PROG_IF);
@@ -373,4 +496,3 @@ static void pci_stage_init(struct boot_info*) {
 }
 INIT_TASK(pci_stage, "dev.pci", pci_stage_init);
 INIT_DEPENDS(pci_stage, mem_stage);
-INIT_ENTAILS(pci_stage, module_stage);

dev/video/fbcon/fbcon.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/console.h>