*: Add SPDX header to all files

Kconfig

@@ -5,5 +5,6 @@ config KERNEL_FLAVOR
   default "vanilla"
 
 source "arch/$(ARCH)/Kconfig"
-source "core/Kconfig"
+source "mem/Kconfig"
+source "sched/Kconfig"
 source "dev/Kconfig"

arch/x86_64/Kconfig

@@ -6,6 +6,7 @@ config X86_64
   select ACPI
   select HAVE_MMU
   select HAVE_MMU_GENERIC
+  select HAVE_TLB_SHOOTDOWN_IPI
   help
     64-bit x86 CPU support for AMD and Intel CPUs.
 

arch/x86_64/acpi.c

@@ -0,0 +1,36 @@
+#include <stdckdint.h>
+#include <stdint.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <x86_64/asm.h>
+
+uacpi_status uacpi_kernel_io_map(uacpi_io_addr base, uacpi_size len, uacpi_handle* out_handle) {
+    uintptr_t end;
+    if (ckd_add(&end, base, len) || end > 0x10000)
+        return UACPI_STATUS_INVALID_ARGUMENT;
+    *out_handle = (uacpi_handle)(uintptr_t)base;
+    return UACPI_STATUS_OK;
+}
+
+void uacpi_kernel_io_unmap(uacpi_handle handle) {
+    // Left intentionally blank.
+}
+
+#define IO_READ(width, suffix) \
+    uacpi_status uacpi_kernel_io_read##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width* out_value) { \
+        *out_value = asm_in##suffix((uintptr_t)handle + offset); \
+        return UACPI_STATUS_OK; \
+    }
+
+#define IO_WRITE(width, suffix) \
+    uacpi_status uacpi_kernel_io_write##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width in_value) { \
+        asm_out##suffix((uintptr_t)handle + offset, in_value); \
+        return UACPI_STATUS_OK; \
+    }
+
+IO_READ(8, b)
+IO_READ(16, w)
+IO_READ(32, l)
+IO_WRITE(8, b)
+IO_WRITE(16, w)
+IO_WRITE(32, l)

arch/x86_64/apic.c

@@ -4,37 +4,50 @@
 #include <zinnia/mmio.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
+#include <zinnia/utils.h>
 #include <zinnia/vma.h>
+#include <arch/cpu.h>
+#include <arch/timer.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
-static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
+static uint32_t lapic_msr(size_t offset) {
+    return 0x800 + (offset >> 4);
+}
+
+static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
+    if (lapic->xapic_regs)
+        return mmio_reg_read(lapic->xapic_regs, reg);
+    else
+        return asm_rdmsr(lapic_msr(reg.offset));
+}
+
+static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) {
+    if (lapic->xapic_regs)
+        mmio_reg_write(lapic->xapic_regs, reg, value);
+    else
+        asm_wrmsr(lapic_msr(reg.offset), value);
+}
+
+static uint64_t lapic_read_icr(struct local_apic* lapic) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
-            uint64_t lo, hi;
-            lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR);
-            hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI);
-            return (hi << 32) | lo;
-        } else {
-            return mmio_read32(lapic->xapic_regs + reg);
-        }
+        uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO);
+        uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI);
+        return (hi << 32) | lo;
     } else {
-        return asm_rdmsr(0x800 + (reg >> 4));
+        return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset));
     }
 }
 
-static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
+static void lapic_write_icr(struct local_apic* lapic, uint64_t value) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32);
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value);
-        } else {
-            mmio_write32(lapic->xapic_regs + reg, (uint32_t)value);
-        }
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32));
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value);
     } else {
-        asm_wrmsr(0x800 + (reg >> 4), value);
+        asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value);
     }
 }
 
@@ -49,19 +62,22 @@ void lapic_init(struct local_apic* lapic) {
         uintptr_t vaddr = vm_alloc(0x1000);
         pmap_map(&kernel_vma.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
         lapic->xapic_regs = (void*)vaddr;
-        assert(lapic->xapic_regs, "Failed to allocate virtual memory!\n");
+        assert(lapic->xapic_regs, "Failed to allocate virtual memory!");
     }
 
     asm_wrmsr(0x1B, apic_msr);
 
     lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
     if (lapic->xapic_regs)
-        lapic->id >>= 24;
+        lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id);
 
     // Reset the TPR.
     lapic_write_reg(lapic, APIC_REG_TPR, 0);
     // Enable APIC bit in the SIVR.
-    lapic_write_reg(lapic, APIC_REG_SIVR, (lapic_read_reg(lapic, APIC_REG_SIVR) & ~0xFF) | 0x100 | IDT_SPURIOUS);
+    uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR);
+    sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS);
+    sivr |= mmio_field(APIC_SIVR_ENABLE, 1);
+    lapic_write_reg(lapic, APIC_REG_SIVR, sivr);
 
     if (lapic->xapic_regs) {
         lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
@@ -80,8 +96,16 @@ void lapic_init(struct local_apic* lapic) {
     clock_spin_ns(10'000'000);
 
     lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
-    lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
-    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
+    lapic_write_reg(lapic, APIC_REG_LVT_TR, mmio_field(APIC_LVT_TR_VECTOR, IDT_TIMER));
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0);
+}
+
+void arch_timer_program(uint64_t deadline_ns) {
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+    const uint64_t now = clock_get_elapsed_ns();
+    const uint64_t delta = MIN(deadline_ns > now ? deadline_ns - now : 0, 1'000'000'000);
+    const uint64_t ticks = delta * lapic->ticks_per_10ms / 10'000'000;
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, MIN(MAX(ticks, 1), 0xFFFF'FFFF));
 }
 
 void lapic_eoi(struct local_apic* lapic) {
@@ -97,22 +121,22 @@ void lapic_send_ipi(
     enum apic_level level,
     enum apic_trigger_mode trigger_mode
 ) {
-    uint64_t icr = vector;
-    icr |= (uint64_t)delivery_mode << 8;
-    icr |= (uint64_t)dest_mode << 11;
-    icr |= (uint64_t)level << 14;
-    icr |= (uint64_t)trigger_mode << 15;
+    uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector);
+    icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode);
+    icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode);
+    icr |= mmio_field(APIC_ICR_LEVEL, level);
+    icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode);
 
     if (lapic->xapic_regs)
-        icr |= (uint64_t)(uint8_t)target << 56;
+        icr |= mmio_field(APIC_ICR_DEST_XAPIC, target);
     else
-        icr |= (uint64_t)target << 32;
+        icr |= mmio_field(APIC_ICR_DEST_X2APIC, target);
 
-    lapic_write_reg(lapic, APIC_REG_ICR, icr);
+    lapic_write_icr(lapic, icr);
 
     if (lapic->xapic_regs) {
-        while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12))
-            asm volatile("pause");
+        while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING)
+            arch_cpu_pause();
     }
 }
 

arch/x86_64/archctl.c

@@ -12,21 +12,21 @@ struct syscall_ret arch_archctl(uintptr_t op, uintptr_t arg) {
     switch (op) {
     case __ARCHCTL_SET_FSBASE:
         if (!arch_mem_addr_is_user(arg)) {
-            ret.errno = EINVAL;
+            ret.error = EINVAL;
             return ret;
         }
         asm_wrmsr(MSR_FS_BASE, arg);
         return ret;
     case __ARCHCTL_SET_GSBASE:
         if (!arch_mem_addr_is_user(arg)) {
-            ret.errno = EINVAL;
+            ret.error = EINVAL;
             return ret;
         }
         // Update the inactive GSBASE MSR, not ours.
         asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
         return ret;
     default:
-        ret.errno = EINVAL;
+        ret.error = EINVAL;
         return ret;
     }
 }

arch/x86_64/gdt.c

@@ -1,6 +1,6 @@
 #include <zinnia/percpu.h>
-#include <zinnia/string.h>
 #include <stddef.h>
+#include <string.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>

arch/x86_64/hpet.c

@@ -1,7 +1,10 @@
+#include <zinnia/acpi.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/utils.h>
@@ -23,18 +26,20 @@ struct hpet {
     uint32_t period;
 };
 
-constexpr uint32_t HPET_CAP = 0x0;
-constexpr uint32_t HPET_CFG = 0x10;
-constexpr uint32_t HPET_COUNTER = 0xF0;
+constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0};
+constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32};
+constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10};
+constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1};
+constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0};
 
 static void hpet_reset(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    mmio_write64(hpet->regs + HPET_COUNTER, 0);
+    mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0);
 }
 
 static uint64_t hpet_get_elapsed_ns(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
+    uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER);
     return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
 }
 
@@ -47,15 +52,15 @@ static struct hpet hpet_clock = {
     },
 };
 
-static char table_buf[0x1000];
-
-void hpet_init() {
+static void hpet_init(struct boot_info*) {
     if (!use_hpet)
         return;
 
-    uacpi_setup_early_table_access(table_buf, sizeof(table_buf));
     uacpi_table table = {0};
-    assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
+    if (uacpi_table_find_by_signature("HPET", &table) != UACPI_STATUS_OK) {
+        log("x86_64/hpet: No HPET table found\n");
+        return;
+    }
 
     struct acpi_hpet* hpet = table.ptr;
 
@@ -66,10 +71,15 @@ void hpet_init() {
     uacpi_table_unref(&table);
     log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
 
-    mmio_write64(hpet_clock.regs + HPET_CFG, mmio_read64(hpet_clock.regs + HPET_CFG) | 1);
+    uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG);
+    cfg |= mmio_field(HPET_CFG_ENABLE, 1);
+    mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg);
 
-    uint64_t caps = mmio_read64(hpet_clock.regs + HPET_CAP);
-    hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
+    uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP);
+    hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap);
 
-    assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");
+    assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!");
 }
+INIT_TASK(hpet_stage, "arch.x86_64.hpet", hpet_init);
+INIT_DEPENDS(hpet_stage, acpi_stage);
+INIT_ENTAILS(hpet_stage, percpu_stage);

arch/x86_64/idt.c

@@ -1,10 +1,14 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/irq.h>
 #include <zinnia/panic.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
+#include <zinnia/timer.h>
 #include <zinnia/usercopy.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
@@ -93,12 +97,25 @@ void interrupt_handler(struct arch_context* ctx) {
         if (arch_sched_preempt_enable())
             sched_reschedule(&cpu->sched);
         break;
+    case IDT_IPI_SHOOTDOWN:
+        lapic_eoi(&arch_cpu->lapic);
+        pmap_shootdown_handle();
+        break;
+    case IDT_TIMER:
+        lapic_eoi(&arch_cpu->lapic);
+        timer_interrupt();
+        if (cpu->sched.need_resched) {
+            arch_sched_preempt_disable();
+            if (arch_sched_preempt_enable())
+                sched_reschedule(&cpu->sched);
+        }
+        break;
     default:
         if (isr < 0x20)
             panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
-        assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu\n", isr);
+        assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
         struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
-        assert(line, "Unhandled interrupt on ISR %lu\n", isr);
+        assert(line, "Unhandled interrupt on ISR %lu", isr);
         irq_line_raise(line);
     }
 }
@@ -240,6 +257,7 @@ static struct idt_entry idt[256] = {};
 
 void idt_init() {
     REPEAT_256(INTERRUPT_SETUP)
+    idt[IDT_BP].attrib |= CPL_USER << 5;
 }
 
 void idt_load() {

arch/x86_64/include/bits/cpu.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+static inline void arch_cpu_pause() {
+    asm volatile("pause");
+}
+
+#define ARCH_CACHE_LINE_SIZE 64

arch/x86_64/include/bits/elf.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/elf.h>
@@ -11,19 +13,21 @@
 #define ELF_ST_BIND ELF64_ST_BIND
 #define ELF_ST_TYPE ELF64_ST_TYPE
 
-#define elf_ehdr elf64_ehdr
-#define elf_phdr elf64_phdr
-#define elf_dyn  elf64_dyn
-#define elf_addr elf64_addr
-#define elf_off  elf64_off
-#define elf_nhdr elf64_nhdr
-#define elf_auxv elf64_auxv
-#define elf_sym  elf64_sym
-#define elf_rela elf64_rela
-#define elf_rel  elf64_rel
+typedef elf64_ehdr_t elf_ehdr_t;
+typedef elf64_phdr_t elf_phdr_t;
+typedef elf64_dyn_t elf_dyn_t;
+typedef elf64_addr_t elf_addr_t;
+typedef elf64_off_t elf_off_t;
+typedef elf64_nhdr_t elf_nhdr_t;
+typedef elf64_auxv_t elf_auxv_t;
+typedef elf64_sym_t elf_sym_t;
+typedef elf64_rela_t elf_rela_t;
+typedef elf64_rel_t elf_rel_t;
+typedef elf64_relr_t elf_relr_t;
 
 enum {
     R_NONE = 0,
+    R_64 = 1,
     R_COPY = 5,
     R_GLOB_DAT = 6,
     R_JUMP_SLOT = 7,

arch/x86_64/include/bits/pmap.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/include/bits/sched.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/types.h>
@@ -37,6 +39,7 @@ struct arch_context {
 };
 
 #define ARCH_CTX_NUM  rax
+#define ARCH_CTX_SP   rsp
 #define ARCH_CTX_A0   rdi
 #define ARCH_CTX_A1   rsi
 #define ARCH_CTX_A2   rdx
@@ -45,3 +48,8 @@ struct arch_context {
 #define ARCH_CTX_A5   r9
 #define ARCH_CTX_RET0 rax
 #define ARCH_CTX_RET1 rdx
+
+struct arch_syscall_restart {
+    uint64_t rax;
+    uint64_t rdx;
+};

arch/x86_64/include/bits/types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/include/x86_64/apic.h

@@ -1,5 +1,8 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/mmio.h>
 #include <stdint.h>
 
 struct local_apic {
@@ -37,21 +40,32 @@ enum apic_trigger_mode {
     APIC_TRIGGER_LEVEL = 1,
 };
 
-enum apic_regs {
-    APIC_REG_ID = 0x20,
-    APIC_REG_TPR = 0x80,
-    APIC_REG_EOI = 0xB0,
-    APIC_REG_LDR = 0xD0,
-    APIC_REG_DFR = 0xE0,
-    APIC_REG_SIVR = 0xF0,
-    APIC_REG_ESR = 0x280,
-    APIC_REG_ICR = 0x300,
-    APIC_REG_ICR_HI = 0x310,
-    APIC_REG_LVT_TR = 0x320,
-    APIC_REG_ICR_TIMER = 0x380,
-    APIC_REG_CCR = 0x390,
-    APIC_REG_DCR = 0x3E0,
-};
+constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
+constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8};
+constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
+constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
+constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
+constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
+constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
+constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8};
+constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1};
+constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
+constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300};
+constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8};
+constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3};
+constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1};
+constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1};
+constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1};
+constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1};
+constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8};
+constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32};
+constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300};
+constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
+constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
+constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8};
+constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
+constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
+constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
 
 void lapic_init(struct local_apic* lapic);
 void lapic_eoi(struct local_apic* lapic);

arch/x86_64/include/x86_64/asm.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
@@ -40,6 +42,12 @@ static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
     asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory");
 }
 
+static inline uint64_t asm_rdxcr(uint32_t reg) {
+    uint32_t eax, edx;
+    asm volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(reg));
+    return ((uint64_t)edx << 32) | eax;
+}
+
 // Saves the FPU state to a 512-byte region of memory using FXSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_fxsave(void* mem) {

arch/x86_64/include/x86_64/defs.h

@@ -1,210 +1,257 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#define CPUID_1C_SSE3       (1 << 0)
-#define CPUID_1C_PCLMUL     (1 << 1)
-#define CPUID_1C_DTES64     (1 << 2)
-#define CPUID_1C_MONITOR    (1 << 3)
-#define CPUID_1C_DS_CPL     (1 << 4)
-#define CPUID_1C_VMX        (1 << 5)
-#define CPUID_1C_SMX        (1 << 6)
-#define CPUID_1C_EST        (1 << 7)
-#define CPUID_1C_TM2        (1 << 8)
-#define CPUID_1C_SSSE3      (1 << 9)
-#define CPUID_1C_CID        (1 << 10)
-#define CPUID_1C_SDBG       (1 << 11)
-#define CPUID_1C_FMA        (1 << 12)
-#define CPUID_1C_CX16       (1 << 13)
-#define CPUID_1C_XTPR       (1 << 14)
-#define CPUID_1C_PDCM       (1 << 15)
-#define CPUID_1C_PCID       (1 << 17)
-#define CPUID_1C_DCA        (1 << 18)
-#define CPUID_1C_SSE4_1     (1 << 19)
-#define CPUID_1C_SSE4_2     (1 << 20)
-#define CPUID_1C_X2APIC     (1 << 21)
-#define CPUID_1C_MOVBE      (1 << 22)
-#define CPUID_1C_POPCNT     (1 << 23)
-#define CPUID_1C_TSC        (1 << 24)
-#define CPUID_1C_AES        (1 << 25)
-#define CPUID_1C_XSAVE      (1 << 26)
-#define CPUID_1C_OSXSAVE    (1 << 27)
-#define CPUID_1C_AVX        (1 << 28)
-#define CPUID_1C_F16C       (1 << 29)
-#define CPUID_1C_RDRAND     (1 << 30)
-#define CPUID_1C_HYPERVISOR (1 << 31)
+#include <stdint.h>
 
-#define CPUID_1D_FPU     (1 << 0)
-#define CPUID_1D_VME     (1 << 1)
-#define CPUID_1D_DE      (1 << 2)
-#define CPUID_1D_PSE     (1 << 3)
-#define CPUID_1D_TSC     (1 << 4)
-#define CPUID_1D_MSR     (1 << 5)
-#define CPUID_1D_PAE     (1 << 6)
-#define CPUID_1D_MCE     (1 << 7)
-#define CPUID_1D_CX8     (1 << 8)
-#define CPUID_1D_APIC    (1 << 9)
-#define CPUID_1D_SEP     (1 << 11)
-#define CPUID_1D_MTRR    (1 << 12)
-#define CPUID_1D_PGE     (1 << 13)
-#define CPUID_1D_MCA     (1 << 14)
-#define CPUID_1D_CMOV    (1 << 15)
-#define CPUID_1D_PAT     (1 << 16)
-#define CPUID_1D_PSE36   (1 << 17)
-#define CPUID_1D_PSN     (1 << 18)
-#define CPUID_1D_CLFLUSH (1 << 19)
-#define CPUID_1D_DS      (1 << 21)
-#define CPUID_1D_ACPI    (1 << 22)
-#define CPUID_1D_MMX     (1 << 23)
-#define CPUID_1D_FXSR    (1 << 24)
-#define CPUID_1D_SSE     (1 << 25)
-#define CPUID_1D_SSE2    (1 << 26)
-#define CPUID_1D_SS      (1 << 27)
-#define CPUID_1D_HTT     (1 << 28)
-#define CPUID_1D_TM      (1 << 29)
-#define CPUID_1D_IA64    (1 << 30)
-#define CPUID_1D_PBE     (1 << 31)
+enum : uint32_t {
+    CPUID_1C_SSE3 = 1u << 0,
+    CPUID_1C_PCLMUL = 1u << 1,
+    CPUID_1C_DTES64 = 1u << 2,
+    CPUID_1C_MONITOR = 1u << 3,
+    CPUID_1C_DS_CPL = 1u << 4,
+    CPUID_1C_VMX = 1u << 5,
+    CPUID_1C_SMX = 1u << 6,
+    CPUID_1C_EST = 1u << 7,
+    CPUID_1C_TM2 = 1u << 8,
+    CPUID_1C_SSSE3 = 1u << 9,
+    CPUID_1C_CID = 1u << 10,
+    CPUID_1C_SDBG = 1u << 11,
+    CPUID_1C_FMA = 1u << 12,
+    CPUID_1C_CX16 = 1u << 13,
+    CPUID_1C_XTPR = 1u << 14,
+    CPUID_1C_PDCM = 1u << 15,
+    CPUID_1C_PCID = 1u << 17,
+    CPUID_1C_DCA = 1u << 18,
+    CPUID_1C_SSE4_1 = 1u << 19,
+    CPUID_1C_SSE4_2 = 1u << 20,
+    CPUID_1C_X2APIC = 1u << 21,
+    CPUID_1C_MOVBE = 1u << 22,
+    CPUID_1C_POPCNT = 1u << 23,
+    CPUID_1C_TSC = 1u << 24,
+    CPUID_1C_AES = 1u << 25,
+    CPUID_1C_XSAVE = 1u << 26,
+    CPUID_1C_OSXSAVE = 1u << 27,
+    CPUID_1C_AVX = 1u << 28,
+    CPUID_1C_F16C = 1u << 29,
+    CPUID_1C_RDRAND = 1u << 30,
+    CPUID_1C_HYPERVISOR = 1u << 31,
+};
 
-#define CPUID_7B_FSGSBASE   (1 << 0)
-#define CPUID_7B_SGX        (1 << 2)
-#define CPUID_7B_BMI        (1 << 3)
-#define CPUID_7B_HLE        (1 << 4)
-#define CPUID_7B_AVX2       (1 << 5)
-#define CPUID_7B_SMEP       (1 << 7)
-#define CPUID_7B_BMI2       (1 << 8)
-#define CPUID_7B_RTM        (1 << 11)
-#define CPUID_7B_AVX512F    (1 << 16)
-#define CPUID_7B_AVX512DQ   (1 << 17)
-#define CPUID_7B_RDSEED     (1 << 18)
-#define CPUID_7B_ADX        (1 << 19)
-#define CPUID_7B_SMAP       (1 << 20)
-#define CPUID_7B_AVX512IFMA (1 << 21)
-#define CPUID_7B_CLFLUSHOPT (1 << 23)
-#define CPUID_7B_CLWB       (1 << 24)
-#define CPUID_7B_AVX512CD   (1 << 28)
-#define CPUID_7B_SHA        (1 << 29)
-#define CPUID_7B_AVX512BW   (1 << 30)
-#define CPUID_7B_AVX512VL   (1 << 31)
+enum : uint32_t {
+    CPUID_1D_FPU = 1u << 0,
+    CPUID_1D_VME = 1u << 1,
+    CPUID_1D_DE = 1u << 2,
+    CPUID_1D_PSE = 1u << 3,
+    CPUID_1D_TSC = 1u << 4,
+    CPUID_1D_MSR = 1u << 5,
+    CPUID_1D_PAE = 1u << 6,
+    CPUID_1D_MCE = 1u << 7,
+    CPUID_1D_CX8 = 1u << 8,
+    CPUID_1D_APIC = 1u << 9,
+    CPUID_1D_SEP = 1u << 11,
+    CPUID_1D_MTRR = 1u << 12,
+    CPUID_1D_PGE = 1u << 13,
+    CPUID_1D_MCA = 1u << 14,
+    CPUID_1D_CMOV = 1u << 15,
+    CPUID_1D_PAT = 1u << 16,
+    CPUID_1D_PSE36 = 1u << 17,
+    CPUID_1D_PSN = 1u << 18,
+    CPUID_1D_CLFLUSH = 1u << 19,
+    CPUID_1D_DS = 1u << 21,
+    CPUID_1D_ACPI = 1u << 22,
+    CPUID_1D_MMX = 1u << 23,
+    CPUID_1D_FXSR = 1u << 24,
+    CPUID_1D_SSE = 1u << 25,
+    CPUID_1D_SSE2 = 1u << 26,
+    CPUID_1D_SS = 1u << 27,
+    CPUID_1D_HTT = 1u << 28,
+    CPUID_1D_TM = 1u << 29,
+    CPUID_1D_IA64 = 1u << 30,
+    CPUID_1D_PBE = 1u << 31,
+};
 
-#define CPUID_7C_AVX512VBMI      (1 << 1)
-#define CPUID_7C_UMIP            (1 << 2)
-#define CPUID_7C_PKU             (1 << 3)
-#define CPUID_7C_OSPKE           (1 << 4)
-#define CPUID_7C_WAITPKG         (1 << 5)
-#define CPUID_7C_AVX512VBMI2     (1 << 6)
-#define CPUID_7C_SHSTK           (1 << 7)
-#define CPUID_7C_GFNI            (1 << 8)
-#define CPUID_7C_VAES            (1 << 9)
-#define CPUID_7C_VPCLMULQDQ      (1 << 10)
-#define CPUID_7C_AVX512VNNI      (1 << 11)
-#define CPUID_7C_AVX512BITALG    (1 << 12)
-#define CPUID_7C_TME_EN          (1 << 13)
-#define CPUID_7C_AVX512VPOPCNTDQ (1 << 14)
-#define CPUID_7C_RDPID           (1 << 22)
-#define CPUID_7C_KL              (1 << 23)
-#define CPUID_7C_CLDEMOTE        (1 << 25)
-#define CPUID_7C_MOVDIRI         (1 << 27)
-#define CPUID_7C_MOVDIR64B       (1 << 28)
-#define CPUID_7C_ENQCMD          (1 << 29)
+enum : uint32_t {
+    CPUID_7B_FSGSBASE = 1u << 0,
+    CPUID_7B_SGX = 1u << 2,
+    CPUID_7B_BMI = 1u << 3,
+    CPUID_7B_HLE = 1u << 4,
+    CPUID_7B_AVX2 = 1u << 5,
+    CPUID_7B_SMEP = 1u << 7,
+    CPUID_7B_BMI2 = 1u << 8,
+    CPUID_7B_RTM = 1u << 11,
+    CPUID_7B_AVX512F = 1u << 16,
+    CPUID_7B_AVX512DQ = 1u << 17,
+    CPUID_7B_RDSEED = 1u << 18,
+    CPUID_7B_ADX = 1u << 19,
+    CPUID_7B_SMAP = 1u << 20,
+    CPUID_7B_AVX512IFMA = 1u << 21,
+    CPUID_7B_CLFLUSHOPT = 1u << 23,
+    CPUID_7B_CLWB = 1u << 24,
+    CPUID_7B_AVX512CD = 1u << 28,
+    CPUID_7B_SHA = 1u << 29,
+    CPUID_7B_AVX512BW = 1u << 30,
+    CPUID_7B_AVX512VL = 1u << 31,
+};
 
-#define CPUID_7D_UINTR              (1 << 5)
-#define CPUID_7D_AVX512VP2INTERSECT (1 << 8)
-#define CPUID_7D_SERIALIZE          (1 << 14)
-#define CPUID_7D_TSXLDTRK           (1 << 16)
-#define CPUID_7D_PCONFIG            (1 << 18)
-#define CPUID_7D_IBT                (1 << 20)
-#define CPUID_7D_AMX_BF16           (1 << 22)
-#define CPUID_7D_AVX512FP16         (1 << 23)
-#define CPUID_7D_AMX_TILE           (1 << 24)
-#define CPUID_7D_AMX_INT8           (1 << 25)
+enum : uint32_t {
+    CPUID_7C_AVX512VBMI = 1u << 1,
+    CPUID_7C_UMIP = 1u << 2,
+    CPUID_7C_PKU = 1u << 3,
+    CPUID_7C_OSPKE = 1u << 4,
+    CPUID_7C_WAITPKG = 1u << 5,
+    CPUID_7C_AVX512VBMI2 = 1u << 6,
+    CPUID_7C_SHSTK = 1u << 7,
+    CPUID_7C_GFNI = 1u << 8,
+    CPUID_7C_VAES = 1u << 9,
+    CPUID_7C_VPCLMULQDQ = 1u << 10,
+    CPUID_7C_AVX512VNNI = 1u << 11,
+    CPUID_7C_AVX512BITALG = 1u << 12,
+    CPUID_7C_TME_EN = 1u << 13,
+    CPUID_7C_AVX512VPOPCNTDQ = 1u << 14,
+    CPUID_7C_RDPID = 1u << 22,
+    CPUID_7C_KL = 1u << 23,
+    CPUID_7C_CLDEMOTE = 1u << 25,
+    CPUID_7C_MOVDIRI = 1u << 27,
+    CPUID_7C_MOVDIR64B = 1u << 28,
+    CPUID_7C_ENQCMD = 1u << 29,
+};
 
-#define RFLAGS_CF   (1u << 0)
-#define RFLAGS_PF   (1u << 2)
-#define RFLAGS_AF   (1u << 4)
-#define RFLAGS_ZF   (1u << 6)
-#define RFLAGS_SF   (1u << 7)
-#define RFLAGS_TF   (1u << 8)
-#define RFLAGS_IF   (1u << 9)
-#define RFLAGS_DF   (1u << 10)
-#define RFLAGS_OF   (1u << 11)
-#define RFLAGS_IOPL (3u << 12)
-#define RFLAGS_NT   (1u << 14)
-#define RFLAGS_RF   (1u << 16)
-#define RFLAGS_VM   (1u << 17)
-#define RFLAGS_AC   (1u << 18)
-#define RFLAGS_VIF  (1u << 19)
-#define RFLAGS_VIP  (1u << 20)
-#define RFLAGS_ID   (1u << 21)
+enum : uint32_t {
+    CPUID_7D_UINTR = 1u << 5,
+    CPUID_7D_AVX512VP2INTERSECT = 1u << 8,
+    CPUID_7D_SERIALIZE = 1u << 14,
+    CPUID_7D_TSXLDTRK = 1u << 16,
+    CPUID_7D_PCONFIG = 1u << 18,
+    CPUID_7D_IBT = 1u << 20,
+    CPUID_7D_AMX_BF16 = 1u << 22,
+    CPUID_7D_AVX512FP16 = 1u << 23,
+    CPUID_7D_AMX_TILE = 1u << 24,
+    CPUID_7D_AMX_INT8 = 1u << 25,
+};
 
-#define XCR0_X87              (1 << 0)
-#define XCR0_SSE              (1 << 1)
-#define XCR0_AVX              (1 << 2)
-#define XCR0_AXV512_OPMASK    (1 << 5)
-#define XCR0_AVX512_ZMM_HI256 (1 << 6)
-#define XCR0_AVX512_HI16_ZMM  (1 << 7)
+enum : uint64_t {
+    RFLAGS_CF = 1ull << 0,
+    RFLAGS_BRKI = 1ull << 1,
+    RFLAGS_PF = 1ull << 2,
+    RFLAGS_AF = 1ull << 4,
+    RFLAGS_ZF = 1ull << 6,
+    RFLAGS_SF = 1ull << 7,
+    RFLAGS_TF = 1ull << 8,
+    RFLAGS_IF = 1ull << 9,
+    RFLAGS_DF = 1ull << 10,
+    RFLAGS_OF = 1ull << 11,
+    RFLAGS_IOPL = 3ull << 12,
+    RFLAGS_NT = 1ull << 14,
+    RFLAGS_RF = 1ull << 16,
+    RFLAGS_VM = 1ull << 17,
+    RFLAGS_AC = 1ull << 18,
+    RFLAGS_VIF = 1ull << 19,
+    RFLAGS_VIP = 1ull << 20,
+    RFLAGS_ID = 1ull << 21,
+};
 
-#define CR0_PE (1 << 0)
-#define CR0_MP (1 << 1)
-#define CR0_EM (1 << 2)
-#define CR0_TS (1 << 3)
-#define CR0_ET (1 << 4)
-#define CR0_NE (1 << 5)
-#define CR0_WP (1 << 16)
-#define CR0_AM (1 << 18)
-#define CR0_NW (1 << 29)
-#define CR0_CD (1 << 30)
-#define CR0_PG (1 << 31)
+enum : uint64_t {
+    XCR0_X87 = 1ull << 0,
+    XCR0_SSE = 1ull << 1,
+    XCR0_AVX = 1ull << 2,
+    XCR0_AXV512_OPMASK = 1ull << 5,
+    XCR0_AVX512_ZMM_HI256 = 1ull << 6,
+    XCR0_AVX512_HI16_ZMM = 1ull << 7,
+};
 
-#define CR4_VME        (1 << 0)
-#define CR4_PVI        (1 << 1)
-#define CR4_TSD        (1 << 2)
-#define CR4_DE         (1 << 3)
-#define CR4_PSE        (1 << 4)
-#define CR4_PAE        (1 << 5)
-#define CR4_MCE        (1 << 6)
-#define CR4_PGE        (1 << 7)
-#define CR4_PCE        (1 << 8)
-#define CR4_OSFXSR     (1 << 9)
-#define CR4_OSXMMEXCPT (1 << 10)
-#define CR4_UMIP       (1 << 11)
-#define CR4_VMXE       (1 << 13)
-#define CR4_SMXE       (1 << 14)
-#define CR4_FSGSBASE   (1 << 16)
-#define CR4_PCIDE      (1 << 17)
-#define CR4_OSXSAVE    (1 << 18)
-#define CR4_SMEP       (1 << 20)
-#define CR4_SMAP       (1 << 21)
-#define CR4_PKE        (1 << 22)
-#define CR4_CET        (1 << 23)
-#define CR4_PKS        (1 << 24)
+enum : uint64_t {
+    CR0_PE = 1ull << 0,
+    CR0_MP = 1ull << 1,
+    CR0_EM = 1ull << 2,
+    CR0_TS = 1ull << 3,
+    CR0_ET = 1ull << 4,
+    CR0_NE = 1ull << 5,
+    CR0_WP = 1ull << 16,
+    CR0_AM = 1ull << 18,
+    CR0_NW = 1ull << 29,
+    CR0_CD = 1ull << 30,
+    CR0_PG = 1ull << 31,
+};
 
-#define MSR_EFER_SCE   (1 << 0)
-#define MSR_EFER_LME   (1 << 8)
-#define MSR_EFER_LMA   (1 << 10)
-#define MSR_EFER_NXE   (1 << 11)
-#define MSR_EFER_SVME  (1 << 12)
-#define MSR_EFER_LMSLE (1 << 13)
-#define MSR_EFER_FFXSR (1 << 14)
-#define MSR_EFER_TCE   (1 << 15)
+enum : uint64_t {
+    CR4_VME = 1ull << 0,
+    CR4_PVI = 1ull << 1,
+    CR4_TSD = 1ull << 2,
+    CR4_DE = 1ull << 3,
+    CR4_PSE = 1ull << 4,
+    CR4_PAE = 1ull << 5,
+    CR4_MCE = 1ull << 6,
+    CR4_PGE = 1ull << 7,
+    CR4_PCE = 1ull << 8,
+    CR4_OSFXSR = 1ull << 9,
+    CR4_OSXMMEXCPT = 1ull << 10,
+    CR4_UMIP = 1ull << 11,
+    CR4_VMXE = 1ull << 13,
+    CR4_SMXE = 1ull << 14,
+    CR4_FSGSBASE = 1ull << 16,
+    CR4_PCIDE = 1ull << 17,
+    CR4_OSXSAVE = 1ull << 18,
+    CR4_SMEP = 1ull << 20,
+    CR4_SMAP = 1ull << 21,
+    CR4_PKE = 1ull << 22,
+    CR4_CET = 1ull << 23,
+    CR4_PKS = 1ull << 24,
+};
 
-#define MSR_PAT            (0x277)
-#define MSR_EFER           (0xC0000080)
-#define MSR_STAR           (0xC0000081)
-#define MSR_LSTAR          (0xC0000082)
-#define MSR_CSTAR          (0xC0000083)
-#define MSR_SFMASK         (0xC0000084)
-#define MSR_FS_BASE        (0xC0000100)
-#define MSR_GS_BASE        (0xC0000101)
-#define MSR_KERNEL_GS_BASE (0xC0000102)
+enum : uint64_t {
+    MSR_EFER_SCE = 1ull << 0,
+    MSR_EFER_LME = 1ull << 8,
+    MSR_EFER_LMA = 1ull << 10,
+    MSR_EFER_NXE = 1ull << 11,
+    MSR_EFER_SVME = 1ull << 12,
+    MSR_EFER_LMSLE = 1ull << 13,
+    MSR_EFER_FFXSR = 1ull << 14,
+    MSR_EFER_TCE = 1ull << 15,
+};
 
-#define PAT_UC  (0x0)
-#define PAT_WC  (0x1)
-#define PAT_WT  (0x4)
-#define PAT_WP  (0x5)
-#define PAT_WB  (0x6)
-#define PAT_UCM (0x7)
+enum : uint32_t {
+    MSR_PAT = 0x277,
+    MSR_EFER = 0xC0000080,
+    MSR_STAR = 0xC0000081,
+    MSR_LSTAR = 0xC0000082,
+    MSR_CSTAR = 0xC0000083,
+    MSR_SFMASK = 0xC0000084,
+    MSR_FS_BASE = 0xC0000100,
+    MSR_GS_BASE = 0xC0000101,
+    MSR_KERNEL_GS_BASE = 0xC0000102,
+};
 
-#define CPL_USER   (0x3)
-#define CPL_KERNEL (0x0)
+enum : uint8_t {
+    PAT_UC = 0x0,
+    PAT_WC = 0x1,
+    PAT_WT = 0x4,
+    PAT_WP = 0x5,
+    PAT_WB = 0x6,
+    PAT_UCM = 0x7,
+};
 
-#define IDT_PF          (0xE)
-#define IDT_SPURIOUS    (0xFE)
-#define IDT_IPI_RESCHED (0xFF)
+enum : uint8_t {
+    CPL_USER = 0x3,
+    CPL_KERNEL = 0x0,
+};
+
+enum : uint8_t {
+    IDT_DE = 0x0,
+    IDT_DB = 0x1,
+    IDT_BP = 0x3,
+    IDT_OF = 0x4,
+    IDT_BR = 0x5,
+    IDT_UD = 0x6,
+    IDT_NM = 0x7,
+    IDT_SS = 0xC,
+    IDT_GP = 0xD,
+    IDT_PF = 0xE,
+    IDT_MF = 0x10,
+    IDT_AC = 0x11,
+    IDT_XM = 0x13,
+    IDT_IPI_SHOOTDOWN = 0xFC,
+    IDT_TIMER = 0xFD,
+    IDT_SPURIOUS = 0xFE,
+    IDT_IPI_RESCHED = 0xFF,
+};

arch/x86_64/include/x86_64/gdt.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/compiler.h>

arch/x86_64/include/x86_64/hpet.h

@@ -1,3 +0,0 @@
-#pragma once
-
-void hpet_init();

arch/x86_64/include/x86_64/idt.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/init.h>

arch/x86_64/include/x86_64/percpu.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/irq.h>

arch/x86_64/include/x86_64/pmap.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 #include <zinnia/pmap.h>
 

arch/x86_64/include/x86_64/smp.h

@@ -1,8 +1,10 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/atomic.h>
 #include <zinnia/compiler.h>
 #include <zinnia/utils.h>
-#include <stddef.h>
 #include <stdint.h>
 #include <x86_64/gdt.h>
 
@@ -17,7 +19,7 @@ struct [[__packed]] smp_info {
     uint64_t hhdm_offset;
     uint64_t percpu;
     uint32_t lapic_id;
-    _Atomic(uint8_t) booted;
+    atomic_bool booted;
 };
 
 // Discovers all APs and starts them.

arch/x86_64/include/x86_64/syscall.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void arch_syscall_stub();

arch/x86_64/include/x86_64/tsc.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void tsc_init();

arch/x86_64/include/x86_64/tss.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/irq.c

@@ -1,6 +1,14 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/percpu.h>
+#include <zinnia/utils.h>
 #include <arch/irq.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
 bool arch_irq_get_state() {
     uint64_t flags;
@@ -20,3 +28,68 @@ bool arch_irq_set_state(bool state) {
 void arch_irq_wait() {
     asm volatile("hlt");
 }
+
+struct apic_msi_line {
+    struct msi_line msi;
+    uint8_t vector;
+    uint32_t lapic;
+};
+
+static phys_t apic_msi_msg_addr(struct msi_line* self) {
+    struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi);
+    return 0xFEE0'0000 | ((phys_t)line->lapic << 12);
+}
+
+static uint32_t apic_msi_msg_data(struct msi_line* self) {
+    return containerof(self, struct apic_msi_line, msi)->vector;
+}
+
+static enum irq_mode apic_msi_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered");
+    return IRQ_MODE_EOI;
+}
+
+static void apic_msi_mask(struct irq_line* self) {}
+
+static void apic_msi_unmask(struct irq_line* self) {}
+
+static void apic_msi_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
+    irq_lock();
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+    size_t slot;
+    for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) {
+        if (!arch_cpu->irq_lines[slot])
+            break;
+    }
+    if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
+        irq_unlock();
+        mem_free(line);
+        return ENOSPC;
+    }
+
+    line->msi.line.set_config = apic_msi_set_config;
+    line->msi.line.mask = apic_msi_mask;
+    line->msi.line.unmask = apic_msi_unmask;
+    line->msi.line.eoi = apic_msi_eoi;
+    line->msi.msg_addr = apic_msi_msg_addr;
+    line->msi.msg_data = apic_msi_msg_data;
+    line->vector = slot + 0x20;
+    line->lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = &line->msi.line;
+    irq_unlock();
+
+    *out = &line->msi;
+    return 0;
+}

arch/x86_64/mem.c

@@ -1,4 +1,3 @@
-#include <zinnia/types.h>
 #include <arch/mem.h>
 #include <stddef.h>
 #include <stdint.h>

arch/x86_64/meson.build

@@ -14,12 +14,20 @@ kernel_sources += files(
     'mem.c',
     'percpu.c',
     'pmap.c',
-    'pmap.c',
+    'random.c',
     'sched.c',
     'serial.c',
     'smp.c',
-    'sys.c',
+    'start.c',
     'syscall.c',
     'tsc.c',
     'usercopy.c',
 )
+
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('pci.c')
+endif
+
+if kconfig.get('CONFIG_ACPI', 'n') == 'y'
+    kernel_sources += files('acpi.c')
+endif

arch/x86_64/pci.c

@@ -0,0 +1,50 @@
+#include <zinnia/init.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <x86_64/asm.h>
+
+constexpr uint16_t PCI_CONFIG_ADDRESS = 0xcf8;
+constexpr uint16_t PCI_CONFIG_DATA = 0xcfc;
+
+static struct spinlock port_lock = {};
+
+static void port_select(const struct pci_address* addr, uint16_t offset) {
+    asm_outl(
+        PCI_CONFIG_ADDRESS,
+        1u << 31 | (uint32_t)addr->bus << 16 | (uint32_t)(addr->device & 0x1f) << 11 |
+            (uint32_t)(addr->function & 0x7) << 8 | (offset & 0xfc)
+    );
+}
+
+static uint32_t port_read32(struct pci_access*, const struct pci_address* addr, uint16_t offset) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    uint32_t value = asm_inl(PCI_CONFIG_DATA);
+    spin_unlock(&port_lock);
+    return value;
+}
+
+static void port_write32(struct pci_access*, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    asm_outl(PCI_CONFIG_DATA, value);
+    spin_unlock(&port_lock);
+}
+
+static struct pci_access port_access = {
+    .name = "port",
+    .domain = 0,
+    .start_bus = 0,
+    .end_bus = 255,
+    .priority = PCI_ACCESS_PRIO_LEGACY,
+    .read32 = port_read32,
+    .write32 = port_write32,
+};
+
+[[__init]]
+static void pci_x86_stage_init(struct boot_info*) {
+    pci_access_register(&port_access);
+}
+INIT_TASK(pci_x86_stage, "arch.x86_64.pci", pci_x86_stage_init);
+INIT_DEPENDS(pci_x86_stage, mem_stage);
+INIT_ENTAILS(pci_x86_stage, pci_stage);

arch/x86_64/percpu.c

@@ -1,3 +1,4 @@
+#include <zinnia/initgraph.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <config.h>
@@ -6,7 +7,6 @@
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
-#include <x86_64/hpet.h>
 #include <x86_64/idt.h>
 #include <x86_64/percpu.h>
 #include <x86_64/smp.h>
@@ -66,10 +66,11 @@ void x86_64_cpu_setup(struct percpu* cpu) {
     asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
     union ia32_star star = {0};
     star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
-    star.user_cs_ss = offsetof(struct gdt, user_code32);
+    star.user_cs_ss = offsetof(struct gdt, user_code32) | CPL_USER;
     asm_wrmsr(MSR_STAR, star.value);
     asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
-    asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
+    // Mask all RFLAGS except for BRKI which is reserved.
+    asm_wrmsr(MSR_SFMASK, ~RFLAGS_BRKI);
 
     // TODO: FRED
 
@@ -83,7 +84,7 @@ void x86_64_cpu_setup(struct percpu* cpu) {
 
     // Enable SSE.
     cr0 &= ~CR0_EM;
-    cr0 |= CR0_MP;
+    cr0 |= CR0_MP | CR0_WP;
     cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
 
     if (cpuid1.ecx & CPUID_1C_XSAVE) {
@@ -132,7 +133,6 @@ void x86_64_cpu_setup(struct percpu* cpu) {
 }
 
 void arch_percpu_init() {
-    hpet_init();
 #if CONFIG_TSC
     tsc_init();
 #endif

arch/x86_64/pmap.c

@@ -1,15 +1,44 @@
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/types.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 #include <x86_64/pmap.h>
 
 void pte_clear(pte_t* pte) {
     *pte = 0;
 }
 
+void pte_write(pte_t* pte, pte_t val) {
+    *pte = val;
+}
+
 void pte_flush(uintptr_t vaddr) {
     asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
 }
 
+void pte_flush_all() {
+    uint64_t cr3;
+    asm volatile("mov %0, cr3" : "=r"(cr3));
+    asm volatile("mov cr3, %0" ::"r"(cr3) : "memory");
+}
+
+void pte_shootdown_kick(struct percpu* cpu) {
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_SHOOTDOWN,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
 
@@ -42,12 +71,19 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     return result;
 }
 
-bool pte_is_present(pte_t* pte) {
-    return *pte & ARCH_FLAG_PRESENT;
+pte_t pte_protect(pte_t old, enum pte_flags flags) {
+    pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+    if (flags & PTE_WRITE)
+        result |= ARCH_FLAG_READ_WRITE;
+    if (!(flags & PTE_EXEC))
+        result |= ARCH_FLAG_EXECUTE_DISABLE;
+    return result;
 }
 
-bool pte_is_dir(pte_t* pte) {
-    return *pte & ARCH_FLAG_SIZE;
+bool pte_is_present(pte_t* pte) {
+    return *pte & ARCH_FLAG_PRESENT;
 }
 
 phys_t pte_address(pte_t* pte) {

arch/x86_64/random.c

@@ -0,0 +1,36 @@
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/random.h>
+#include <x86_64/asm.h>
+
+#define CPUID_1C_RDRAND (1u << 30)
+#define RDRAND_RETRIES  10
+
+static bool rdrand_get_u64(struct random_source*, uint64_t* out) {
+    for (int i = 0; i < RDRAND_RETRIES; i++) {
+        uint64_t value;
+        uint8_t ok;
+        asm volatile("rdrand %0\n setc %1" : "=r"(value), "=qm"(ok) : : "cc");
+        if (ok) {
+            *out = value;
+            return true;
+        }
+    }
+    return false;
+}
+
+static struct random_source rdrand_source = {
+    .name = "x86_64.rdrand",
+    .quality = 100,
+    .get_u64 = rdrand_get_u64,
+};
+
+static void rdrand_init(struct boot_info*) {
+    if (asm_cpuid(1, 0).ecx & CPUID_1C_RDRAND) {
+        random_register(&rdrand_source);
+    } else {
+        log_warn("random: This CPU doesn't support RDRAND, ignoring\n");
+    }
+}
+INIT_TASK(rdrand_stage, "arch.x86_64.rdrand", rdrand_init);
+INIT_ENTAILS(rdrand_stage, random_stage);

arch/x86_64/sched.c

@@ -2,6 +2,7 @@
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/sched.h>
@@ -10,6 +11,8 @@
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
+#include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
@@ -91,6 +94,11 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
     return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
 }
 
+constexpr size_t FPU_FCW_OFFSET = 0;
+constexpr size_t FPU_MXCSR_OFFSET = 24;
+constexpr uint16_t FPU_DEFAULT_FCW = 0x37F;
+constexpr uint32_t FPU_DEFAULT_MXCSR = 0x1F80;
+
 // Calls task_entry with the correct parameters.
 [[__naked]]
 void task_entry_thunk() {
@@ -98,9 +106,9 @@ void task_entry_thunk() {
         "mov rdi, rbx\n"
         "mov rsi, r12\n"
         "mov rdx, rax\n"
-        "jmp %0"
+        "jmp task_entry"
+        :
         :
-        : "r"(task_entry)
         : "memory", "rdi", "rsi", "rdx"
     );
 }
@@ -114,7 +122,7 @@ errno_t arch_task_init(
 ) {
     struct arch_percpu* cpu = percpu_ref(arch_data);
 
-    assert((stack_start & 0xF) == 0, "Task stack is not 16-byte aligned!\n");
+    assert((stack_start & 0xF) == 0, "Task stack is not 16-byte aligned!");
 
     // Prepare a dummy stack with an entry point function to return to.
     struct task_frame* frame = (struct task_frame*)(stack_start - sizeof(uint64_t)) - 1;
@@ -140,6 +148,11 @@ errno_t arch_task_init(
         context->fpu_region = (void*)fpu_base;
         context->fpu_phys = pages;
 
+        const uint16_t fcw = FPU_DEFAULT_FCW;
+        const uint32_t mxcsr = FPU_DEFAULT_MXCSR;
+        memcpy((uint8_t*)fpu_base + FPU_FCW_OFFSET, &fcw, sizeof(fcw));
+        memcpy((uint8_t*)fpu_base + FPU_MXCSR_OFFSET, &mxcsr, sizeof(mxcsr));
+
         context->ds = asm_read_ds();
         context->es = asm_read_es();
         context->fs = asm_read_fs();
@@ -160,12 +173,28 @@ void arch_task_fini(struct arch_task_context* context) {
     const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
     for (size_t i = 0; i < fpu_size; i += page_size)
         pmap_unmap(&kernel_vma.pmap, fpu_base + i);
+    pmap_shootdown(&kernel_vma.pmap);
     mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
     vm_free(fpu_base, fpu_size);
 
     context->fpu_region = nullptr;
 }
 
+void arch_task_fork(struct arch_task_context* dst) {
+    if (dst->fpu_region)
+        percpu_ref(arch_data)->fpu_save(dst->fpu_region);
+}
+
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) {
+    *ctx = (struct arch_context){
+        .rip = ip,
+        .rsp = sp,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+}
+
 [[__naked]]
 void arch_sched_preempt_disable() {
     asm volatile(
@@ -189,6 +218,23 @@ bool arch_sched_preempt_enable() {
     );
 }
 
+void arch_sched_kick(struct percpu* cpu) {
+    if (!cpu->online)
+        return;
+    // The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler.
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_RESCHED,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 [[__naked]]
 void arch_sched_jump_to_context(struct arch_context* context) {
     asm volatile(
@@ -208,14 +254,14 @@ void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
     };
 
     uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
-    const uint16_t zero = 0;
     asm volatile(
         "mov ds, %0\n"
         "mov es, %0\n"
         "mov fs, %0\n"
         "mov gs, %0\n"
         :
-        : "r"(zero)
+        : "a"(0)
+        : "memory"
     );
 
     asm_wrmsr(MSR_FS_BASE, 0);

arch/x86_64/serial.c

@@ -1,4 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/initgraph.h>
+#include <arch/cpu.h>
 #include <stdint.h>
 #include <x86_64/asm.h>
 
@@ -11,7 +16,7 @@ static bool is_ready() {
 static void serial_write(struct console* con, const char* buf, size_t count) {
     for (size_t i = 0; i < count; i++) {
         while (!is_ready())
-            asm volatile("pause");
+            arch_cpu_pause();
 
         asm_outb(COM1_BASE, buf[i]);
     }
@@ -44,4 +49,10 @@ static struct console serial_con = {
     .write = serial_write,
     .init = serial_setup,
 };
-DEFINE_CONSOLE(&serial_con);
+
+[[__init]]
+static void serial_init(struct boot_info*) {
+    console_register(&serial_con);
+}
+INIT_TASK(serial_stage, "arch.x86_64.serial", serial_init);
+INIT_ENTAILS(serial_stage, console_stage);

arch/x86_64/smp.c

@@ -1,5 +1,8 @@
+#define pr_fmt(fmt) "smp: " fmt
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
+#include <zinnia/atomic.h>
 #include <zinnia/clock.h>
 #include <zinnia/compiler.h>
 #include <zinnia/irq.h>
@@ -7,13 +10,12 @@
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
-#include <zinnia/string.h>
 #include <zinnia/types.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
-#include <stdatomic.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 #include <uacpi/acpi.h>
 #include <uacpi/tables.h>
 #include <x86_64/apic.h>
@@ -37,26 +39,30 @@ static void smp_trampoline() {
         "    cld\n"
         "    jmp .Lmode16\n"
 
+        ".org smp_trampoline_start + %c[data_off]\n"
         ".global smp_trampoline_data\n"
         "smp_trampoline_data:\n"
         "    .skip %c[info_size]\n"
 
-        ".set .Ldata_offset, (smp_trampoline_data - smp_trampoline_start)\n"
+        ".set .Ldata_offset, %c[data_off]\n"
         ".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
         ".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
         ".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
         ".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
         ".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
         ".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
+        ".set .Linvalid_idtr_offset, (.Linvalid_idtr - smp_trampoline_start)\n"
+        ".set .Lmode32_offset, (.Lmode32 - smp_trampoline_start)\n"
+        ".set .Lmode64_offset, (.Lmode64 - smp_trampoline_start)\n"
 
         ".Lmode16:\n"
         "    mov bx, cs\n"
         "    shl ebx, 4\n"
 
-        "    lidtd cs:(.Linvalid_idtr - smp_trampoline_start)\n"
+        "    lidtd cs:.Linvalid_idtr_offset\n"
         "    lgdtd cs:.Lgdtr_offset\n"
 
-        "    lea eax, [ebx + (.Lmode32 - smp_trampoline_start)]\n"
+        "    lea eax, [ebx + .Lmode32_offset]\n"
         "    mov dword ptr cs:.Lfarjmp_offset, eax\n"
 
         "    mov eax, %c[cr0]\n"
@@ -95,7 +101,7 @@ static void smp_trampoline() {
         "    or eax, %c[cr0_pg]\n"
         "    mov cr0, eax\n"
 
-        "    lea eax, [ebx + (.Lmode64 - smp_trampoline_start)]\n"
+        "    lea eax, [ebx + .Lmode64_offset]\n"
         "    push %c[kernel64_cs]\n"
         "    push eax\n"
         "    retf\n"
@@ -130,7 +136,8 @@ static void smp_trampoline() {
         ".popsection\n"
         "    ud2\n"
         :
-        : [info_size] "i"(sizeof(struct smp_info)),
+        : [data_off] "i"(8),
+          [info_size] "i"(sizeof(struct smp_info)),
           [gdtr] "i"(offsetof(struct smp_info, gdtr)),
           [farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
           [temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
@@ -141,7 +148,7 @@ static void smp_trampoline() {
           [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
           [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
           [cr0] "i"(CR0_PE | CR0_ET),
-          [cr0_pg] "i"(CR0_PG),
+          [cr0_pg] "i"((uint32_t)CR0_PG),
           [cr4_pae] "i"(CR4_PAE),
           [msr_efer] "i"(MSR_EFER),
           [efer_lme] "i"(MSR_EFER_LME),
@@ -168,9 +175,9 @@ static void ap_entry(phys_t info_addr) {
 
     // Let the BSP know that we're alive.
     // `info` must not be touched after this point.
-    atomic_store_explicit(&info->booted, 1, memory_order_release);
+    atomic_store(&info->booted, 1, ATOMIC_RELEASE);
 
-    log("smp: CPU %zu online\n", cpu->id);
+    log("CPU %zu online\n", cpu->id);
 
     sched_yield(&cpu->sched);
     unreachable();
@@ -181,18 +188,18 @@ static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
 
     phys_t stack;
     if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
-        log("smp: Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
+        log_err("Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
         return false;
     }
 
     struct percpu* cpu = percpu_new();
     if (!cpu) {
-        log("smp: Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
+        log_err("Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
         mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
         return false;
     }
 
-    log("smp: Starting CPU %zu\n", cpu->id);
+    log("Starting CPU %zu\n", cpu->id);
 
     memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
 
@@ -227,10 +234,10 @@ static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
     );
     clock_spin_ns(10'000'000);
 
-    for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) {
+    for (uint64_t waited = 0; !atomic_load(&info->booted, ATOMIC_ACQUIRE); waited += 1'000'000) {
         if (waited >= AP_BOOT_TIMEOUT_NS) {
             // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
-            log("smp: CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
+            log_err("CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
             return false;
         }
         clock_spin_ns(1'000'000);
@@ -271,7 +278,7 @@ static errno_t build_temp_table(phys_t* out) {
 
 void x86_64_smp_init() {
     const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
-    assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page\n");
+    assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page");
 
     const size_t max_cpus = percpu_max_cpus();
     if (max_cpus <= 1)
@@ -279,20 +286,20 @@ void x86_64_smp_init() {
 
     uacpi_table table;
     if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
-        log("smp: No MADT found, not starting any APs\n");
+        log_warn("No MADT found, not starting any APs\n");
         return;
     }
 
     phys_t trampoline;
     if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
-        log("smp: Failed to allocate 20-bit memory for the AP trampoline\n");
+        log_err("Failed to allocate 20-bit memory for the AP trampoline\n");
         uacpi_table_unref(&table);
         return;
     }
 
     phys_t temp_table;
     if (build_temp_table(&temp_table)) {
-        log("smp: Failed to allocate a temporary page table\n");
+        log_err("Failed to allocate a temporary page table\n");
         mem_phys_free(trampoline, 1);
         uacpi_table_unref(&table);
         return;
@@ -311,7 +318,7 @@ void x86_64_smp_init() {
 
         // Don't trust malformed firmware tables.
         if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
-            log("smp: MADT contains a malformed entry\n");
+            log_warn("MADT contains a malformed entry\n");
             break;
         }
 
@@ -348,5 +355,5 @@ void x86_64_smp_init() {
         mem_phys_free(trampoline, 1);
     }
 
-    log("smp: %zu CPU(s) online\n", num_cpus);
+    log("%zu CPU(s) online\n", num_cpus);
 }

arch/x86_64/syscall.c

@@ -58,8 +58,10 @@ void arch_syscall_stub() {
         "    pop rbx\n"
         "    pop rax\n"
         "    add rsp, 0x10\n" // Skip .error and .isr fields
-        "2:\n"
-        "    mov rsp, gs:%c[user_stack]\n"
+        "    pop rcx\n"       // rip
+        "    add rsp, 0x08\n" // Skip cs
+        "    pop r11\n"       // rflags
+        "    pop rsp\n"       // rsp
         "    swapgs\n"
         "    sysretq\n"
         :

arch/x86_64/tsc.c

@@ -1,3 +1,4 @@
+#define pr_fmt(fmt) "x86_64/tsc: " fmt
 #include <zinnia/clock.h>
 #include <zinnia/cmdline.h>
 #include <zinnia/print.h>
@@ -42,11 +43,13 @@ static struct clock tsc_clock = {
 };
 
 void tsc_init() {
-    if (!use_tsc)
+    if (!use_tsc) {
+        log_warn("TSC disabled on the command line");
         return;
+    }
 
     if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
-        log("x86_64/tsc: No invariant TSC detected!\n");
+        log_warn("No invariant TSC detected!\n");
         return;
     }
 
@@ -69,17 +72,17 @@ void tsc_init() {
         // On a normal system, this should usually never be called and is a last resort
         // since at this point we have at least the HPET timer.
         if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
-            log("x86_64/tsc: Calibrating TSC using CPUID 0x15");
+            log("Calibrating TSC using CPUID 0x15");
             tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
         } else {
-            log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information");
+            log_err("Unable to calibrate TSC using CPUID frequency information");
             return;
         }
     } else {
-        log("x86_64/tsc: Unable to calibrate the TSC without a clock or static frequency info!\n");
+        log_err("Unable to calibrate the TSC without a clock or static frequency info!\n");
         return;
     }
 
-    log("x86_64/tsc: Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
+    log("Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
     clock_switch(&tsc_clock);
 }

boot/cmdline.c

@@ -1,26 +1,30 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/init.h>
-#include <stddef.h>
 #include <zinnia/stdlib.h>
-#include <zinnia/string.h>
+#include <stddef.h>
+#include <string.h>
 
 // Defined in linker script.
 extern const struct cmdline_option* __ld_cmdline_start[];
 extern const struct cmdline_option* __ld_cmdline_end[];
 
 static char cmdline_buf[0x1000];
+static char cmdline_raw[0x1000];
 
-[[__init]]
-void cmdline_parse(const char* cmdline) {
-    if (cmdline == nullptr)
-        cmdline = "";
+const char* cmdline_get() {
+    return cmdline_raw;
+}
 
-    const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
+void cmdline_parse_buf(
+    char* cmdline_buf,
+    size_t len,
+    const struct cmdline_option* const* start,
+    const struct cmdline_option* const* end
+) {
     size_t idx = 0;
-    memcpy(cmdline_buf, cmdline, len);
 
     while (1) {
-        const struct cmdline_option** opt = __ld_cmdline_start;
+        const struct cmdline_option* const* opt = start;
         char* name = nullptr;
         char* value = nullptr;
 
@@ -62,7 +66,7 @@ void cmdline_parse(const char* cmdline) {
         }
 
         // Find the corresponding option.
-        while (opt < __ld_cmdline_end) {
+        while (opt < end) {
             const struct cmdline_option* current = *opt;
             if (!strcmp(current->name, name)) {
                 if (value == nullptr)
@@ -105,3 +109,14 @@ void cmdline_parse(const char* cmdline) {
             break;
     }
 }
+
+[[__init]]
+void cmdline_parse(const char* cmdline) {
+    if (cmdline == nullptr)
+        cmdline = "";
+
+    const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
+    memcpy(cmdline_buf, cmdline, len);
+    memcpy(cmdline_raw, cmdline, len);
+    cmdline_parse_buf(cmdline_buf, len, __ld_cmdline_start, __ld_cmdline_end);
+}

boot/kernel.ld

@@ -6,6 +6,7 @@ PHDRS {
     data    PT_LOAD     FLAGS(6);
     dynamic PT_DYNAMIC  FLAGS(6);
     percpu  PT_LOAD     FLAGS(6);
+    cmdline 0x60000005;
 }
 
 SECTIONS
@@ -28,7 +29,7 @@ SECTIONS
         __ld_cmdline_start = .;
         KEEP(*(.cmdline))
         __ld_cmdline_end = .;
-    } :rodata
+    } :rodata :cmdline
 
     .rodata : {
         *(.rodata .rodata.*)
@@ -39,6 +40,15 @@ SECTIONS
         KEEP(*(.mod.entry .mod.entry.*))
         __ld_mod_entry_end = .;
 
+        . = ALIGN(8);
+        __ld_initgraph_nodes_start = .;
+        KEEP(*(.initgraph.nodes))
+        __ld_initgraph_nodes_end = .;
+
+        __ld_initgraph_edges_start = .;
+        KEEP(*(.initgraph.edges))
+        __ld_initgraph_edges_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -52,12 +62,6 @@ SECTIONS
         *(.data .data.*)
     } :data
 
-    .console : {
-        __ld_console_start = .;
-        KEEP(*(.console))
-        __ld_console_end = .;
-    } :data
-
     .dynsym : {
         __ld_dynsym_start = .;
         KEEP(*(.dynsym*))

boot/limine/entry.c

@@ -1,9 +1,11 @@
 #include <zinnia/alloc.h>
+#include <zinnia/assert.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/fdt.h>
 #include <zinnia/init.h>
 #include <zinnia/print.h>
 #include <zinnia/utils.h>
-#include <zinnia/string.h>
+#include <string.h>
 #include "limine.h"
 
 [[__initdata_sorted("limine.0")]]
@@ -41,12 +43,19 @@ void kernel_entry() {
 
     struct boot_info info = {};
 
-    struct limine_memmap_response* mm_res = memmap_request.response;
-    struct limine_module_response* module_res = module_request.response;
-    struct limine_executable_address_response* exec_res = executable_address_request.response;
-    struct limine_executable_file_response* exec_file_res = executable_file_request.response;
+    const struct limine_memmap_response* mm_res = memmap_request.response;
+    const struct limine_hhdm_response* hhdm_res = hhdm_request.response;
+    const struct limine_module_response* module_res = module_request.response;
+    const struct limine_executable_address_response* exec_res = executable_address_request.response;
+    const struct limine_executable_file_response* exec_file_res = executable_file_request.response;
+    const struct limine_rsdp_response* rsdp_res = rsdp_request.response;
+
+    assert(mm_res, "memmap_request returned no response!");
+    assert(hhdm_res, "hhdm_request returned no response!");
 
     info.cmdline = exec_file_res->executable_file->string;
+    info.kernel_file.data = (phys_t)((uintptr_t)exec_file_res->executable_file->address - hhdm_res->offset);
+    info.kernel_file.length = exec_file_res->executable_file->size;
 
     info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
     for (size_t i = 0; i < info.num_mem_maps; i++) {
@@ -74,7 +83,7 @@ void kernel_entry() {
         info.files = files;
         info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
         for (size_t i = 0; i < info.num_files; i++) {
-            files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset;
+            files[i].data = (phys_t)module_res->modules[i]->address - hhdm_res->offset;
             files[i].length = module_res->modules[i]->size;
             strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1);
             files[i].path[sizeof(files[i].path) - 1] = 0;
@@ -84,7 +93,35 @@ void kernel_entry() {
     info.mem_map = mem;
     info.phys_base = (phys_t)exec_res->physical_base;
     info.virt_base = (uintptr_t)exec_res->virtual_base;
-    info.hhdm_base = (uintptr_t)hhdm_request.response->offset;
+    info.hhdm_base = (uintptr_t)hhdm_res->offset;
+
+    const struct limine_dtb_response* dtb_res = dtb_request.response;
+    if (dtb_res && dtb_res->dtb_ptr) {
+        info.dtb = (phys_t)((uintptr_t)dtb_res->dtb_ptr - hhdm_res->offset);
+        info.dtb_size = fdt_read_u32((const uint8_t*)dtb_res->dtb_ptr + 4);
+    }
+
+    const struct limine_framebuffer_response* fb_res = framebuffer_request.response;
+    if (fb_res) {
+        // Only use the first framebuffer.
+        if (fb_res->framebuffer_count >= 1) {
+            const struct limine_framebuffer* fb = framebuffer_request.response->framebuffers[0];
+            info.fb.base = (phys_t)((uintptr_t)fb->address - hhdm_res->offset);
+            info.fb.width = fb->width;
+            info.fb.height = fb->height;
+            info.fb.pitch = fb->pitch;
+            info.fb.bytes_per_pixel = fb->bpp / 8;
+
+            info.fb.red.shift = fb->red_mask_shift;
+            info.fb.red.size = fb->red_mask_size;
+
+            info.fb.green.shift = fb->green_mask_shift;
+            info.fb.green.size = fb->green_mask_size;
+
+            info.fb.blue.shift = fb->blue_mask_shift;
+            info.fb.blue.size = fb->blue_mask_size;
+        }
+    }
 
 #if CONFIG_ACPI
     if (rsdp_request.response) {

boot/main.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
@@ -6,63 +8,59 @@
 #include <zinnia/console.h>
 #include <zinnia/elf.h>
 #include <zinnia/init.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/initramfs.h>
 #include <zinnia/irq.h>
 #include <zinnia/module.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/utils.h>
+#include <zinnia/vfs.h>
 #include <zinnia/vma.h>
 #include <zinnia/vmo.h>
 #include <config.h>
 #include <stdint.h>
 
-#if CONFIG_ACPI
-#include <zinnia/acpi.h>
-#endif
-
 static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
                                     ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
 
+// Path to the init executable.
+static const char* init_path = "/init";
+static void init_option(const char* value) {
+    init_path = value;
+}
+CMDLINE_OPTION("init", "Path to the init executable", init_option);
+
 static void kernel_main_task(uintptr_t arg0) {
+    errno_t status = 0;
     struct boot_info* info = (struct boot_info*)arg0;
 
-    log("main: Hello from main task!\n");
+    log("Hello from main task!\n");
 
-    // Initialize built-in modules.
+    // Finish the initgraph.
+    initgraph_run(nullptr, info);
+
+    // After everything is loaded, load all builtin drivers.
     module_init();
 
-    assert(info->num_files >= 1, "No init executable provided");
+    // Load all initramfs files.
+    initramfs_load_boot_modules(info);
 
-    struct vma* init_space;
-    assert(vma_new(&init_space) == EOK, "");
+    const char* init_argv[] = {init_path, nullptr};
+    const char* init_envp[] = {"HOME=/", "TERM=console", nullptr};
 
-    struct boot_file* file_data = &info->files[0];
-    struct paged_vmo* init_file;
-    assert(vmo_new_phys(&init_file) == EOK, "");
-    vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
+    log("Starting init \"%s\"\n", init_path);
+    log("With arguments:\n");
+    for (const char** arg = init_argv; *arg; arg++)
+        log("    %s\n", *arg);
+    log("With environment:\n");
+    for (const char** env = init_envp; *env; env++)
+        log("    %s\n", *env);
 
-    // Load init.
-    const char* init_argv[] = {"init", nullptr};
-    const char* init_envp[] = {nullptr};
-
-    struct exec_info exec = {
-        .space = init_space,
-        .file_obj = &init_file->object,
-        .argv = init_argv,
-        .envp = init_envp,
-    };
-
-    struct task* init_task = nullptr;
-    errno_t status = elf_load(&exec, &init_task);
-    vmo_unref(&init_file->object);
-
-    assert(status == 0, "Failed to load init process (%i)\n", status);
-    sched_add_task(&percpu_get()->sched, init_task);
-
-    while (true) {
-        irq_wait();
-    }
+    status = process_spawn_init(init_path, init_argv, init_envp);
+    assert(status == 0, "Failed to start init \"%s\" (%i)", init_path, status);
 }
 
 [[__init]]
@@ -74,25 +72,15 @@ void kernel_early_init() {
 
 [[noreturn]]
 void kernel_main(struct boot_info* info) {
-    // TODO: Initialize in-memory log ring.
-    kprintf("%s\n", zinnia_banner); // Say hello!
-    kprintf("Command line: %s\n", info->cmdline);
-
     cmdline_parse(info->cmdline);
-    console_init();
-    kprintf("\ec");
 
-    mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
+    log("%s\n", zinnia_banner); // Say hello!
+    log("Command line: %s\n", info->cmdline);
 
-#if CONFIG_ACPI
-    acpi_init(info);
-#endif
-
-    percpu_init();
-    sched_init(&percpu_get()->sched);
+    initgraph_run(sched_stage, info);
 
     struct task* main_task;
-    task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
+    task_create("main", process_kernel(), kernel_main_task, (uintptr_t)info, &main_task);
     sched_add_task(&percpu_get()->sched, main_task);
 
     irq_unlock();

core/clock.c

@@ -1,64 +0,0 @@
-#include <zinnia/clock.h>
-#include <zinnia/compiler.h>
-#include <zinnia/print.h>
-#include <zinnia/spin.h>
-#include <stdint.h>
-
-static struct clock* active_clock = nullptr;
-static uint64_t counter_base = 0;
-static struct spinlock lock = {};
-
-static uint64_t clock_get_elapsed_ns_locked() {
-    if (__unlikely(!active_clock))
-        return 0;
-
-    return counter_base + active_clock->get_elapsed_ns(active_clock);
-}
-
-bool clock_available() {
-    spin_lock(&lock);
-    bool available = active_clock != nullptr;
-    spin_unlock(&lock);
-    return available;
-}
-
-uint64_t clock_get_elapsed_ns() {
-    spin_lock(&lock);
-    uint64_t elapsed = clock_get_elapsed_ns_locked();
-    spin_unlock(&lock);
-    return elapsed;
-}
-
-bool clock_switch(struct clock* clock) {
-    spin_lock(&lock);
-
-    if (active_clock) {
-        if (clock->priority <= active_clock->priority) {
-            spin_unlock(&lock);
-            return false;
-        }
-    }
-
-    uint64_t elapsed = clock_get_elapsed_ns_locked();
-    clock->reset(clock);
-
-    counter_base = elapsed;
-    active_clock = clock;
-
-    spin_unlock(&lock);
-
-    log("clock: Switching to clock source \"%s\"\n", clock->name);
-    return true;
-}
-
-void clock_spin_ns(uint64_t ns) {
-    if (!clock_available()) {
-        log("clock: Unable to sleep for %lu ns, no clock available!\n", ns);
-        return;
-    }
-
-    uint64_t target = clock_get_elapsed_ns() + ns;
-    while (clock_get_elapsed_ns() < target) {
-        // TODO: Maybe pause?
-    }
-}

core/console.c

@@ -1,45 +0,0 @@
-#include <zinnia/cmdline.h>
-#include <zinnia/console.h>
-#include <zinnia/print.h>
-#include <zinnia/string.h>
-
-extern struct console* const __ld_console_start[];
-extern struct console* const __ld_console_end[];
-
-struct console* active = nullptr;
-
-void console_write(const char* buf, size_t len) {
-    if (__likely(active))
-        active->write(active, buf, len);
-}
-
-static void console_setup(const char* value) {
-    if (!value)
-        return;
-
-    // Find a suitable boot console.
-    struct console* const* cur = __ld_console_start;
-    while (cur < __ld_console_end) {
-        if (!strcmp(value, (*cur)->name)) {
-            active = *cur;
-        }
-        cur++;
-    }
-}
-
-CMDLINE_OPTION("console", "Set a default console", console_setup);
-
-void console_init() {
-    // If we have no console set and at least one set, use that as a default.
-    if (!active && &__ld_console_start[0] != &__ld_console_end[0])
-        active = __ld_console_start[0];
-
-    if (active) {
-        if (active->init(active) == false) {
-            active = nullptr;
-            return;
-        }
-
-        log("console: Logging on console \"%s\"\n", active->name);
-    }
-}

core/elf.c

@@ -1,252 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/elf.h>
-#include <zinnia/errno.h>
-#include <zinnia/mman.h>
-#include <zinnia/percpu.h>
-#include <zinnia/pmap.h>
-#include <zinnia/print.h>
-#include <zinnia/sched.h>
-#include <zinnia/string.h>
-#include <zinnia/utils.h>
-#include <zinnia/vma.h>
-#include <zinnia/vmo.h>
-#include <bits/elf.h>
-#include <stdckdint.h>
-
-errno_t elf_load(struct exec_info* info, struct task** out) {
-    errno_t err;
-
-    size_t read;
-    struct elf_ehdr ehdr = {};
-    err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
-    if (err)
-        return err;
-    if (read != sizeof(ehdr))
-        return ENOEXEC;
-
-    // Check if the file is an ELF.
-    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
-        return EINVAL;
-    if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
-        return EINVAL;
-    if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS)
-        return EINVAL;
-    if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
-        return EINVAL;
-    if (ehdr.e_machine != ELF_ARCH_MACHINE)
-        return EINVAL;
-    if (ehdr.e_type != ET_EXEC)
-        return EINVAL;
-    if (!arch_mem_addr_is_user(ehdr.e_entry))
-        return ENOEXEC;
-
-    const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
-    uintptr_t phdrs_end;
-    if (ehdr.e_phentsize < sizeof(struct elf_phdr) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
-        return ENOEXEC;
-    if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size))
-        return ENOEXEC;
-
-    const size_t page_size = arch_mem_page_size();
-    uintptr_t phdr_addr = 0;
-
-    for (size_t i = 0; i < ehdr.e_phnum; i++) {
-        struct elf_phdr phdr = {};
-        err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
-        if (err)
-            return err;
-        if (read != sizeof(phdr))
-            return ENOEXEC;
-
-        if (phdr.p_type == PT_PHDR)
-            phdr_addr = phdr.p_vaddr;
-
-        if (phdr.p_type == PT_LOAD) {
-            if (phdr.p_memsz == 0)
-                continue;
-            if (phdr.p_filesz > phdr.p_memsz)
-                return ENOEXEC;
-
-            uintptr_t seg_end;
-            if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz))
-                return ENOEXEC;
-            if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1))
-                return ENOEXEC;
-
-            if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
-                return ENOEXEC;
-
-            if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
-                phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset);
-
-            enum prot_flags prot = 0;
-            if (phdr.p_flags & PF_R)
-                prot |= PROT_READ;
-            if (phdr.p_flags & PF_W)
-                prot |= PROT_WRITE;
-            if (phdr.p_flags & PF_X)
-                prot |= PROT_EXEC;
-
-            const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
-            const uintptr_t map_address = phdr.p_vaddr - misalign;
-            const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
-            const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
-
-            // Copy the file data into its own mapping.
-            struct paged_vmo* phdr_obj;
-            err = vmo_new_phys(&phdr_obj);
-            if (err)
-                return err;
-
-            err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
-            if (err) {
-                vmo_unref(&phdr_obj->object);
-                return err;
-            }
-
-            // We map more than we copied so the rest is filled with zeroed pages.
-            err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
-            vmo_unref(&phdr_obj->object);
-            if (err)
-                return err;
-        }
-    }
-
-    const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
-    const size_t stack_size = 2 * 1024 * 1024; // 2MiB
-    const uintptr_t stack_start = highest - stack_size;
-
-    // Allocate stack.
-    struct paged_vmo* stack;
-    err = vmo_new_phys(&stack);
-    if (err)
-        return err;
-
-    // Fill in stack info.
-    uintptr_t stack_off = stack_size;
-
-    // TODO: Instead of allocating, calculate it on the fly.
-    size_t num_envp = 0;
-    for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
-        num_envp++;
-    }
-    size_t num_argv = 0;
-    for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
-        num_argv++;
-    }
-
-    size_t strings_size = 0;
-    for (size_t env = 0; env < num_envp; env++)
-        strings_size += strlen(info->envp[env]) + 1;
-    for (size_t arg = 0; arg < num_argv; arg++)
-        strings_size += strlen(info->argv[arg]) + 1;
-
-    uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
-    uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
-    if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
-        mem_free(envp_offsets);
-        mem_free(argv_offsets);
-        vmo_unref(&stack->object);
-        return ENOMEM;
-    }
-
-    for (size_t env = 0; env < num_envp; env++) {
-        const char nul = 0;
-        stack_off -= 1;
-        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
-
-        const size_t len = strlen(info->envp[env]);
-        stack_off -= len;
-        vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr);
-
-        envp_offsets[env] = stack_start + stack_off;
-    }
-
-    for (size_t arg = 0; arg < num_argv; arg++) {
-        const char nul = 0;
-        stack_off -= 1;
-        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
-
-        const size_t len = strlen(info->argv[arg]);
-        stack_off -= len;
-        vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr);
-
-        argv_offsets[arg] = stack_start + stack_off;
-    }
-
-    stack_off = ALIGN_DOWN(stack_off, 16);
-
-    // Align the stack if argc + argv + envp does not add up to 16 byte alignment.
-    if ((1 + num_argv + num_envp) & 1) {
-        stack_off -= sizeof(uintptr_t);
-        uintptr_t zero = 0;
-        vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    }
-
-#define WRITE_AUXV(auxv, value) \
-    do { \
-        stack_off -= sizeof(uintptr_t); \
-        uintptr_t auxv_val = value; \
-        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
-        stack_off -= sizeof(uintptr_t); \
-        auxv_val = auxv; \
-        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
-    } while (0)
-
-    // Write auxiliary values.
-    WRITE_AUXV(AT_NULL, 0);
-    WRITE_AUXV(AT_SECURE, 0);
-    WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
-    WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
-    WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
-    if (phdr_addr)
-        WRITE_AUXV(AT_PHDR, phdr_addr);
-
-    // envp pointers.
-    stack_off -= sizeof(uintptr_t);
-    const uintptr_t zero = 0;
-    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    for (size_t env = num_envp; env > 0; env--) {
-        stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr);
-    }
-
-    // argv pointers.
-    stack_off -= sizeof(uintptr_t);
-    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    for (size_t arg = num_argv; arg > 0; arg--) {
-        stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr);
-    }
-
-    mem_free(envp_offsets);
-    mem_free(argv_offsets);
-
-    // argc
-    stack_off -= sizeof(uintptr_t);
-    uintptr_t argc = num_argv;
-    vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
-
-    err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
-    vmo_unref(&stack->object);
-    if (err)
-        return err;
-
-    struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
-    if (!start_info)
-        return ENOMEM;
-    start_info->ip = ehdr.e_entry;
-    start_info->sp = highest - stack_size + stack_off;
-    start_info->arg = 0;
-
-    struct task* result;
-    err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
-    if (err) {
-        mem_free(start_info);
-        return err;
-    }
-
-    *out = result;
-    return 0;
-}

core/lib/meson.build

@@ -1,7 +0,0 @@
-# Freestanding libc functions that are needed in the kernel
-
-kernel_sources += files(
-    'runtime.c',
-    'stdlib.c',
-    'string.c',
-)

core/mem/vma.c

@@ -1,159 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/errno.h>
-#include <zinnia/mman.h>
-#include <zinnia/pmap.h>
-#include <zinnia/utils.h>
-#include <zinnia/vma.h>
-#include <stdckdint.h>
-
-// Try to create a new, empty virtual address space.
-errno_t vma_new(struct vma** out) {
-    debug_assert(out != nullptr, "Missing out");
-
-    errno_t res;
-    if (!out)
-        return EINVAL;
-
-    struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
-    if (!vmspace)
-        return ENOMEM;
-
-    res = pmap_new_user(&vmspace->pmap, 0);
-    if (res)
-        goto err1;
-
-    tailq_init(&vmspace->mappings);
-
-    *out = vmspace;
-    return 0;
-
-err1:
-    mem_free(vmspace);
-err0:
-    return res;
-}
-
-// Delete a virtual address space.
-void vma_free(struct vma* v) {
-    struct vma_mapping* mapping;
-    while ((mapping = tailq_first(&v->mappings)) != nullptr) {
-        tailq_remove(&v->mappings, mapping, link);
-        vmo_unref(mapping->object);
-        mem_free(mapping);
-    }
-}
-
-errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
-    panic("TODO");
-}
-
-errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
-    panic("TODO");
-}
-
-static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
-    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
-        if (page >= m->start_page && page < m->end_page)
-            return m;
-    }
-    return nullptr;
-}
-
-static bool access_allowed(enum prot_flags prot, const struct vma_fault_info* info) {
-    if (info->caused_by_write)
-        return prot & PROT_WRITE;
-    if (info->caused_by_fetch)
-        return prot & PROT_EXEC;
-    return prot & (PROT_READ | PROT_WRITE);
-}
-
-static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
-    enum pte_flags pf = 0;
-    if (prot & PROT_READ)
-        pf |= PTE_READ;
-    if (prot & PROT_WRITE)
-        pf |= PTE_WRITE;
-    if (prot & PROT_EXEC)
-        pf |= PTE_EXEC;
-    if (v->pmap.is_user)
-        pf |= PTE_USER;
-    return pf;
-}
-
-bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
-    const size_t page_size = arch_mem_page_size();
-    const size_t faulty_page = info->addr / page_size;
-
-    spin_lock(&v->lock);
-    struct vma_mapping* mapping = find_mapping(v, faulty_page);
-    if (!mapping || !access_allowed(mapping->prot, info)) {
-        spin_unlock(&v->lock);
-        return false;
-    }
-
-    struct vmo* object = mapping->object;
-    const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
-    const enum prot_flags prot = mapping->prot;
-    vmo_ref(object);
-    spin_unlock(&v->lock);
-
-    struct page* page;
-    if (object->get_page(object, object_page, &page)) {
-        vmo_unref(object);
-        return false;
-    }
-
-    spin_lock(&v->lock);
-    mapping = find_mapping(v, faulty_page);
-    const bool still_valid = mapping && mapping->object == object &&
-        (faulty_page - mapping->start_page) + mapping->offset_page == object_page &&
-        access_allowed(mapping->prot, info);
-
-    if (still_valid) {
-        errno_t status =
-            pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
-        assert(status == 0, "Failed to map a demand-loaded page\n");
-    }
-    spin_unlock(&v->lock);
-
-    vmo_unref(object);
-    return true;
-}
-
-errno_t vma_map(
-    struct vma* vma,
-    struct vmo* vmo,
-    uintptr_t addr,
-    size_t len,
-    enum prot_flags flags,
-    uintptr_t vmo_offset
-) {
-    if (!vma || !vmo)
-        return EINVAL;
-
-    uintptr_t end;
-    if (ckd_add(&end, addr, len))
-        return ENOMEM;
-
-    const size_t page_size = arch_mem_page_size();
-    if (addr % page_size != vmo_offset % page_size)
-        return EINVAL;
-
-    struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
-    if (!mapping)
-        return ENOMEM;
-
-    vmo_ref(vmo);
-    mapping->start_page = addr / page_size;
-    mapping->end_page = ALIGN_UP(end, page_size) / page_size;
-    mapping->offset_page = vmo_offset / page_size;
-    mapping->object = vmo;
-    mapping->prot = flags;
-
-    spin_lock(&vma->lock);
-    tailq_insert_head(&vma->mappings, mapping, link);
-    spin_unlock(&vma->lock);
-
-    return 0;
-}

core/mem/vmo.c

@@ -1,245 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/errno.h>
-#include <zinnia/list.h>
-#include <zinnia/utils.h>
-#include <zinnia/vmo.h>
-#include <zinnia/string.h>
-
-void vmo_ref(struct vmo* obj) {
-    refcount_inc(&obj->rc);
-}
-
-void vmo_unref(struct vmo* obj) {
-    if (refcount_dec(&obj->rc))
-        obj->destroy(obj);
-}
-
-errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
-
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
-
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_read)
-        *actual_read = progress;
-    return status;
-}
-
-errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
-
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
-
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_written)
-        *actual_written = progress;
-    return status;
-}
-
-errno_t vmo_copy(
-    struct vmo* target,
-    uintptr_t target_offset,
-    struct vmo* src,
-    uintptr_t src_offset,
-    size_t len,
-    size_t* actual_copied
-) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t target_misalign = (progress + target_offset) % page_size;
-        const size_t src_misalign = (progress + src_offset) % page_size;
-        const size_t target_page_index = (progress + target_offset) / page_size;
-        const size_t src_page_index = (progress + src_offset) / page_size;
-
-        const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
-
-        struct page* tpage;
-        status = target->get_page(target, target_page_index, &tpage);
-        if (status != 0)
-            break;
-
-        struct page* spage;
-        status = src->get_page(src, src_page_index, &spage);
-        if (status != 0)
-            break;
-
-        uintptr_t taddr = vm_page_addr(tpage);
-        uintptr_t saddr = vm_page_addr(spage);
-
-        memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_copied)
-        *actual_copied = progress;
-    return status;
-}
-
-static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    // Try to find the requested page in the cache.
-    struct page_list* iter;
-    spin_lock(&paged->lock);
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
-            continue;
-
-        // We found a cached page!
-        *out = iter->value;
-        spin_unlock(&paged->lock);
-        return 0;
-    }
-    spin_unlock(&paged->lock);
-
-    // Get page from out pager if we don't have one already.
-    struct page* new_page;
-    errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
-    if (status)
-        return status;
-
-    struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
-    if (!entry) {
-        paged->source.put_page(&paged->source, offset_idx, new_page);
-        return ENOMEM;
-    }
-
-    entry->offset = offset_idx;
-    entry->value = new_page;
-
-    spin_lock(&paged->lock);
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
-            continue;
-
-        *out = iter->value;
-        spin_unlock(&paged->lock);
-        mem_free(entry);
-        paged->source.put_page(&paged->source, offset_idx, new_page);
-        return 0;
-    }
-
-    // Add page to the object cache.
-    list_insert_head(&paged->cache, entry, next);
-    spin_unlock(&paged->lock);
-
-    *out = new_page;
-    return 0;
-}
-
-static void paged_destroy(struct vmo* vmo) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    struct page_list* iter = list_first(&paged->cache);
-    while (iter) {
-        struct page_list* next = list_next(iter, next);
-        paged->source.put_page(&paged->source, iter->offset, iter->value);
-        mem_free(iter);
-        iter = next;
-    }
-
-    mem_free(paged);
-}
-
-errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
-    struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
-    if (!result)
-        return ENOMEM;
-
-    refcount_init(&result->object.rc);
-    result->source = pager;
-    result->object.get_page = paged_get_page;
-    result->object.destroy = paged_destroy;
-
-    *out = result;
-    return 0;
-}
-
-static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
-    phys_t addr;
-    errno_t status = mem_phys_alloc(1, 0, &addr);
-    if (status != 0)
-        return status;
-    size_t idx = addr / arch_mem_page_size();
-    *out = &vm_pfndb[idx];
-    return 0;
-}
-
-static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
-    return mem_phys_free(vm_page_addr(page), 1);
-}
-
-static const struct pager_ops phys_pager = {
-    .get_page = phys_get_page,
-    .put_page = phys_put_page,
-};
-
-errno_t vmo_new_phys(struct paged_vmo** out) {
-    return vmo_new_paged(phys_pager, out);
-}
-
-struct phys_range_vmo {
-    struct vmo object;
-    uintptr_t start;
-    size_t len;
-};
-
-static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
-    if (offset_idx >= p->len / arch_mem_page_size())
-        return EINVAL;
-    *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
-    return 0;
-}
-
-static void phys_range_destroy(struct vmo* vmo) {
-    mem_free(containerof(vmo, struct phys_range_vmo, object));
-}
-
-errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
-    struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
-    if (!vmo)
-        return ENOMEM;
-
-    refcount_init(&vmo->object.rc);
-    vmo->object.get_page = phys_range_get_page;
-    vmo->object.destroy = phys_range_destroy;
-    vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
-    vmo->len = ALIGN_UP(length, arch_mem_page_size());
-
-    *out = &vmo->object;
-    return 0;
-}

core/meson.build

@@ -1,16 +0,0 @@
-kernel_sources += files(
-    'clock.c',
-    'console.c',
-    'elf.c',
-    'irq.c',
-    'module.c',
-    'panic.c',
-    'percpu.c',
-    'print.c',
-    'spin.c',
-    'wait.c',
-)
-
-subdir('lib')
-subdir('mem')
-subdir('sched')

core/module.c

@@ -1,12 +0,0 @@
-#include <zinnia/assert.h>
-#include <zinnia/module.h>
-
-extern errno_t (*__ld_mod_entry_start[])(struct module*);
-extern errno_t (*__ld_mod_entry_end[])(struct module*);
-
-void module_init() {
-    for (auto start = __ld_mod_entry_start; start < __ld_mod_entry_end; start += 1) {
-        errno_t mod_status = (*start)(nullptr);
-        assert(mod_status == EOK, "Built-in entry point returned %i", mod_status);
-    }
-}

core/panic.c

@@ -1,36 +0,0 @@
-#include <zinnia/irq.h>
-#include <zinnia/panic.h>
-#include <zinnia/percpu.h>
-#include <zinnia/print.h>
-#include <stdarg.h>
-
-#define panic_log(msg, ...) kprintf("[    !!!!    ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
-
-[[noreturn]]
-void panic(const char* msg, ...) {
-    // TODO: Stop all other CPUs with IPI.
-    irq_set_state(false);
-
-    struct percpu* cpu = percpu_get();
-    struct task* current = cpu->sched.current;
-    kprintf(
-        "[    !!!!    ] \e[31m"
-        "Kernel panic on CPU %zu (T%zu, \"%s\"): ",
-        cpu->id,
-        current ? current->id : 0,
-        current ? current->name : nullptr
-    );
-
-    va_list args;
-    va_start(args, msg);
-    kvprintf(msg, args);
-    va_end(args);
-
-    kprintf("\e[0m");
-
-    panic_log("End of panic message\n");
-
-    while (1) {
-        irq_wait();
-    }
-}

core/percpu.c

@@ -1,57 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/cmdline.h>
-#include <zinnia/compiler.h>
-#include <zinnia/percpu.h>
-#include <zinnia/string.h>
-#include <zinnia/utils.h>
-#include <arch/percpu.h>
-#include <stdatomic.h>
-
-extern uint8_t __ld_percpu_start[];
-extern uint8_t __ld_percpu_end[];
-
-static _Atomic(size_t) num_cpus = 1;
-
-static size_t max_cpus = SIZE_MAX;
-static void smp_option(uintptr_t value) {
-    max_cpus = value;
-}
-CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option);
-
-[[__used, __section(".percpu.init")]]
-struct percpu percpu_bsp = {
-    .self = &percpu_bsp,
-    .id = 0,
-    .online = false,
-};
-
-struct percpu* percpu_new() {
-    const size_t size = __ld_percpu_end - __ld_percpu_start;
-    const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
-
-    phys_t mem;
-    if (mem_phys_alloc(num_pages, 0, &mem))
-        return nullptr;
-
-    struct percpu* cpu = HHDM_PTR(mem);
-    cpu->self = cpu;
-    cpu->id = atomic_fetch_add_explicit(&num_cpus, 1, memory_order_relaxed);
-    cpu->online = false;
-    return cpu;
-}
-
-size_t percpu_max_cpus() {
-    return max_cpus;
-}
-
-struct percpu* percpu_get() {
-    return arch_percpu_get();
-}
-
-void percpu_bsp_init() {
-    arch_percpu_bsp_init();
-}
-
-void percpu_init() {
-    arch_percpu_init();
-}

core/sched/core.c

@@ -1 +0,0 @@
-#include <zinnia/sched.h>

core/sched/sched_dumb.c

@@ -1,167 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/errno.h>
-#include <zinnia/irq.h>
-#include <zinnia/list.h>
-#include <zinnia/percpu.h>
-#include <zinnia/pmap.h>
-#include <zinnia/print.h>
-#include <zinnia/sched.h>
-#include <zinnia/spin.h>
-#include <zinnia/stack.h>
-#include <zinnia/string.h>
-#include <zinnia/tailq.h>
-#include <zinnia/vma.h>
-#include <arch/sched.h>
-#include <stdatomic.h>
-#include <stdint.h>
-
-static void idle_func(uintptr_t) {
-    while (1) {
-        irq_set_state(true);
-        irq_wait();
-    }
-}
-
-void sched_to_user(uintptr_t info_ptr) {
-    struct task_start_info* info = (struct task_start_info*)info_ptr;
-    uintptr_t ip = info->ip;
-    uintptr_t sp = info->sp;
-    uintptr_t arg = info->arg;
-    mem_free(info);
-    arch_sched_jump_to_user(ip, sp, arg);
-}
-
-void sched_to_user_context(uintptr_t ctx) {
-    arch_sched_jump_to_context((struct arch_context*)ctx);
-}
-
-void sched_init(struct scheduler* sched) {
-    struct task* idle_task;
-    assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
-    sched->idle_task = idle_task;
-
-    struct task* boot_task = mem_alloc(sizeof(struct task), 0);
-    assert(boot_task != nullptr, "Unable to create boot task!\n");
-    strncpy(boot_task->name, "boot", sizeof(boot_task->name));
-    boot_task->space = &kernel_vma;
-    sched->current = boot_task;
-
-    tailq_init(&sched->run_queue);
-
-    log("sched: Scheduler initialized\n");
-}
-
-void sched_add_task(struct scheduler* sched, struct task* task) {
-    assert(task != nullptr, "Tried to add NULL to the run queue\n");
-    tailq_insert_tail(&sched->run_queue, task, next);
-}
-
-static struct task* next_task(struct scheduler* sched) {
-    struct task* t = tailq_first(&sched->run_queue);
-
-    if (t == nullptr)
-        return sched->idle_task;
-
-    tailq_remove(&sched->run_queue, t, next);
-
-    return t;
-}
-
-static void finish_switch(struct task* prev) {
-    if (prev->state == TASK_STATE_DEAD)
-        task_destroy(prev);
-}
-
-static void do_reschedule(struct scheduler* sched) {
-    struct task* from = sched->current;
-    struct task* to = next_task(sched);
-
-    // We'd end up on the same task, skip switching altogether.
-    if (from == to) {
-        irq_unlock();
-        return;
-    }
-
-    sched->current = to;
-    pmap_set(&to->space->pmap);
-
-    struct percpu* cpu = percpu_get();
-    from->user_stack = cpu->user_stack;
-    cpu->kernel_stack = kernel_stack_top(&to->kernel_stack);
-    cpu->user_stack = to->user_stack;
-
-    struct irq_percpu irq_state = *percpu_ref(irq_data);
-    struct task* prev = arch_sched_switch(from, to);
-    *percpu_ref(irq_data) = irq_state;
-    finish_switch(prev);
-    irq_unlock();
-}
-
-void sched_reschedule(struct scheduler* sched) {
-    irq_lock();
-    if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD)
-        sched_add_task(sched, sched->current);
-    do_reschedule(sched);
-}
-
-void sched_yield(struct scheduler* sched) {
-    irq_lock();
-    do_reschedule(sched);
-}
-
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
-    if (!out)
-        return EINVAL;
-
-    struct task* new_task = mem_alloc(sizeof(struct task), 0);
-    if (new_task == nullptr)
-        return ENOMEM;
-
-    static _Atomic(size_t) last_tid = 0;
-    new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire);
-    new_task->space = space;
-    strncpy(new_task->name, name, sizeof(new_task->name));
-
-    // Allocate a kernel stack.
-    errno_t s = stack_init(&new_task->kernel_stack, KERNEL_STACK_SIZE);
-    if (s != 0) {
-        mem_free(new_task);
-        return s;
-    }
-    new_task->user_stack = 0;
-
-    // Initialize the arch-dependent parts.
-    s = arch_task_init(&new_task->context, entry, arg0, kernel_stack_top(&new_task->kernel_stack), true);
-    if (s != 0) {
-        stack_destroy(&new_task->kernel_stack);
-        mem_free(new_task);
-        return s;
-    }
-
-    *out = new_task;
-    return 0;
-}
-
-void task_destroy(struct task* task) {
-    arch_task_fini(&task->context);
-    stack_destroy(&task->kernel_stack);
-    mem_free(task);
-}
-
-void task_entry(task_fn_t entry, uintptr_t arg0, struct task* prev) {
-    finish_switch(prev);
-
-    *percpu_ref(irq_data) = (struct irq_percpu){
-        .depth = 1,
-        .saved_state = true,
-    };
-    irq_unlock();
-
-    entry(arg0);
-
-    struct scheduler* sched = &percpu_get()->sched;
-    sched->current->state = TASK_STATE_DEAD;
-    sched_yield(sched);
-    panic("unreachable");
-}

core/spin.c

@@ -1,15 +0,0 @@
-#include <zinnia/spin.h>
-#include <stdatomic.h>
-
-void spin_lock(struct spinlock* spin) {
-    uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed);
-
-    while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) {
-        asm volatile("pause");
-    }
-}
-
-void spin_unlock(struct spinlock* spin) {
-    uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed);
-    atomic_store_explicit(&spin->owner, val + 1, memory_order_release);
-}

core/wait.c

@@ -1,17 +0,0 @@
-#include <zinnia/waitq.h>
-
-void waitq_init(struct waitq* wq) {
-    tailq_init(&wq->waiters);
-}
-
-void waitq_wait(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_one(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_all(struct waitq* wq) {
-    // TODO
-}

dev/Kconfig

@@ -14,5 +14,6 @@ config ACPI
 
 source "dev/block/Kconfig"
 source "dev/pci/Kconfig"
+source "dev/video/Kconfig"
 
 endmenu

dev/acpi/include/uacpi_types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdarg.h>

dev/acpi/include/zinnia/acpi.h

@@ -1,8 +1,19 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/init.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/types.h>
+#include <stdint.h>
 
-void acpi_init(struct boot_info* info);
+INIT_EXTERN(acpi_stage);
 
 extern phys_t acpi_rsdp_addr;
+
+struct acpi_match {
+    const char* hid;
+    uintptr_t data;
+};
+
+#define ACPI_MATCH_HID(id) .hid = (id)

dev/acpi/init.c

@@ -1,25 +1,47 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "acpi: " fmt
+
 #include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <zinnia/types.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <uacpi/uacpi.h>
 
 phys_t acpi_rsdp_addr = 0;
 
+static char table_buf[0x1000];
+
 static bool use_acpi = true;
 void use_acpi_option(bool use) {
     use_acpi = use;
 }
 CMDLINE_OPTION("acpi", "Enable ACPI firmware subsystem", use_acpi_option);
 
-void acpi_init(struct boot_info* info) {
+[[__init]]
+static bool acpi_stage_init(struct boot_info* info) {
     if (!use_acpi) {
-        log("acpi: ACPI disabled on the command line!\n");
-        return;
+        log_warn("ACPI disabled on the command line!\n");
+        return false;
     }
 
     if (!info->rsdp) {
-        log("acpi: No RSDP was passed, unable to start ACPI subsystem!\n");
-        return;
+        log_warn("No RSDP was passed, unable to start ACPI subsystem!\n");
+        return false;
     }
     acpi_rsdp_addr = info->rsdp;
+
+    if (uacpi_setup_early_table_access(table_buf, sizeof(table_buf)) != UACPI_STATUS_OK) {
+        log_err("Failed to set up early table access!\n");
+        return false;
+    }
+
+    return true;
 }
+INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
+INIT_DEPENDS(acpi_stage, mem_stage);
+INIT_ENTAILS(acpi_stage, percpu_stage);

dev/acpi/mcfg.c

@@ -0,0 +1,32 @@
+#define pr_fmt(fmt) "acpi: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/pci.h>
+#include <zinnia/print.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+
+[[__init]]
+static void acpi_mcfg_stage_init(struct boot_info*) {
+    uacpi_table table = {};
+    if (uacpi_table_find_by_signature(ACPI_MCFG_SIGNATURE, &table) != UACPI_STATUS_OK)
+        return;
+
+    struct acpi_mcfg* mcfg = table.ptr;
+    size_t count = 0;
+    if (mcfg->hdr.length > sizeof(*mcfg))
+        count = (mcfg->hdr.length - sizeof(*mcfg)) / sizeof(struct acpi_mcfg_allocation);
+
+    for (size_t i = 0; i < count; i++) {
+        struct acpi_mcfg_allocation* entry = &mcfg->entries[i];
+        errno_t status = pci_ecam_register(entry->address, entry->segment, entry->start_bus, entry->end_bus);
+        if (status != 0)
+            log_warn("Failed to register ECAM for segment %hu: %i\n", entry->segment, status);
+    }
+
+    uacpi_table_unref(&table);
+}
+INIT_TASK(acpi_mcfg_stage, "dev.acpi.mcfg", acpi_mcfg_stage_init);
+INIT_DEPENDS(acpi_mcfg_stage, acpi_stage);
+INIT_ENTAILS(acpi_mcfg_stage, pci_stage);

dev/acpi/meson.build

@@ -6,6 +6,10 @@ kernel_sources += files(
     'uacpi_glue.c',
 )
 
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('mcfg.c')
+endif
+
 kernel_includes += uacpi.get_variable('includes')
 kernel_includes += include_directories('include')
 

dev/acpi/uacpi_glue.c

@@ -1,4 +1,7 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
 #include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/mmio.h>
@@ -28,27 +31,45 @@ void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
         return nullptr;
 
     for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) {
-        pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PROT_READ | PROT_WRITE, CACHE_ANY);
+        pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PTE_READ | PTE_WRITE, CACHE_ANY);
     }
 
     return (void*)(vaddr + difference);
 }
 
 void uacpi_kernel_unmap(void* addr, uacpi_size len) {
-    // TODO
+    uintptr_t aligned_addr = ALIGN_DOWN((uintptr_t)addr, arch_mem_page_size());
+    uintptr_t difference = (uintptr_t)addr - aligned_addr;
+    size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
+
+    for (size_t i = 0; i < aligned_len; i += arch_mem_page_size())
+        pmap_unmap(&kernel_vma.pmap, aligned_addr + i);
+    vm_free(aligned_addr, aligned_len);
+}
+
+static enum log_level acpi_log_level(uacpi_log_level lvl) {
+    switch (lvl) {
+    case UACPI_LOG_ERROR:
+        return LOG_ERR;
+
+    case UACPI_LOG_WARN:
+        return LOG_WARN;
+
+    case UACPI_LOG_INFO:
+        return LOG_INFO;
+
+    default:
+        return LOG_DEBUG;
+    }
 }
 
 void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
-    log("acpi: ");
-
     va_list args;
     va_start(args, msg);
-
-    kvprintf(msg, args);
-
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
     va_end(args);
 }
 
 void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
-    kvprintf(msg, args);
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
 }

dev/block/bio.c

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/utils.h>
+
+struct bio* bio_alloc(enum bio_op op, size_t num_segs) {
+    struct bio* bio = mem_alloc(sizeof(struct bio) + num_segs * sizeof(struct bio_seg), 0);
+    if (!bio)
+        return nullptr;
+    bio->op = op;
+    bio->num_segs = num_segs;
+    event_init(&bio->event);
+    atomic_init(&bio->pending, 0);
+    atomic_init(&bio->short_offset, UINT64_MAX);
+    atomic_init(&bio->child_status, 0);
+    return bio;
+}
+
+void bio_free(struct bio* bio) {
+    mem_free(bio);
+}
+
+static void record_short(struct bio* parent, uint64_t offset, errno_t status) {
+    uint64_t current = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    while (offset < current) {
+        if (atomic_compare_exchange_weak(&parent->short_offset, &current, offset, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            break;
+    }
+    if (status) {
+        errno_t expected = 0;
+        atomic_compare_exchange(&parent->child_status, &expected, status, ATOMIC_RELAXED, ATOMIC_RELAXED);
+    }
+}
+
+static void parent_put(struct bio* parent) {
+    if (atomic_fetch_sub(&parent->pending, 1, ATOMIC_ACQ_REL) != 1)
+        return;
+    const uint64_t offset = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    const size_t done = offset == UINT64_MAX ? parent->num_lbas : offset;
+    const errno_t status = atomic_load(&parent->child_status, ATOMIC_RELAXED);
+    bio_complete(parent, done == 0 ? status : 0, done);
+}
+
+void bio_complete(struct bio* bio, errno_t status, size_t done_lbas) {
+    bio->status = status;
+    bio->done_lbas = done_lbas;
+
+    struct bio* parent = bio->parent;
+    if (parent) {
+        if (status || done_lbas < bio->num_lbas)
+            record_short(parent, bio->parent_offset + done_lbas, status);
+        bio_free(bio);
+        parent_put(parent);
+        return;
+    }
+
+    if (bio->end)
+        bio->end(bio);
+    else
+        event_set(&bio->event);
+}
+
+errno_t bio_wait(struct bio* bio, size_t* out_lbas) {
+    event_wait(&bio->event);
+    if (out_lbas)
+        *out_lbas = bio->done_lbas;
+    return bio->status;
+}
+
+// Submits one bio and completes early if it failed.
+static void submit_one(struct blockdev* dev, struct bio* bio) {
+    errno_t status = dev->ops->submit(dev, bio);
+    if (status)
+        bio_complete(bio, status, 0);
+}
+
+void bio_submit(struct blockdev* dev, struct bio* bio) {
+    bio->dev = dev;
+    if (bio->op == BIO_FLUSH) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    if (bio->op == BIO_READ) {
+        if (bio->lba >= dev->lba_count) {
+            bio_complete(bio, 0, 0);
+            return;
+        }
+        bio->num_lbas = MIN(bio->num_lbas, dev->lba_count - bio->lba);
+    }
+    if (bio->num_lbas == 0) {
+        bio_complete(bio, 0, 0);
+        return;
+    }
+
+    const size_t lba_size = dev->lba_size;
+    const size_t lbas_per_page = MAX(arch_mem_page_size() / lba_size, 1);
+    size_t max_lbas = dev->limits.max_lbas ? dev->limits.max_lbas : SIZE_MAX;
+    if (max_lbas > lbas_per_page)
+        max_lbas -= max_lbas % lbas_per_page;
+    const size_t max_segs = dev->limits.max_segs ? dev->limits.max_segs : SIZE_MAX;
+
+    if (bio->num_lbas <= max_lbas && bio->num_segs <= max_segs) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    atomic_store(&bio->pending, 1, ATOMIC_RELAXED);
+    atomic_store(&bio->short_offset, UINT64_MAX, ATOMIC_RELAXED);
+    atomic_store(&bio->child_status, 0, ATOMIC_RELAXED);
+
+    size_t seg = 0;
+    size_t seg_off = 0;
+    uint64_t consumed = 0;
+    while (consumed < bio->num_lbas) {
+        const size_t cap = MIN(max_segs, bio->num_segs - seg);
+        struct bio* child = cap ? bio_alloc(bio->op, cap) : nullptr;
+        if (!child) {
+            record_short(bio, consumed, cap ? ENOMEM : EINVAL);
+            break;
+        }
+
+        size_t count = 0;
+        size_t lbas = 0;
+        while (lbas < max_lbas && count < cap && consumed + lbas < bio->num_lbas) {
+            const struct bio_seg* src = &bio->segs[seg];
+            const size_t take =
+                MIN(MIN(src->len - seg_off, (max_lbas - lbas) * lba_size),
+                    (bio->num_lbas - consumed - lbas) * lba_size);
+            if (take < lba_size)
+                break;
+            child->segs[count++] = (struct bio_seg){.addr = src->addr + seg_off, .len = take};
+            lbas += take / lba_size;
+            seg_off += take;
+            if (seg_off == src->len) {
+                seg++;
+                seg_off = 0;
+            }
+        }
+
+        if (lbas == 0) {
+            bio_free(child);
+            record_short(bio, consumed, EINVAL);
+            break;
+        }
+
+        child->num_segs = count;
+        child->num_lbas = lbas;
+        child->lba = bio->lba + consumed;
+        child->parent = bio;
+        child->parent_offset = consumed;
+        child->dev = dev;
+        consumed += lbas;
+        atomic_fetch_add(&bio->pending, 1, ATOMIC_RELAXED);
+        submit_one(dev, child);
+    }
+
+    parent_put(bio);
+}

dev/block/blockdev.c

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "block: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/tailq.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vec.h>
+#include <zinnia/vma.h>
+#include <string.h>
+#include <uapi/block.h>
+
+static size_t lbas_per_page(struct blockdev* dev) {
+    return arch_mem_page_size() / dev->lba_size;
+}
+
+static errno_t cache_get_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t page_size = arch_mem_page_size();
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+
+    if (lba >= dev->lba_count) {
+        *out_valid = 0;
+        return 0;
+    }
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_READ, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+
+    const size_t valid = done / per_page + (done % per_page != 0);
+    if (done % per_page != 0) {
+        const size_t tail = (done % per_page) * dev->lba_size;
+        memset((uint8_t*)HHDM_PTR(vm_page_addr(pages[valid - 1])) + tail, 0, page_size - tail);
+    }
+    *out_valid = valid;
+    return valid == 0 && num_lbas > 0 ? EIO : 0;
+}
+
+static errno_t cache_put_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+    if (lba >= dev->lba_count)
+        return 0;
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_WRITE, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+    return done < num_lbas ? EIO : 0;
+}
+
+static void cache_destroy(struct vmo* vmo) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    if (dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static const struct vmo_pager_ops cache_pager = {
+    .get_pages = cache_get_pages,
+    .put_pages = cache_put_pages,
+    .max_cluster = 64,
+    .destroy = cache_destroy,
+};
+
+void blockdev_init(
+    struct blockdev* dev,
+    const struct blockdev_ops* ops,
+    uint32_t major,
+    uint32_t minor,
+    size_t lba_size,
+    uint64_t lba_count
+) {
+    vmo_init(&dev->cache, &cache_pager);
+    dev->ops = ops;
+    dev->dev = dev_make(major, minor);
+    dev->lba_size = lba_size;
+    dev->lba_count = lba_count;
+    atomic_init(&dev->claimed, false);
+}
+
+void blockdev_ref(struct blockdev* dev) {
+    vmo_ref(&dev->cache);
+}
+
+void blockdev_unref(struct blockdev* dev) {
+    vmo_unref(&dev->cache);
+}
+
+errno_t blockdev_rw(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    const struct bio_seg* segs,
+    size_t num_segs,
+    size_t* out_lbas
+) {
+    struct bio* bio = bio_alloc(op, num_segs);
+    if (!bio)
+        return ENOMEM;
+    memcpy(bio->segs, segs, num_segs * sizeof(struct bio_seg));
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_rw_pages(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    struct page** pages,
+    size_t count,
+    size_t* out_lbas
+) {
+    const size_t page_size = arch_mem_page_size();
+    size_t bytes = num_lbas * dev->lba_size;
+    if (bytes > count * page_size)
+        return EINVAL;
+
+    struct bio* bio = bio_alloc(op, count);
+    if (!bio)
+        return ENOMEM;
+
+    size_t num_segs = 0;
+    for (size_t i = 0; i < count && bytes > 0; i++) {
+        const size_t len = MIN(bytes, page_size);
+        bio->segs[num_segs++] = (struct bio_seg){.addr = vm_page_addr(pages[i]), .len = len};
+        bytes -= len;
+    }
+    bio->num_segs = num_segs;
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_flush(struct blockdev* dev) {
+    struct bio* bio = bio_alloc(BIO_FLUSH, 0);
+    if (!bio)
+        return ENOMEM;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, nullptr);
+    bio_free(bio);
+    return status;
+}
+
+static struct blockdev* file_dev(struct vfs_file* file) {
+    return file->priv;
+}
+
+static uint64_t dev_size(struct blockdev* dev) {
+    return dev->lba_count * dev->lba_size;
+}
+
+static errno_t blockdev_file_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct blockdev* dev = file_dev(file);
+    *out_read = 0;
+    if (offset >= dev_size(dev))
+        return 0;
+
+    size_t progress;
+    errno_t status =
+        vmo_read_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_read = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct blockdev* dev = file_dev(file);
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (offset >= dev_size(dev))
+        return ENOSPC;
+
+    size_t progress;
+    errno_t status =
+        vmo_write_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_written = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    struct blockdev* dev = file_dev(file);
+    switch (request) {
+    case __BLKGETSIZE64: {
+        const uint64_t size = dev_size(dev);
+        if (!usercopy_write((__user void*)arg, &size, sizeof(size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKSSZGET: {
+        const unsigned int lba_size = dev->lba_size;
+        if (!usercopy_write((__user void*)arg, &lba_size, sizeof(lba_size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKRRPART: {
+        errno_t status = blockdev_rescan(dev);
+        if (!status)
+            *out_result = 0;
+        return status;
+    }
+    default:
+        if (dev->ops->ioctl)
+            return dev->ops->ioctl(dev, file, request, arg, out_result);
+        return ENOTTY;
+    }
+}
+
+static errno_t blockdev_file_sync(struct vfs_file* file, bool data_only) {
+    struct blockdev* dev = file_dev(file);
+    errno_t status = vmo_sync(&dev->cache, 0, UINT64_MAX);
+    if (status)
+        return status;
+    return blockdev_flush(dev);
+}
+
+static errno_t blockdev_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    errno_t status =
+        vma_map(vma, &file_dev(file)->cache, addr, len, prot, (flags & MAP_SHARED) ? VMA_SHARED : 0, offset);
+    if (status)
+        return status;
+    *out_addr = addr;
+    return 0;
+}
+
+static void blockdev_file_release(struct vfs_file* file) {
+    blockdev_unref(file_dev(file));
+}
+
+static const struct vfs_file_ops blockdev_file_ops = {
+    .read = blockdev_file_read,
+    .write = blockdev_file_write,
+    .ioctl = blockdev_file_ioctl,
+    .sync = blockdev_file_sync,
+    .mmap = blockdev_file_mmap,
+    .release = blockdev_file_release,
+};
+
+static errno_t blockdev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    file->priv = dev;
+    file->ops = &blockdev_file_ops;
+    return 0;
+}
+
+static void blockdev_node_destroy(struct vfs_node* node) {
+    blockdev_unref(node->priv);
+}
+
+static const struct vfs_node_ops blockdev_node_ops = {
+    .destroy = blockdev_node_destroy,
+    .device = {
+        .open = blockdev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_BLOCKDEV,
+        .ops = &blockdev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    blockdev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        blockdev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out) {
+    if (node->type != VNODE_BLOCKDEV || node->ops != &blockdev_node_ops)
+        return ENOTBLK;
+
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    *out = dev;
+    return 0;
+}
+
+errno_t blockdev_claim(struct blockdev* dev) {
+    bool expected = false;
+    if (!atomic_compare_exchange(&dev->claimed, &expected, true, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return EBUSY;
+    return 0;
+}
+
+void blockdev_release(struct blockdev* dev) {
+    atomic_store(&dev->claimed, false, ATOMIC_RELEASE);
+}
+
+struct disk_node {
+    char name[64];
+    struct blockdev* dev;
+};
+
+struct disk {
+    char name[32];
+    struct blockdev* dev;
+    vec(struct disk_node) nodes;
+    tailq_link(struct disk) link;
+};
+
+static tailq_head(struct disk) disks;
+static mutex_t disks_lock;
+
+static void add_node(struct disk* disk, const char* name, struct blockdev* dev) {
+    struct disk_node node = {.dev = dev};
+    ksnprintf(node.name, sizeof(node.name), "%s", name);
+    if (vec_push(&disk->nodes, node) && dev)
+        blockdev_unref(dev);
+}
+
+static void scan_partitions(struct disk* disk) {
+    struct gpt_partition* parts;
+    size_t count;
+    if (gpt_scan(disk->dev, &parts, &count))
+        return;
+
+    for (size_t i = 0; i < count; i++) {
+        const struct gpt_partition* part = &parts[i];
+        if (part->end_lba < part->start_lba || part->end_lba >= disk->dev->lba_count) {
+            log_warn("Ignoring partition %zu of \"%s\" with invalid range\n", i + 1, disk->name);
+            continue;
+        }
+
+        char part_name[48];
+        ksnprintf(part_name, sizeof(part_name), "%sp%zu", disk->name, i + 1);
+
+        struct blockdev* dev;
+        errno_t status = partition_new(disk->dev, part->start_lba, part->end_lba - part->start_lba + 1, &dev);
+        if (status) {
+            log_warn("Unable to create partition \"%s\" (%i)\n", part_name, status);
+            continue;
+        }
+
+        char path[64];
+        ksnprintf(path, sizeof(path), "block/%s", part_name);
+        status = devtmpfs_add_blockdev(path, dev, 0660);
+        if (status) {
+            blockdev_unref(dev);
+            log_warn("Unable to register \"%s\" (%i)\n", path, status);
+            continue;
+        }
+        add_node(disk, path, dev);
+
+        char type_str[GUID_STR_LEN];
+        char uuid_str[GUID_STR_LEN];
+        guid_format(&part->type_guid, type_str);
+        guid_format(&part->unique_guid, uuid_str);
+
+        ksnprintf(path, sizeof(path), "block/parttype-%s", type_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+        else if (status == EEXIST)
+            log("Partition type %s already claimed, no \"%s\" link for \"%s\"\n", type_str, path, part_name);
+
+        ksnprintf(path, sizeof(path), "block/partuuid-%s", uuid_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+
+        log("Partition %zu: \"%s\" Type: %s UUID: %s\n", i + 1, part_name, type_str, uuid_str);
+    }
+
+    mem_free(parts);
+}
+
+errno_t blockdev_register(const char* name, struct blockdev* dev) {
+    struct disk* disk = mem_alloc(sizeof(struct disk), 0);
+    if (!disk)
+        return ENOMEM;
+    ksnprintf(disk->name, sizeof(disk->name), "%s", name);
+    vec_init(&disk->nodes);
+
+    char path[64];
+    ksnprintf(path, sizeof(path), "block/%s", name);
+    errno_t status = devtmpfs_add_blockdev(path, dev, 0660);
+    if (status) {
+        mem_free(disk);
+        return status;
+    }
+
+    blockdev_ref(dev);
+    disk->dev = dev;
+    log("Registered block device \"%s\"\n", name);
+
+    mutex_lock(&disks_lock);
+    tailq_insert_tail(&disks, disk, link);
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_rescan(struct blockdev* dev) {
+    mutex_lock(&disks_lock);
+    struct disk* disk;
+    for (disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        if (disk->dev == dev)
+            break;
+    }
+    if (!disk) {
+        mutex_unlock(&disks_lock);
+        return ENOTTY;
+    }
+
+    for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+        struct disk_node* node = &vec_elem_get(&disk->nodes, i);
+        errno_t status = devtmpfs_remove(node->name);
+        if (status)
+            log_warn("Unable to remove stale node \"%s\" (%i)\n", node->name, status);
+        if (node->dev)
+            blockdev_unref(node->dev);
+    }
+    disk->nodes.length = 0;
+
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_sync_all() {
+    errno_t result = 0;
+    mutex_lock(&disks_lock);
+    for (struct disk* disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+            struct blockdev* part = vec_elem_get(&disk->nodes, i).dev;
+            errno_t status = part ? vmo_sync(&part->cache, 0, UINT64_MAX) : 0;
+            if (status && !result)
+                result = status;
+        }
+        errno_t status = vmo_sync(&disk->dev->cache, 0, UINT64_MAX);
+        if (!status)
+            status = blockdev_flush(disk->dev);
+        if (status && !result)
+            result = status;
+    }
+    mutex_unlock(&disks_lock);
+    return result;
+}
+
+static void block_stage_init(struct boot_info* info) {
+    tailq_init(&disks);
+    mutex_init(&disks_lock);
+    errno_t status = devtmpfs_mkdir("block", 0755);
+    if (status && status != EEXIST)
+        log_warn("Unable to create /dev/block (%i)\n", status);
+}
+
+INIT_TASK(block_stage, "dev.block", block_stage_init);
+INIT_DEPENDS(block_stage, devtmpfs_stage);

dev/block/gpt.c

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+struct [[__packed]] gpt_header {
+    char signature[8];
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t header_crc32;
+    uint32_t reserved;
+    uint64_t my_lba;
+    uint64_t alternate_lba;
+    uint64_t first_usable_lba;
+    uint64_t last_usable_lba;
+    struct guid disk_guid;
+    uint64_t partition_entry_lba;
+    uint32_t num_partition_entries;
+    uint32_t partition_entry_size;
+    uint32_t partition_entry_array_crc32;
+};
+
+struct [[__packed]] gpt_entry {
+    struct guid type_guid;
+    struct guid unique_guid;
+    uint64_t start_lba;
+    uint64_t end_lba;
+    uint64_t attributes;
+    uint16_t name[36];
+};
+
+constexpr size_t GPT_MAX_ENTRIES = 1024;
+
+errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count) {
+    struct gpt_header header;
+    size_t read;
+    errno_t status = vmo_read(&dev->cache, dev->lba_size, &header, sizeof(header), &read);
+    if (status)
+        return status;
+    if (read != sizeof(header) || memcmp(header.signature, "EFI PART", 8) != 0)
+        return ENODATA;
+    if (header.partition_entry_size < sizeof(struct gpt_entry) || header.num_partition_entries > GPT_MAX_ENTRIES)
+        return EINVAL;
+
+    struct gpt_partition* parts = mem_alloc(MAX(header.num_partition_entries, 1) * sizeof(struct gpt_partition), 0);
+    if (!parts)
+        return ENOMEM;
+
+    size_t count = 0;
+    const uint64_t base = header.partition_entry_lba * dev->lba_size;
+    for (size_t i = 0; i < header.num_partition_entries; i++) {
+        struct gpt_entry entry;
+        status = vmo_read(&dev->cache, base + i * header.partition_entry_size, &entry, sizeof(entry), &read);
+        if (status || read != sizeof(entry))
+            break;
+        if (guid_is_zero(&entry.type_guid))
+            continue;
+
+        struct gpt_partition* part = &parts[count++];
+        part->type_guid = entry.type_guid;
+        part->unique_guid = entry.unique_guid;
+        part->start_lba = entry.start_lba;
+        part->end_lba = entry.end_lba;
+    }
+
+    *out = parts;
+    *out_count = count;
+    return 0;
+}

dev/block/meson.build

@@ -1,3 +1,11 @@
+kernel_sources += files(
+    'bio.c',
+    'blockdev.c',
+    'gpt.c',
+    'partition.c',
+    'ram.c',
+)
+
 if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
     subdir('nvme')
 endif

dev/block/nvme/core.c

@@ -1,16 +1,34 @@
-#include <zinnia/cmdline.h>
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/fdt.h>
 #include <zinnia/module.h>
+#include <zinnia/pci.h>
 #include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <config.h>
+
+#if CONFIG_NVME_PCI
+#include "nvme_pci.h"
+#endif
+
+struct nvme_dev {
+#if CONFIG_NVME_PCI
+    struct pci_device pci_dev;
+#endif
+};
 
 errno_t nvme_init(struct module* mod) {
-    log("Hello world from NVMe!\n");
+    errno_t status = 0;
 
-    return 0;
+#if CONFIG_NVME_PCI
+    status = pci_driver_register(&nvme_driver);
+#endif
+
+    return status;
 }
 
 errno_t nvme_exit(struct module* mod) {
-    log("Good bye from NVMe!\n");
-
+    // TODO: Unregister
     return 0;
 }
 

dev/block/nvme/nvme_pci.h

@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+extern struct pci_driver nvme_driver;

dev/block/nvme/pci.c

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/errno.h>
+#include <zinnia/module.h>
+#include <zinnia/pci.h>
+
+static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
+    pci_log(dev, "Detected controller \n");
+
+    return 0;
+}
+
+static const struct pci_match nvme_match[] = {
+    // Generic NVMe device. Assume no quirks or any funny business.
+    {PCI_MATCH_CLASS(1, 8, 2), .data = 0},
+};
+
+MATCH_TABLE("pci", nvme_match);
+
+struct pci_driver nvme_driver = {
+    .name = "nvme_pci",
+    .match = nvme_match,
+    .num_matches = ARRAY_SIZE(nvme_match),
+    .probe = nvme_pci_probe,
+};

dev/block/nvme/spec.h

@@ -1 +1,3 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once

dev/block/partition.c

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <stdint.h>
+
+struct partition {
+    struct blockdev blk;
+    struct blockdev* parent;
+    uint64_t start;
+};
+
+static _Atomic(uint32_t) next_minor;
+
+static errno_t partition_submit(struct blockdev* blk, struct bio* bio) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    struct blockdev* parent = part->parent;
+
+    if (bio->op != BIO_FLUSH) {
+        if (bio->lba >= blk->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+        if (bio->op == BIO_WRITE && bio->num_lbas > blk->lba_count - bio->lba) {
+            bio_complete(bio, ENOSPC, 0);
+            return 0;
+        }
+        bio->lba += part->start;
+    }
+    return parent->ops->submit(parent, bio);
+}
+
+static errno_t partition_ioctl(
+    struct blockdev* blk,
+    struct vfs_file* file,
+    uintptr_t request,
+    uintptr_t arg,
+    uintptr_t* out
+) {
+    struct blockdev* parent = containerof(blk, struct partition, blk)->parent;
+    if (!parent->ops->ioctl)
+        return ENOTTY;
+    return parent->ops->ioctl(parent, file, request, arg, out);
+}
+
+static void partition_destroy(struct blockdev* blk) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    blockdev_unref(part->parent);
+    mem_free(part);
+}
+
+static const struct blockdev_ops partition_ops = {
+    .submit = partition_submit,
+    .ioctl = partition_ioctl,
+    .destroy = partition_destroy,
+};
+
+errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out) {
+    struct partition* part = mem_alloc(sizeof(struct partition), 0);
+    if (!part)
+        return ENOMEM;
+
+    blockdev_init(
+        &part->blk,
+        &partition_ops,
+        259,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        parent->lba_size,
+        lba_count
+    );
+    part->blk.limits = parent->limits;
+    blockdev_ref(parent);
+    part->parent = parent;
+    part->start = start_lba;
+    *out = &part->blk;
+    return 0;
+}

dev/block/ram.c

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+constexpr uint32_t RAM_MAJOR = 1;
+constexpr size_t RAM_LBA_SIZE = 512;
+
+struct ramdisk {
+    struct blockdev blk;
+    phys_t base;
+};
+
+static atomic_uint next_minor;
+
+static errno_t ramdisk_submit(struct blockdev* blk, struct bio* bio) {
+    struct ramdisk* ram = containerof(blk, struct ramdisk, blk);
+
+    if (bio->op == BIO_FLUSH) {
+        bio_complete(bio, 0, 0);
+        return 0;
+    }
+    if (bio->lba >= blk->lba_count || bio->num_lbas > blk->lba_count - bio->lba) {
+        bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+        return 0;
+    }
+
+    uint64_t lba = bio->lba;
+    for (size_t i = 0; i < bio->num_segs; i++) {
+        const struct bio_seg* seg = &bio->segs[i];
+        void* store = HHDM_PTR(ram->base + lba * RAM_LBA_SIZE);
+        void* buffer = HHDM_PTR(seg->addr);
+        if (bio->op == BIO_READ)
+            memcpy(buffer, store, seg->len);
+        else
+            memcpy(store, buffer, seg->len);
+        lba += seg->len / RAM_LBA_SIZE;
+    }
+
+    bio_complete(bio, 0, bio->num_lbas);
+    return 0;
+}
+
+static void ramdisk_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct ramdisk, blk));
+}
+
+static const struct blockdev_ops ramdisk_ops = {
+    .submit = ramdisk_submit,
+    .destroy = ramdisk_destroy,
+};
+
+errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out) {
+    struct ramdisk* ram = mem_alloc(sizeof(struct ramdisk), 0);
+    if (!ram)
+        return ENOMEM;
+
+    blockdev_init(
+        &ram->blk,
+        &ramdisk_ops,
+        RAM_MAJOR,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        RAM_LBA_SIZE,
+        length / RAM_LBA_SIZE
+    );
+    ram->base = base;
+    *out = &ram->blk;
+    return 0;
+}

dev/char/chardev.c

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+
+void chardev_init(
+    struct chardev* dev,
+    const struct chardev_ops* ops,
+    const struct vfs_file_ops* file_ops,
+    uint32_t major,
+    uint32_t minor
+) {
+    refcount_init(&dev->rc);
+    dev->ops = ops;
+    dev->file_ops = file_ops;
+    dev->dev = dev_make(major, minor);
+}
+
+void chardev_ref(struct chardev* dev) {
+    refcount_inc(&dev->rc);
+}
+
+void chardev_unref(struct chardev* dev) {
+    if (refcount_dec(&dev->rc) && dev->ops && dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static errno_t chardev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct chardev* dev = node->priv;
+    file->ops = dev->file_ops;
+    if (dev->ops && dev->ops->open)
+        return dev->ops->open(dev, file);
+    return 0;
+}
+
+static void chardev_node_destroy(struct vfs_node* node) {
+    chardev_unref(node->priv);
+}
+
+static const struct vfs_node_ops chardev_node_ops = {
+    .destroy = chardev_node_destroy,
+    .device = {
+        .open = chardev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_chardev(const char* name, struct chardev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_CHARDEV,
+        .ops = &chardev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    chardev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        chardev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t devtmpfs_mkdir(const char* name, mode_t mode) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mkdir(root, root, name, mode, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}
+
+errno_t devtmpfs_remove(const char* name) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_unlink(root, root, name, ident_kernel());
+    vfs_path_drop(root);
+    if (status)
+        return status;
+    devctl_notify_destroy(name);
+    return 0;
+}
+
+errno_t devtmpfs_symlink(const char* name, const char* target) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_symlink(root, root, name, target, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}

dev/char/devctl.c

@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/mutex.h>
+#include <zinnia/poll.h>
+#include <zinnia/tailq.h>
+#include <string.h>
+
+constexpr size_t DEVCTL_QUEUE_CAPACITY = 256;
+constexpr size_t DEVCTL_BACKLOG_CAPACITY = 256;
+
+struct devctl_line {
+    tailq_link(struct devctl_line) link;
+    size_t len;
+    char data[];
+};
+
+struct devctl_queue {
+    tailq_head(struct devctl_line) lines;
+    size_t count;
+};
+
+struct devctl_reader {
+    tailq_link(struct devctl_reader) link;
+    struct devctl_queue queue;
+};
+
+static mutex_t devctl_lock;
+static tailq_head(struct devctl_reader) devctl_readers;
+static struct devctl_queue devctl_backlog;
+static bool devctl_had_reader;
+static struct poll_event devctl_event;
+static atomic_bool devctl_ready;
+static struct chardev devctl_dev;
+
+static void queue_init(struct devctl_queue* queue) {
+    tailq_init(&queue->lines);
+    queue->count = 0;
+}
+
+static struct devctl_line* queue_pop(struct devctl_queue* queue) {
+    struct devctl_line* line = tailq_first(&queue->lines);
+    if (line) {
+        tailq_remove(&queue->lines, line, link);
+        queue->count--;
+    }
+    return line;
+}
+
+static void queue_push(struct devctl_queue* queue, struct devctl_line* line, size_t capacity) {
+    if (queue->count >= capacity)
+        mem_free(queue_pop(queue));
+    tailq_insert_tail(&queue->lines, line, link);
+    queue->count++;
+}
+
+static void queue_clear(struct devctl_queue* queue) {
+    struct devctl_line* line;
+    while ((line = queue_pop(queue)))
+        mem_free(line);
+}
+
+static struct devctl_line* line_new(const char* data, size_t len) {
+    struct devctl_line* line = mem_alloc(sizeof(*line) + len, 0);
+    if (!line)
+        return nullptr;
+    line->len = len;
+    memcpy(line->data, data, len);
+    return line;
+}
+
+static void devctl_notify(const char* type, const char* name) {
+    if (!atomic_load(&devctl_ready, ATOMIC_ACQUIRE))
+        return;
+
+    while (*name == '/')
+        name++;
+
+    const char prefix[] = "!system=DEVFS subsystem=CDEV type=";
+    const char middle[] = " cdev=";
+    const size_t prefix_len = sizeof(prefix) - 1;
+    const size_t type_len = strlen(type);
+    const size_t middle_len = sizeof(middle) - 1;
+    const size_t name_len = strlen(name);
+    const size_t len = prefix_len + type_len + middle_len + name_len + 1;
+
+    char* buf = mem_alloc(len, 0);
+    if (!buf)
+        return;
+    char* pos = buf;
+    memcpy(pos, prefix, prefix_len);
+    pos += prefix_len;
+    memcpy(pos, type, type_len);
+    pos += type_len;
+    memcpy(pos, middle, middle_len);
+    pos += middle_len;
+    memcpy(pos, name, name_len);
+    pos += name_len;
+    *pos = '\n';
+
+    mutex_lock(&devctl_lock);
+    if (tailq_empty(&devctl_readers)) {
+        if (!devctl_had_reader) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&devctl_backlog, line, DEVCTL_BACKLOG_CAPACITY);
+        }
+    } else {
+        for (struct devctl_reader* reader = tailq_first(&devctl_readers); reader; reader = tailq_next(reader, link)) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    mem_free(buf);
+
+    poll_event_notify(&devctl_event);
+}
+
+void devctl_notify_create(const char* name) {
+    devctl_notify("CREATE", name);
+}
+
+void devctl_notify_destroy(const char* name) {
+    devctl_notify("DESTROY", name);
+}
+
+static bool file_nonblocking(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_NONBLOCK;
+}
+
+static errno_t drain_lines(struct devctl_queue* queue, iovec_iter_t* buffer, size_t* out_total) {
+    size_t total = 0;
+    errno_t status = 0;
+    struct devctl_line* line;
+    while ((line = tailq_first(&queue->lines))) {
+        if (line->len > iovec_iter_len(buffer)) {
+            if (total == 0)
+                status = EINVAL;
+            break;
+        }
+        queue_pop(queue);
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, line->data, line->len, &copied);
+        mem_free(line);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_total = total;
+    return status;
+}
+
+static errno_t devctl_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct devctl_reader* reader = file->priv;
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &devctl_event);
+
+    errno_t status = 0;
+    size_t read = 0;
+    mutex_lock(&devctl_lock);
+    while (true) {
+        if (reader->queue.count) {
+            status = drain_lines(&reader->queue, buffer, &read);
+            break;
+        }
+        if (file_nonblocking(file)) {
+            status = EAGAIN;
+            break;
+        }
+        if (poll_table_sleep(&table, &devctl_lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    poll_table_fini(&table);
+
+    *out_read = read;
+    return status;
+}
+
+static errno_t devctl_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    return EBADF;
+}
+
+static enum poll_flags devctl_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    struct devctl_reader* reader = file->priv;
+
+    if (events & (POLLIN | POLLRDNORM))
+        poll_wait(table, &devctl_event);
+
+    enum poll_flags revents = 0;
+    mutex_lock(&devctl_lock);
+    if (reader->queue.count)
+        revents |= POLLIN | POLLRDNORM;
+    mutex_unlock(&devctl_lock);
+    return revents;
+}
+
+static void devctl_release(struct vfs_file* file) {
+    struct devctl_reader* reader = file->priv;
+
+    mutex_lock(&devctl_lock);
+    tailq_remove(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    queue_clear(&reader->queue);
+    mem_free(reader);
+}
+
+static const struct vfs_file_ops devctl_file_ops = {
+    .read = devctl_read,
+    .write = devctl_write,
+    .poll = devctl_poll,
+    .release = devctl_release,
+};
+
+static errno_t devctl_open(struct chardev* dev, struct vfs_file* file) {
+    struct devctl_reader* reader = mem_alloc(sizeof(*reader), 0);
+    if (!reader)
+        return ENOMEM;
+    queue_init(&reader->queue);
+
+    mutex_lock(&devctl_lock);
+    if (!devctl_had_reader) {
+        devctl_had_reader = true;
+        struct devctl_line* line;
+        while ((line = queue_pop(&devctl_backlog)))
+            queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+    }
+    tailq_insert_tail(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    file->priv = reader;
+    vfs_stream_open(file);
+    return 0;
+}
+
+static const struct chardev_ops devctl_dev_ops = {.open = devctl_open};
+
+static void devctl_stage_init(struct boot_info* info) {
+    mutex_init(&devctl_lock);
+    tailq_init(&devctl_readers);
+    queue_init(&devctl_backlog);
+    poll_event_init(&devctl_event);
+    atomic_store(&devctl_ready, true, ATOMIC_RELEASE);
+
+    chardev_init(&devctl_dev, &devctl_dev_ops, &devctl_file_ops, 5, 3);
+    errno_t status = devtmpfs_add_chardev("devctl", &devctl_dev, 0600);
+    assert(status == 0, "Unable to create /dev/devctl (%i)", status);
+}
+
+INIT_TASK(devctl_stage, "dev.devctl", devctl_stage_init);
+INIT_DEPENDS(devctl_stage, devtmpfs_stage);

dev/char/memfiles.c

@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/chardev.h>
+#include <zinnia/random.h>
+#include <zinnia/utils.h>
+
+constexpr size_t MEMFILE_CHUNK = 64;
+
+static const uint8_t zero_chunk[MEMFILE_CHUNK];
+
+static errno_t memfile_discard(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = iovec_iter_len(buffer);
+    return 0;
+}
+
+static errno_t memfile_read_zero(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, zero_chunk, MIN(iovec_iter_len(buffer), MEMFILE_CHUNK), &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static errno_t null_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    *out_read = 0;
+    return 0;
+}
+
+static const struct vfs_file_ops null_file_ops = {
+    .read = null_read,
+    .write = memfile_discard,
+};
+
+static const struct vfs_file_ops zero_file_ops = {
+    .read = memfile_read_zero,
+    .write = memfile_discard,
+};
+
+static errno_t full_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = 0;
+    return ENOSPC;
+}
+
+static const struct vfs_file_ops full_file_ops = {
+    .read = memfile_read_zero,
+    .write = full_write,
+};
+
+static errno_t random_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    uint8_t chunk[MEMFILE_CHUNK];
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        const size_t len = MIN(iovec_iter_len(buffer), sizeof(chunk));
+
+        // TODO: This function sucks and is probably not useful for cryptography.
+        random_fill(chunk, len);
+
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, chunk, len, &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static const struct vfs_file_ops random_file_ops = {
+    .read = random_read,
+    .write = memfile_discard,
+};
+
+static struct chardev null_dev;
+static struct chardev zero_dev;
+static struct chardev full_dev;
+static struct chardev random_dev;
+static struct chardev urandom_dev;
+
+static void memfiles_add(const char* name, struct chardev* dev, const struct vfs_file_ops* ops, uint32_t minor) {
+    chardev_init(dev, nullptr, ops, 1, minor);
+    errno_t status = devtmpfs_add_chardev(name, dev, 0666);
+    assert(status == 0, "Unable to create /dev/%s (%i)", name, status);
+}
+
+static void memfiles_stage_init(struct boot_info* info) {
+    memfiles_add("null", &null_dev, &null_file_ops, 3);
+    memfiles_add("zero", &zero_dev, &zero_file_ops, 5);
+    memfiles_add("full", &full_dev, &full_file_ops, 7);
+    memfiles_add("random", &random_dev, &random_file_ops, 8);
+    memfiles_add("urandom", &urandom_dev, &random_file_ops, 9);
+}
+
+INIT_TASK(memfiles_stage, "dev.memfiles", memfiles_stage_init);
+INIT_DEPENDS(memfiles_stage, devtmpfs_stage, random_stage);

dev/char/meson.build

@@ -1,3 +1,5 @@
 kernel_sources += files(
+    'chardev.c',
+    'devctl.c',
     'memfiles.c',
 )

dev/common/memfiles.c

@@ -1 +0,0 @@
-

dev/dt/driver.c

@@ -0,0 +1,96 @@
+#define pr_fmt(fmt) "fdt: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/fdt.h>
+#include <zinnia/init.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <string.h>
+
+static struct fdt tree;
+static bool tree_valid = false;
+
+static struct spinlock driver_lock = {};
+static vec(struct fdt_driver*) drivers = {};
+
+struct fdt* fdt_get() {
+    return tree_valid ? &tree : nullptr;
+}
+
+static const struct fdt_match* fdt_match_table(
+    const struct fdt_match* table,
+    size_t num_matches,
+    struct fdt_node node
+) {
+    for (size_t i = 0; i < num_matches; i++) {
+        if (fdt_node_is_compatible(node, table[i].compatible))
+            return &table[i];
+    }
+    return nullptr;
+}
+
+errno_t fdt_driver_register(struct fdt_driver* driver) {
+    if (driver->match == nullptr || driver->num_matches == 0) {
+        log_warn("Driver \"%s\" does not define any matches, ignoring\n", driver->name);
+        return 0;
+    }
+
+    if (!tree_valid)
+        return ENODEV;
+
+    spin_lock(&driver_lock);
+    vec_foreach(&drivers, i) {
+        if (strcmp(vec_elem_get(&drivers, i)->name, driver->name) == 0) {
+            spin_unlock(&driver_lock);
+            log_warn("Driver \"%s\" is already registered\n", driver->name);
+            return EEXIST;
+        }
+    }
+    errno_t status = vec_push(&drivers, driver);
+    spin_unlock(&driver_lock);
+    if (status != 0)
+        return status;
+
+    log("Registered platform driver \"%s\" with %zu match%s\n",
+        driver->name,
+        driver->num_matches,
+        driver->num_matches == 1 ? "" : "es");
+    for (size_t i = 0; i < driver->num_matches; i++)
+        log("  [%zu] = \"%s\"\n", i, driver->match[i].compatible);
+
+    for (struct fdt_node node = fdt_root(&tree); node.tree; node = fdt_next_node(node)) {
+        const struct fdt_match* match = fdt_match_table(driver->match, driver->num_matches, node);
+        if (match == nullptr)
+            continue;
+
+        log("Matched driver \"%s\" on rule [%zu]\n", driver->name, (size_t)(match - driver->match));
+
+        status = driver->probe(node, match);
+        if (status != 0)
+            log_warn("%s failed to probe: %i\n", driver->name, status);
+    }
+
+    return 0;
+}
+
+[[__init]]
+static bool fdt_stage_init(struct boot_info* info) {
+    if (!info->dtb)
+        return false;
+
+    const void* blob = HHDM_PTR(info->dtb);
+    if (!fdt_init(&tree, blob, info->dtb_size)) {
+        log_warn("Unable to parse the device tree blob\n");
+        return false;
+    }
+    tree_valid = true;
+
+    struct fdt_prop model = fdt_property(fdt_root(&tree), "model");
+    if (model.tree && model.length)
+        log("Running on \"%.*s\"\n", (int)model.length, (const char*)model.data);
+
+    return true;
+}
+INIT_TASK(fdt_stage, "dev.dt", fdt_stage_init);
+INIT_DEPENDS(fdt_stage, mem_stage);

dev/dt/fdt.c

@@ -0,0 +1,281 @@
+#include <zinnia/fdt.h>
+#include <string.h>
+
+uint32_t fdt_read_u32(const void* data) {
+    const uint8_t* p = data;
+    return (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | p[3];
+}
+
+uint64_t fdt_read_u64(const void* data) {
+    const uint8_t* p = data;
+    return (uint64_t)fdt_read_u32(p) << 32 | fdt_read_u32(p + 4);
+}
+
+static bool range_valid(size_t offset, size_t length, size_t total) {
+    return offset <= total && length <= total - offset;
+}
+
+struct token {
+    uint32_t kind;
+    size_t next;
+    const char* name;
+    const void* data;
+    size_t length;
+};
+
+static bool read_token(struct fdt* tree, size_t offset, struct token* token) {
+    if (!range_valid(offset, 4, tree->structs_len) || offset % 4)
+        return false;
+    *token = (struct token){.kind = fdt_read_u32(tree->structs + offset), .next = offset + 4};
+    if (token->kind == FDT_BEGIN_NODE) {
+        token->name = (const char*)tree->structs + token->next;
+        size_t left = tree->structs_len - token->next;
+        size_t length = strnlen(token->name, left);
+        if (length == left)
+            return false;
+        token->next += (length + 1 + 3) & ~(size_t)3;
+    } else if (token->kind == FDT_PROP) {
+        if (!range_valid(token->next, 8, tree->structs_len))
+            return false;
+        token->length = fdt_read_u32(tree->structs + token->next);
+        size_t name_offset = fdt_read_u32(tree->structs + token->next + 4);
+        token->next += 8;
+        if (name_offset >= tree->strings_len || !range_valid(token->next, token->length, tree->structs_len))
+            return false;
+        token->name = (const char*)tree->strings + name_offset;
+        size_t left = tree->strings_len - name_offset;
+        if (strnlen(token->name, left) == left || !*token->name)
+            return false;
+        token->data = tree->structs + token->next;
+        token->next += (token->length + 3) & ~(size_t)3;
+    } else if (token->kind != FDT_END_NODE && token->kind != FDT_NOP && token->kind != FDT_END) {
+        return false;
+    }
+    return token->next <= tree->structs_len;
+}
+
+bool fdt_init(struct fdt* tree, const void* data, size_t length) {
+    if (!tree)
+        return false;
+    *tree = (struct fdt){};
+    if (!data || length < sizeof(struct fdt_header))
+        return false;
+    const uint8_t* blob = data;
+    size_t total = fdt_read_u32(blob + 4);
+    size_t structs = fdt_read_u32(blob + 8);
+    size_t strings = fdt_read_u32(blob + 12);
+    size_t reserves = fdt_read_u32(blob + 16);
+    size_t strings_len = fdt_read_u32(blob + 32);
+    size_t structs_len = fdt_read_u32(blob + 36);
+    uint32_t version = fdt_read_u32(blob + 20);
+    uint32_t compat = fdt_read_u32(blob + 24);
+    if (fdt_read_u32(blob) != FDT_MAGIC || total > length || total < sizeof(struct fdt_header) || version < 17 ||
+        compat > 17 || compat > version || structs % 4 || reserves % 8 || structs < sizeof(struct fdt_header) ||
+        strings < sizeof(struct fdt_header) || reserves < sizeof(struct fdt_header) ||
+        !range_valid(structs, structs_len, total) || !range_valid(strings, strings_len, total) ||
+        (structs < strings + strings_len && strings < structs + structs_len))
+        return false;
+    size_t reserved_end = reserves;
+    for (;;) {
+        if (!range_valid(reserved_end, 16, total))
+            return false;
+        bool end = !fdt_read_u64(blob + reserved_end) && !fdt_read_u64(blob + reserved_end + 8);
+        reserved_end += 16;
+        if (end)
+            break;
+    }
+    if ((reserves < structs + structs_len && structs < reserved_end) ||
+        (reserves < strings + strings_len && strings < reserved_end))
+        return false;
+    struct fdt candidate = {
+        .blob = blob,
+        .total_size = total,
+        .version = version,
+        .compat_version = compat,
+        .boot_cpuid = fdt_read_u32(blob + 28),
+        .structs = blob + structs,
+        .structs_len = structs_len,
+        .strings = blob + strings,
+        .strings_len = strings_len,
+    };
+    size_t depth = 0;
+    bool started = false;
+    bool children[64] = {};
+    for (size_t offset = 0; offset < structs_len;) {
+        struct token token;
+        if (!read_token(&candidate, offset, &token))
+            return false;
+        if (token.kind == FDT_BEGIN_NODE) {
+            if (depth == 64 || (!depth && (started || *token.name)) || (depth && !*token.name))
+                return false;
+            if (depth)
+                children[depth - 1] = true;
+            children[depth++] = false;
+            started = true;
+        } else if (token.kind == FDT_END_NODE) {
+            if (!depth)
+                return false;
+            depth--;
+        } else if (token.kind == FDT_PROP) {
+            if (!depth || children[depth - 1])
+                return false;
+        } else if (token.kind == FDT_END) {
+            if (!started || depth || token.next != structs_len)
+                return false;
+            *tree = candidate;
+            return true;
+        }
+        offset = token.next;
+    }
+    return false;
+}
+
+struct fdt_node fdt_root(struct fdt* tree) {
+    struct token token;
+    size_t offset = 0;
+    if (tree && tree->blob) {
+        while (read_token(tree, offset, &token)) {
+            if (token.kind == FDT_BEGIN_NODE)
+                return (struct fdt_node){tree, offset};
+            if (token.kind != FDT_NOP)
+                break;
+            offset = token.next;
+        }
+    }
+    return (struct fdt_node){};
+}
+
+static struct fdt_node find_child(struct fdt_node node, const char* name, size_t length) {
+    struct token token;
+    if (!node.tree || !read_token(node.tree, node.offset, &token) || token.kind != FDT_BEGIN_NODE)
+        return (struct fdt_node){};
+    size_t depth = 0;
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
+        if (token.kind == FDT_BEGIN_NODE) {
+            if (!depth && strlen(token.name) == length && !memcmp(token.name, name, length))
+                return (struct fdt_node){node.tree, offset};
+            depth++;
+        } else if (token.kind == FDT_END_NODE) {
+            if (!depth)
+                break;
+            depth--;
+        }
+    }
+    return (struct fdt_node){};
+}
+
+struct fdt_node fdt_find(struct fdt* tree, const char* path) {
+    if (!path || *path != '/')
+        return (struct fdt_node){};
+    struct fdt_node node = fdt_root(tree);
+    path++;
+    while (*path && node.tree) {
+        size_t length = 0;
+        while (path[length] && path[length] != '/')
+            length++;
+        if (!length)
+            return (struct fdt_node){};
+        node = find_child(node, path, length);
+        path += length;
+        if (*path == '/')
+            path++;
+    }
+    return node;
+}
+
+struct fdt_prop fdt_property(struct fdt_node node, const char* name) {
+    struct token token;
+    if (!node.tree || !name || !read_token(node.tree, node.offset, &token) || token.kind != FDT_BEGIN_NODE)
+        return (struct fdt_prop){};
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
+        if (token.kind == FDT_PROP && !strcmp(token.name, name))
+            return (struct fdt_prop){node.tree, token.data, token.length};
+        if (token.kind == FDT_BEGIN_NODE || token.kind == FDT_END_NODE)
+            break;
+    }
+    return (struct fdt_prop){};
+}
+
+bool fdt_property_u32(struct fdt_node node, const char* name, uint32_t* value) {
+    struct fdt_prop property = fdt_property(node, name);
+    if (!property.tree || property.length != 4 || !value)
+        return false;
+    *value = fdt_read_u32(property.data);
+    return true;
+}
+
+bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value) {
+    struct fdt_prop property = fdt_property(node, name);
+    if (!property.tree || property.length != 8 || !value)
+        return false;
+    *value = fdt_read_u64(property.data);
+    return true;
+}
+
+struct fdt_node fdt_next_node(struct fdt_node node) {
+    struct token token;
+    if (!node.tree || !read_token(node.tree, node.offset, &token))
+        return (struct fdt_node){};
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
+        if (token.kind == FDT_BEGIN_NODE)
+            return (struct fdt_node){node.tree, offset};
+        if (token.kind == FDT_END)
+            break;
+    }
+    return (struct fdt_node){};
+}
+
+bool fdt_node_is_compatible(struct fdt_node node, const char* compatible) {
+    struct fdt_prop property = fdt_property(node, "compatible");
+    const char* data = property.data;
+    for (size_t i = 0; i < property.length;) {
+        size_t length = strnlen(data + i, property.length - i);
+        if (length == property.length - i)
+            break;
+        if (!strcmp(data + i, compatible))
+            return true;
+        i += length + 1;
+    }
+    return false;
+}
+
+static uint64_t read_cells(const uint8_t* data, uint32_t cells) {
+    uint64_t value = 0;
+    for (uint32_t i = 0; i < cells; i++)
+        value = value << 32 | fdt_read_u32(data + i * 4);
+    return value;
+}
+
+bool fdt_node_reg(struct fdt_node node, size_t index, phys_t* addr, size_t* size) {
+    if (!node.tree)
+        return false;
+    struct fdt_node root = fdt_root(node.tree);
+    uint32_t addr_cells = 2;
+    uint32_t size_cells = 2;
+    fdt_property_u32(root, "#address-cells", &addr_cells);
+    fdt_property_u32(root, "#size-cells", &size_cells);
+    if (addr_cells > 2 || size_cells > 2)
+        return false;
+
+    struct fdt_prop reg = fdt_property(node, "reg");
+    const size_t stride = (addr_cells + size_cells) * 4;
+    if (!reg.tree || index >= reg.length / stride)
+        return false;
+
+    const uint8_t* entry = (const uint8_t*)reg.data + index * stride;
+    if (addr)
+        *addr = read_cells(entry, addr_cells);
+    if (size)
+        *size = read_cells(entry + addr_cells * 4, size_cells);
+    return true;
+}
+
+struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
+    if (!tree || !tree->blob || !compatible)
+        return (struct fdt_node){};
+    for (struct fdt_node node = fdt_root(tree); node.tree; node = fdt_next_node(node)) {
+        if (fdt_node_is_compatible(node, compatible))
+            return node;
+    }
+    return (struct fdt_node){};
+}

dev/dt/meson.build

@@ -0,0 +1,4 @@
+kernel_sources += files(
+    'driver.c',
+    'fdt.c',
+)

dev/meson.build

@@ -1,12 +1,14 @@
-subdir('common')
 subdir('block')
+subdir('char')
+subdir('dt')
+subdir('video')
+
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    subdir('pci')
+endif
 
 if kconfig.get('CONFIG_ACPI', 'n') == 'y'
     subdir('acpi')
 endif
 
-if kconfig.get('CONFIG_DT', 'n') == 'y'
-    subdir('dt')
-endif
-
 module_lds = meson.current_source_dir() / 'module.ld'

dev/module.ld

@@ -1,5 +1,7 @@
 /* Linker script for kernel modules */
 
+ENTRY(__module_entry)
+
 PHDRS
 {
     text        PT_LOAD     FLAGS(5);
@@ -11,8 +13,8 @@ PHDRS
     moddesc     0x60000003;
     modlicense  0x60000004;
     modcmdline  0x60000005;
-    modentry    0x60000006;
-    modexit     0x60000007;
+    modexit     0x60000006;
+    modmatch    0x60000007;
 }
 
 SECTIONS
@@ -57,14 +59,14 @@ SECTIONS
         KEEP(*(.cmdline))
     } :rodata :modcmdline
 
-    .mod.entry : {
-        KEEP(*(.mod.entry))
-    } :rodata :modentry
-
     .mod.exit : {
         KEEP(*(.mod.exit))
     } :rodata :modexit
 
+    .mod.match : {
+        KEEP(*(.mod.match))
+    } :rodata :modmatch
+
     . = ALIGN(CONSTANT(MAXPAGESIZE));
 
     /* Read-Write data */
@@ -91,6 +93,10 @@ SECTIONS
         *(.rela.dyn .rela.dyn.*)
     } :data
 
+    .relr.dyn : {
+        *(.relr.dyn .relr.dyn.*)
+    } :data
+
     /* Dynamic tags */
     .dynamic : {
         *(.dynamic)
@@ -101,7 +107,6 @@ SECTIONS
         *(.comment*)
         *(.eh_frame*)
         *(.rel.eh_frame*)
-        *(.relr.dyn .relr.dyn.*)
         *(.gcc_except_table*)
         *(.note*)
         *(.interp)

dev/pci/core.c

@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "pci: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/init.h>
+#include <zinnia/module.h>
+#include <zinnia/pci.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <string.h>
+
+static struct spinlock access_lock = {};
+static vec(struct pci_access*) accesses = {};
+
+static struct spinlock device_lock = {};
+static vec(struct pci_device*) devices = {};
+static vec(struct pci_driver*) drivers = {};
+
+errno_t pci_access_register(struct pci_access* access) {
+    spin_lock(&access_lock);
+    errno_t status = vec_push(&accesses, access);
+    spin_unlock(&access_lock);
+    return status;
+}
+
+static bool pci_access_decodes(const struct pci_access* access, const struct pci_address* addr) {
+    return access->domain == addr->domain && addr->bus >= access->start_bus && addr->bus <= access->end_bus;
+}
+
+struct pci_access* pci_access_find(const struct pci_address* addr) {
+    struct pci_access* found = nullptr;
+
+    spin_lock(&access_lock);
+    vec_foreach(&accesses, i) {
+        struct pci_access* access = vec_elem_get(&accesses, i);
+        if (!pci_access_decodes(access, addr))
+            continue;
+        if (found == nullptr || access->priority > found->priority)
+            found = access;
+    }
+    spin_unlock(&access_lock);
+
+    return found;
+}
+
+static uint32_t cfg_read32(struct pci_access* access, const struct pci_address* addr, uint16_t offset) {
+    return access->read32(access, addr, offset & ~3);
+}
+
+static void cfg_write32(struct pci_access* access, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    access->write32(access, addr, offset & ~3, value);
+}
+
+uint32_t pci_read32(struct pci_device* dev, uint16_t offset) {
+    return cfg_read32(dev->access, &dev->address, offset);
+}
+
+uint16_t pci_read16(struct pci_device* dev, uint16_t offset) {
+    return pci_read32(dev, offset) >> ((offset & 2) * 8);
+}
+
+uint8_t pci_read8(struct pci_device* dev, uint16_t offset) {
+    return pci_read32(dev, offset) >> ((offset & 3) * 8);
+}
+
+void pci_write32(struct pci_device* dev, uint16_t offset, uint32_t value) {
+    cfg_write32(dev->access, &dev->address, offset, value);
+}
+
+void pci_write16(struct pci_device* dev, uint16_t offset, uint16_t value) {
+    uint32_t shift = (offset & 2) * 8;
+    uint32_t reg = pci_read32(dev, offset);
+    reg &= ~(0xffffu << shift);
+    reg |= (uint32_t)value << shift;
+    pci_write32(dev, offset, reg);
+}
+
+void pci_write8(struct pci_device* dev, uint16_t offset, uint8_t value) {
+    uint32_t shift = (offset & 3) * 8;
+    uint32_t reg = pci_read32(dev, offset);
+    reg &= ~(0xffu << shift);
+    reg |= (uint32_t)value << shift;
+    pci_write32(dev, offset, reg);
+}
+
+errno_t pci_bar_get(struct pci_device* dev, size_t index, struct pci_bar* out) {
+    if (index >= PCI_NUM_BARS)
+        return EINVAL;
+
+    uint16_t bar_offset = PCI_BAR0 + index * sizeof(uint32_t);
+    uint16_t high_offset = bar_offset + sizeof(uint32_t);
+    uint32_t bar = pci_read32(dev, bar_offset);
+
+    bool is_mmio = (bar & 0x1) == 0;
+    bool is_mmio64 = is_mmio && ((bar >> 1) & 0x3) == 0x2;
+    bool is_prefetchable = is_mmio && (bar & (1 << 3));
+    bool is_64bit_pair = is_mmio64 && index + 1 < PCI_NUM_BARS;
+
+    uint16_t command = pci_read16(dev, PCI_COMMAND);
+    pci_write16(dev, PCI_COMMAND, command & ~(PCI_COMMAND_IO | PCI_COMMAND_MEM));
+
+    uint32_t high = is_64bit_pair ? pci_read32(dev, high_offset) : 0;
+
+    pci_write32(dev, bar_offset, 0xffff'ffff);
+    if (is_64bit_pair)
+        pci_write32(dev, high_offset, 0xffff'ffff);
+
+    uint32_t mask_low = pci_read32(dev, bar_offset);
+    uint32_t mask_high = is_64bit_pair ? pci_read32(dev, high_offset) : 0;
+
+    pci_write32(dev, bar_offset, bar);
+    if (is_64bit_pair)
+        pci_write32(dev, high_offset, high);
+
+    pci_write16(dev, PCI_COMMAND, command);
+
+    uint64_t addr_mask = is_mmio ? ~0xfull : ~0x3ull;
+    uint64_t raw = (uint64_t)mask_high << 32 | mask_low;
+    uint64_t masked = raw & addr_mask & (is_64bit_pair ? UINT64_MAX : UINT32_MAX);
+
+    struct pci_bar result = {
+        .size = masked == 0 ? 0 : (size_t)1 << __builtin_ctzll(masked),
+        .prefetchable = is_prefetchable,
+    };
+
+    if (is_mmio && is_64bit_pair) {
+        result.type = PCI_BAR_MMIO64;
+        result.address = (phys_t)high << 32 | (bar & 0xffff'fff0);
+    } else if (is_mmio) {
+        result.type = PCI_BAR_MMIO32;
+        result.address = bar & 0xffff'fff0;
+    } else {
+        result.type = PCI_BAR_IO;
+        result.address = (uint16_t)(bar & 0xffff'fffc);
+    }
+
+    if (result.address == 0)
+        return ENOENT;
+
+    *out = result;
+    return 0;
+}
+
+uint8_t pci_cap_next(struct pci_device* dev, uint8_t cur) {
+    uint8_t ptr;
+    if (cur == 0) {
+        if (!(pci_read16(dev, PCI_STATUS) & PCI_STATUS_CAP_LIST))
+            return 0;
+        ptr = pci_read8(dev, PCI_CAP_PTR) & ~3;
+    } else {
+        ptr = (pci_read32(dev, cur) >> 8) & ~3;
+    }
+
+    if (ptr == 0)
+        return 0;
+
+    if ((pci_read32(dev, ptr) & 0xff) == 0xff)
+        return 0;
+
+    return ptr;
+}
+
+uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) {
+    for (uint8_t cap = pci_cap_next(dev, 0); cap != 0; cap = pci_cap_next(dev, cap)) {
+        if (pci_read8(dev, cap) == id)
+            return cap;
+    }
+    return 0;
+}
+
+static void pci_device_name(const struct pci_device* dev, char buf[static 18]) {
+    ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
+}
+
+static bool scan_function(struct pci_access* access, struct pci_address addr) {
+    uint32_t reg0 = cfg_read32(access, &addr, PCI_VENDOR_ID);
+    uint16_t vendor_id = reg0;
+    uint16_t device_id = reg0 >> 16;
+
+    if (vendor_id == 0xffff || vendor_id == 0)
+        return false;
+    if (device_id == 0xffff || device_id == 0)
+        return false;
+
+    struct pci_device* dev = mem_alloc(sizeof(*dev), 0);
+    if (dev == nullptr) {
+        log_err("Out of memory while scanning " PCI_ADDR_FMT "\n", PCI_ADDR_ARGS(&addr));
+        return false;
+    }
+
+    dev->address = addr;
+    dev->access = access;
+    dev->id.vendor_id = vendor_id;
+    dev->id.device_id = device_id;
+    dev->class.prog_if = pci_read8(dev, PCI_PROG_IF);
+    dev->class.subclass = pci_read8(dev, PCI_SUBCLASS);
+    dev->class.class = pci_read8(dev, PCI_CLASS);
+    dev->header_type = pci_read8(dev, PCI_HEADER_TYPE);
+
+    if ((dev->header_type & PCI_HEADER_TYPE_MASK) == 0) {
+        dev->id.subvendor_id = pci_read16(dev, PCI_SUBSYS_VENDOR_ID);
+        dev->id.subdevice_id = pci_read16(dev, PCI_SUBSYS_ID);
+    }
+
+    pci_log(
+        dev,
+        "<id = %.4hx:%.4hx, class = %.2hhx:%.2hhx:%.2hhx>\n",
+        dev->id.vendor_id,
+        dev->id.device_id,
+        dev->class.class,
+        dev->class.subclass,
+        dev->class.prog_if
+    );
+
+    if (vec_push(&devices, dev) != 0) {
+        mem_free(dev);
+        return false;
+    }
+
+    return true;
+}
+
+static void scan_slot(struct pci_access* access, struct pci_address addr) {
+    if (!scan_function(access, addr))
+        return;
+
+    if (!(vec_elem_get(&devices, vec_length(&devices) - 1)->header_type & PCI_HEADER_TYPE_MULTIFUNCTION))
+        return;
+
+    for (addr.function = 1; addr.function < PCI_NUM_FUNCTIONS; addr.function++)
+        scan_function(access, addr);
+}
+
+static void format_match_id(char buf[static 5], uint16_t value) {
+    if (value == PCI_ANY_ID)
+        ksnprintf(buf, 5, "****");
+    else
+        ksnprintf(buf, 5, "%.4hx", value);
+}
+
+static void format_match_class(char buf[static 3], uint8_t value) {
+    if (value == PCI_ANY_CLASS)
+        ksnprintf(buf, 3, "**");
+    else
+        ksnprintf(buf, 3, "%.2hhx", value);
+}
+
+static void format_match(const struct pci_match* match, char* buf, size_t size) {
+    char vendor[5], device[5], class[3], subclass[3], prog_if[3];
+    format_match_id(vendor, match->vendor_id);
+    format_match_id(device, match->device_id);
+    format_match_class(class, match->class);
+    format_match_class(subclass, match->subclass);
+    format_match_class(prog_if, match->prog_if);
+    ksnprintf(buf, size, "<id = %s:%s, class = %s:%s:%s>", vendor, device, class, subclass, prog_if);
+}
+
+static bool pci_match_device(const struct pci_match* m, const struct pci_device* dev) {
+    return (m->vendor_id == PCI_ANY_ID || m->vendor_id == dev->id.vendor_id) &&
+        (m->device_id == PCI_ANY_ID || m->device_id == dev->id.device_id) &&
+        (m->class == PCI_ANY_CLASS || m->class == dev->class.class) &&
+        (m->subclass == PCI_ANY_CLASS || m->subclass == dev->class.subclass) &&
+        (m->prog_if == PCI_ANY_CLASS || m->prog_if == dev->class.prog_if);
+}
+
+// Returns the first entry that matches, or nullptr.
+static const struct pci_match* pci_match_table(
+    const struct pci_match* table,
+    size_t num_matches,
+    const struct pci_device* dev
+) {
+    for (size_t i = 0; i < num_matches; i++) {
+        if (pci_match_device(&table[i], dev))
+            return &table[i];
+    }
+    return nullptr;
+}
+
+errno_t pci_driver_register(struct pci_driver* driver) {
+    if (driver->match == nullptr || driver->num_matches == 0) {
+        log_warn("Driver \"%s\" does not define any matches, ignoring\n", driver->name);
+        return 0;
+    }
+
+    spin_lock(&device_lock);
+
+    vec_foreach(&drivers, i) {
+        if (strcmp(vec_elem_get(&drivers, i)->name, driver->name) == 0) {
+            spin_unlock(&device_lock);
+            log_warn("Driver \"%s\" is already registered\n", driver->name);
+            return EEXIST;
+        }
+    }
+
+    errno_t status = vec_push(&drivers, driver);
+    spin_unlock(&device_lock);
+    if (status != 0)
+        return status;
+
+    log("Registered PCI driver \"%s\" with %zu match%s\n",
+        driver->name,
+        driver->num_matches,
+        driver->num_matches == 1 ? "" : "es");
+    for (size_t i = 0; i < driver->num_matches; i++) {
+        char rule[48];
+        format_match(&driver->match[i], rule, sizeof(rule));
+        log("  [%zu] = %s\n", i, rule);
+    }
+
+    for (size_t i = 0;; i++) {
+        spin_lock(&device_lock);
+        if (i >= vec_length(&devices)) {
+            spin_unlock(&device_lock);
+            break;
+        }
+
+        struct pci_device* dev = vec_elem_get(&devices, i);
+        const struct pci_match* match = nullptr;
+        if (dev->driver == nullptr) {
+            match = pci_match_table(driver->match, driver->num_matches, dev);
+            if (match != nullptr)
+                dev->driver = driver;
+        }
+        spin_unlock(&device_lock);
+
+        if (match == nullptr)
+            continue;
+
+        pci_log(dev, "Matched driver \"%s\" on rule [%zu]\n", driver->name, (size_t)(match - driver->match));
+
+        status = driver->probe(dev, match);
+        if (status != 0) {
+            pci_log_warn(dev, "%s failed to probe: %i\n", driver->name, status);
+            spin_lock(&device_lock);
+            dev->driver = nullptr;
+            spin_unlock(&device_lock);
+        }
+    }
+
+    return 0;
+}
+
+[[__init]]
+static void pci_stage_init(struct boot_info*) {
+    if (vec_is_empty(&accesses)) {
+        log_warn("No configuration space access method available\n");
+        return;
+    }
+
+    log("Scanning PCI devices\n");
+
+    spin_lock(&device_lock);
+    vec_foreach(&accesses, i) {
+        struct pci_access* access = vec_elem_get(&accesses, i);
+        for (uint32_t bus = access->start_bus; bus <= access->end_bus; bus++) {
+            for (uint8_t slot = 0; slot < PCI_NUM_SLOTS; slot++) {
+                struct pci_address addr = {
+                    .domain = access->domain,
+                    .bus = bus,
+                    .device = slot,
+                };
+
+                if (pci_access_find(&addr) != access)
+                    continue;
+
+                scan_slot(access, addr);
+            }
+        }
+    }
+    spin_unlock(&device_lock);
+
+    log("Found %zu device(s)\n", vec_length(&devices));
+}
+INIT_TASK(pci_stage, "dev.pci", pci_stage_init);
+INIT_DEPENDS(pci_stage, mem_stage);

dev/pci/ecam.c

@@ -0,0 +1,76 @@
+#define pr_fmt(fmt) "pci: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/mmio.h>
+#include <zinnia/pci.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+
+struct pci_ecam {
+    struct pci_access access;
+    void* base;
+};
+
+static void* ecam_ptr(struct pci_access* self, const struct pci_address* addr, uint16_t offset) {
+    struct pci_ecam* ecam = containerof(self, struct pci_ecam, access);
+    size_t index = (size_t)(addr->bus - self->start_bus) << 20 | (size_t)(addr->device & 0x1f) << 15 |
+        (size_t)(addr->function & 0x7) << 12 | (offset & 0xffc);
+    return (uint8_t*)ecam->base + index;
+}
+
+static uint32_t ecam_read32(struct pci_access* self, const struct pci_address* addr, uint16_t offset) {
+    return mmio_read32(ecam_ptr(self, addr, offset));
+}
+
+static void ecam_write32(struct pci_access* self, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    mmio_write32(ecam_ptr(self, addr, offset), value);
+}
+
+errno_t pci_ecam_register(phys_t base, uint16_t domain, uint8_t start_bus, uint8_t end_bus) {
+    if (end_bus < start_bus)
+        return EINVAL;
+
+    size_t length = ((size_t)(end_bus - start_bus) + 1) << 20;
+    phys_t phys = base + ((phys_t)start_bus << 20);
+
+    uintptr_t vaddr = vm_alloc(length);
+    if (!vaddr)
+        return ENOMEM;
+
+    for (size_t i = 0; i < length; i += arch_mem_page_size()) {
+        errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, phys + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status != 0)
+            return status;
+    }
+
+    struct pci_ecam* ecam = mem_alloc(sizeof(*ecam), 0);
+    if (ecam == nullptr)
+        return ENOMEM;
+
+    *ecam = (struct pci_ecam){
+        .access =
+            {
+                .name = "ecam",
+                .domain = domain,
+                .start_bus = start_bus,
+                .end_bus = end_bus,
+                .priority = PCI_ACCESS_PRIO_ECAM,
+                .read32 = ecam_read32,
+                .write32 = ecam_write32,
+            },
+        .base = (void*)vaddr,
+    };
+
+    errno_t status = pci_access_register(&ecam->access);
+    if (status != 0) {
+        mem_free(ecam);
+        return status;
+    }
+
+    log("ECAM for %.4hx:%.2hhx - %.4hx:%.2hhx at %p\n", domain, start_bus, domain, end_bus, (void*)phys);
+    return 0;
+}

dev/pci/meson.build

@@ -0,0 +1,4 @@
+kernel_sources += files(
+    'core.c',
+    'ecam.c',
+)

dev/video/Kconfig

@@ -0,0 +1,9 @@
+menu "Video Drivers"
+
+config FBCON
+  bool "Framebuffer console"
+  default y
+  help
+    Adds a basic console that can draw on a framebuffer passed by firmware.
+
+endmenu

dev/video/fbcon/fbcon.c

@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/pmap.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stdint.h>
+#include <string.h>
+
+static const uint8_t fbcon_font[] = {
+#embed "fbcon_font.bin"
+};
+
+constexpr size_t FONT_WIDTH = 8;
+constexpr size_t FONT_HEIGHT = 12;
+constexpr uint32_t FG_COLOR = 0xaaaaaa;
+constexpr uint32_t BG_COLOR = 0x000000;
+
+enum esc_state {
+    ESC_NONE,
+    ESC_START,
+    ESC_CSI,
+};
+
+static struct {
+    struct boot_framebuffer fb;
+    uint32_t* back;
+    uint32_t fg, bg;
+    size_t cols, rows;
+    size_t offset_x, offset_y;
+    size_t cx, cy;
+    enum esc_state esc;
+} fbcon;
+
+static inline uint8_t* fb_ptr() {
+    return HHDM_PTR(fbcon.fb.base);
+}
+
+static inline uint32_t* back_row(size_t y) {
+    return fbcon.back + y * fbcon.fb.width;
+}
+
+static void flush_rect(size_t x, size_t y, size_t w, size_t h) {
+    for (size_t j = 0; j < h; j++)
+        memcpy((uint32_t*)(fb_ptr() + (y + j) * fbcon.fb.pitch) + x, back_row(y + j) + x, w * sizeof(uint32_t));
+}
+
+static uint32_t pack_color(uint32_t rgb) {
+    const struct boot_framebuffer* fb = &fbcon.fb;
+    uint32_t r = (rgb >> 16) & 0xff;
+    uint32_t g = (rgb >> 8) & 0xff;
+    uint32_t b = rgb & 0xff;
+    return ((r >> (8 - fb->red.size)) << fb->red.shift) | ((g >> (8 - fb->green.size)) << fb->green.shift) |
+        ((b >> (8 - fb->blue.size)) << fb->blue.shift);
+}
+
+static void fill_rect(size_t x, size_t y, size_t w, size_t h, uint32_t color) {
+    for (size_t j = 0; j < h; j++) {
+        uint32_t* row = back_row(y + j) + x;
+        for (size_t i = 0; i < w; i++)
+            row[i] = color;
+    }
+    flush_rect(x, y, w, h);
+}
+
+static void clear_screen() {
+    fill_rect(0, 0, fbcon.fb.width, fbcon.fb.height, fbcon.bg);
+    fbcon.cx = fbcon.cy = 0;
+}
+
+static void draw_glyph(size_t col, size_t row, uint8_t ch) {
+    const uint8_t* glyph = &fbcon_font[ch * FONT_HEIGHT];
+    size_t x = fbcon.offset_x + col * FONT_WIDTH;
+    size_t y = fbcon.offset_y + row * FONT_HEIGHT;
+
+    for (size_t j = 0; j < FONT_HEIGHT; j++) {
+        uint32_t* line = back_row(y + j) + x;
+        for (size_t i = 0; i < FONT_WIDTH; i++)
+            line[i] = (glyph[j] & (0x80 >> i)) ? fbcon.fg : fbcon.bg;
+    }
+    flush_rect(x, y, FONT_WIDTH, FONT_HEIGHT);
+}
+
+static void scroll_up() {
+    const size_t width = fbcon.fb.width;
+    const size_t text_y = fbcon.offset_y;
+    const size_t text_h = fbcon.rows * FONT_HEIGHT;
+
+    memmove(back_row(text_y), back_row(text_y + FONT_HEIGHT), (text_h - FONT_HEIGHT) * width * sizeof(uint32_t));
+    for (size_t j = text_y + text_h - FONT_HEIGHT; j < text_y + text_h; j++) {
+        uint32_t* row = back_row(j);
+        for (size_t i = 0; i < width; i++)
+            row[i] = fbcon.bg;
+    }
+    flush_rect(0, text_y, width, text_h);
+}
+
+static void newline() {
+    fbcon.cx = 0;
+    if (++fbcon.cy >= fbcon.rows) {
+        fbcon.cy = fbcon.rows - 1;
+        scroll_up();
+    }
+}
+
+static void put_char(uint8_t ch) {
+    switch (fbcon.esc) {
+    case ESC_START:
+        fbcon.esc = ch == '[' ? ESC_CSI : ESC_NONE;
+        if (ch == 'c')
+            clear_screen();
+        return;
+    case ESC_CSI:
+        if (ch >= 0x40 && ch <= 0x7e)
+            fbcon.esc = ESC_NONE;
+        return;
+    case ESC_NONE:
+        break;
+    }
+
+    switch (ch) {
+    case 0x1b:
+        fbcon.esc = ESC_START;
+        break;
+    case '\n':
+        newline();
+        break;
+    case '\r':
+        fbcon.cx = 0;
+        break;
+    case '\b':
+        if (fbcon.cx > 0)
+            fbcon.cx--;
+        break;
+    case '\t':
+        fbcon.cx = (fbcon.cx + 8) & ~(size_t)7;
+        if (fbcon.cx >= fbcon.cols)
+            newline();
+        break;
+    default:
+        if (fbcon.cx >= fbcon.cols)
+            newline();
+        draw_glyph(fbcon.cx++, fbcon.cy, ch);
+        break;
+    }
+}
+
+static void fbcon_write(struct console* con, const char* buf, size_t count) {
+    for (size_t i = 0; i < count; i++)
+        put_char((uint8_t)buf[i]);
+}
+
+static bool fbcon_setup(struct console* con) {
+    const struct boot_framebuffer* fb = &fbcon.fb;
+    if (fb->bytes_per_pixel != 4 || fb->red.size > 8 || fb->green.size > 8 || fb->blue.size > 8)
+        return false;
+
+    fbcon.fg = pack_color(FG_COLOR);
+    fbcon.bg = pack_color(BG_COLOR);
+    fbcon.cols = fb->width / FONT_WIDTH;
+    fbcon.rows = fb->height / FONT_HEIGHT;
+    fbcon.offset_x = (fb->width % FONT_WIDTH) / 2;
+    fbcon.offset_y = (fb->height % FONT_HEIGHT) / 2;
+    if (!fbcon.cols || !fbcon.rows)
+        return false;
+
+    clear_screen();
+    return true;
+}
+
+static struct console fbcon_con = {
+    .name = "fbcon",
+    .init = fbcon_setup,
+    .write = fbcon_write,
+};
+
+[[__init]]
+void fbcon_init(struct boot_info* info) {
+    const struct boot_framebuffer* fb = &info->fb;
+    if (!fb->base)
+        return;
+
+    fbcon.fb = *fb;
+
+    const size_t pgsz = arch_mem_page_size();
+    const phys_t start = ALIGN_DOWN(fb->base, pgsz);
+    const phys_t end = ALIGN_UP(fb->base + fb->pitch * fb->height, pgsz);
+
+    for (phys_t p = start; p < end; p += pgsz) {
+        errno_t status =
+            pmap_map(&kernel_vma.pmap, (uintptr_t)HHDM_PTR(p), p, PTE_READ | PTE_WRITE, CACHE_WRITE_COMBINE);
+        if (status)
+            return;
+    }
+
+    fbcon.back = mem_alloc(fb->width * fb->height * sizeof(uint32_t), ALLOC_NOZERO);
+    if (!fbcon.back)
+        return;
+
+    console_register(&fbcon_con);
+}
+INIT_TASK(fbcon_stage, "core.fbcon", fbcon_init);
+INIT_DEPENDS(fbcon_stage, sched_stage);

dev/video/fbcon/meson.build

@@ -0,0 +1 @@
+kernel_sources += files('fbcon.c')

dev/video/meson.build

@@ -0,0 +1,3 @@
+if kconfig.get('CONFIG_FBCON', 'n') == 'y'
+    subdir('fbcon')
+endif

fs/anon.c

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: ISC
+
+// anonfs is a virtual file system. It backs files that don't live in any other
+// file system, like pipes or unbound sockets.
+
+#include <zinnia/atomic.h>
+#include <zinnia/vfs.h>
+
+static const struct vfs_super_block_ops anon_super_ops = {};
+
+static struct vfs_super_block anon_super = {
+    .rc = {.count = 1}, // There's always just this exact super block.
+    .ops = &anon_super_ops,
+};
+
+static const struct vfs_node_ops anon_node_ops = {};
+
+static _Atomic(ino_t) anon_inode_counter = 1;
+
+errno_t vfs_create_anon_inode(
+    const struct vfs_file_ops* file_ops,
+    enum vfs_node_type type,
+    mode_t mode,
+    const struct ident* identity,
+    struct vfs_node** out
+) {
+    struct vfs_node* node;
+    errno_t status = vfs_node_new(&anon_super, type, &anon_node_ops, mode, &node);
+    if (status)
+        return status;
+
+    node->file_ops = file_ops;
+    // Every anon node gets its own inode counter.
+    node->id = atomic_fetch_add(&anon_inode_counter, 1, ATOMIC_RELAXED);
+    node->uid = identity->euid;
+    node->gid = identity->egid;
+
+    *out = node;
+    return 0;
+}

fs/cache.c

@@ -0,0 +1,411 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/vfs.h>
+#include <string.h>
+#include "internal.h"
+
+constexpr size_t MAX_SYMLINK_DEPTH = 40;
+
+// If `path` is a mount point, crosses into the root of the topmost mounted file system.
+// Returns a new reference.
+static struct vfs_path cross_mounts(struct vfs_path path) {
+    struct vfs_mount* mount = path.mount;
+    struct vfs_entry* entry = path.entry;
+
+again:
+    mutex_lock(&entry->lock);
+    vec_foreach(&entry->mounts, i) {
+        struct vfs_mount* child = vec_elem_get(&entry->mounts, i);
+        if (child != mount) {
+            mutex_unlock(&entry->lock);
+            mount = child;
+            entry = child->root;
+            goto again;
+        }
+    }
+    mutex_unlock(&entry->lock);
+
+    return vfs_path_clone((struct vfs_path){.mount = mount, .entry = entry});
+}
+
+errno_t vfs_path_lookup_parent(struct vfs_path path, struct vfs_path* out) {
+    struct vfs_mount* mount = path.mount;
+    struct vfs_entry* entry = path.entry;
+
+    // Get the top mount point.
+    while (mount->has_mount_point && entry == mount->root) {
+        entry = mount->mount_point.entry;
+        mount = mount->mount_point.mount;
+    }
+
+    if (!entry->parent)
+        return ENOENT;
+
+    *out = vfs_path_clone((struct vfs_path){.mount = mount, .entry = entry->parent});
+    return 0;
+}
+
+errno_t vfs_path_lookup_child(
+    struct vfs_path path,
+    struct vfs_path* root,
+    const char* name,
+    size_t name_len,
+    struct vfs_path* out
+) {
+    errno_t status = 0;
+
+    // Traverse the mounts on the current entry.
+    struct vfs_path crossed = cross_mounts(path);
+
+    // A lookup only makes sense on directories.
+    struct vfs_node* parent = vfs_entry_get_inode(crossed.entry);
+    if (!parent) {
+        status = ENOENT;
+        goto fail;
+    }
+    if (parent->type != VNODE_DIRECTORY) {
+        status = ENOTDIR;
+        goto fail;
+    }
+
+    if (name_len == 1 && name[0] == '.') {
+        vfs_node_unref(parent);
+        *out = crossed;
+        return 0;
+    }
+
+    if (name_len == 2 && name[0] == '.' && name[1] == '.') {
+        vfs_node_unref(parent);
+
+        struct vfs_path crossed_root = cross_mounts(*root);
+        bool at_root = vfs_path_equal(crossed, crossed_root);
+        vfs_path_drop(crossed_root);
+        if (at_root) {
+            *out = crossed;
+            return 0;
+        }
+
+        status = vfs_path_lookup_parent(crossed, out);
+        if (status == ENOENT) {
+            *out = crossed;
+            return 0;
+        }
+        vfs_path_drop(crossed);
+        return status;
+    }
+
+    mutex_lock(&crossed.entry->lock);
+
+    // If this entry has already been looked up before, return that.
+    struct vfs_entry* child = nullptr;
+    for (struct vfs_entry* it = tailq_first(&crossed.entry->children); it; it = tailq_next(it, child_link)) {
+        if (it->name_len == name_len && !memcmp(it->name, name, name_len)) {
+            child = it;
+            vfs_entry_ref(child);
+            break;
+        }
+    }
+
+    if (!child) {
+        status = vfs_entry_new(name, name_len, crossed.entry, &child);
+        if (status) {
+            mutex_unlock(&crossed.entry->lock);
+            goto fail;
+        }
+
+        struct vfs_path child_path = {.mount = crossed.mount, .entry = child};
+        status = parent->ops->dir.lookup ? parent->ops->dir.lookup(parent, &child_path) : ENOENT;
+        if (status && status != ENOENT) {
+            mutex_unlock(&crossed.entry->lock);
+            vfs_entry_unref(child);
+            goto fail;
+        }
+
+        // Allow lookup failures so we can cache it as a negative entry.
+        if (status == ENOENT)
+            vfs_entry_set_negative(child);
+        status = 0;
+
+        // The children list takes over the initial reference.
+        tailq_insert_tail(&crossed.entry->children, child, child_link);
+        vfs_entry_ref(child);
+    }
+
+    mutex_unlock(&crossed.entry->lock);
+
+    *out = cross_mounts((struct vfs_path){.mount = crossed.mount, .entry = child});
+    vfs_entry_unref(child);
+    vfs_node_unref(parent);
+    vfs_path_drop(crossed);
+    return 0;
+
+fail:
+    if (parent)
+        vfs_node_unref(parent);
+    vfs_path_drop(crossed);
+    return status;
+}
+
+static errno_t lookup_inner(
+    struct vfs_path root,
+    struct vfs_path start,
+    const char* path,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    size_t* symlink_depth,
+    struct vfs_path* out
+);
+
+// Resolves `current` in place until it no longer points to a symbolic link.
+static errno_t resolve_symlink(
+    struct vfs_path root,
+    struct vfs_path* current,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    size_t* symlink_depth
+) {
+    errno_t status = 0;
+    char* link_buf = nullptr;
+
+    while (true) {
+        struct vfs_node* node = vfs_entry_get_inode(current->entry);
+        if (!node)
+            break;
+        if (node->type != VNODE_SYMLINK) {
+            vfs_node_unref(node);
+            break;
+        }
+
+        if (*symlink_depth >= MAX_SYMLINK_DEPTH || !node->ops->symlink.read_link) {
+            vfs_node_unref(node);
+            status = ELOOP;
+            break;
+        }
+        (*symlink_depth)++;
+
+        if (!link_buf) {
+            link_buf = mem_alloc(__PATH_MAX + 1, ALLOC_NOZERO);
+            if (!link_buf) {
+                vfs_node_unref(node);
+                status = ENOMEM;
+                break;
+            }
+        }
+
+        iovec_t vec = {.base = (uintptr_t)link_buf, .len = __PATH_MAX};
+        iovec_iter_t iter = iovec_iter_kernel(&vec, 1);
+        size_t link_len = 0;
+        status = node->ops->symlink.read_link(node, &iter, &link_len);
+        vfs_node_unref(node);
+        if (status)
+            break;
+        link_buf[link_len] = '\0';
+
+        struct vfs_path parent;
+        status = vfs_path_lookup_parent(*current, &parent);
+        if (status)
+            break;
+
+        struct vfs_path result;
+        status = lookup_inner(
+            root,
+            parent,
+            link_buf,
+            identity,
+            flags & ~(VFS_LOOKUP_MUST_EXIST | VFS_LOOKUP_MUST_NOT_EXIST),
+            symlink_depth,
+            &result
+        );
+        vfs_path_drop(parent);
+        if (status)
+            break;
+
+        if (vfs_path_equal(result, *current)) {
+            vfs_path_drop(result);
+            status = ELOOP;
+            break;
+        }
+
+        vfs_path_drop(*current);
+        *current = result;
+    }
+
+    mem_free(link_buf);
+    return status;
+}
+
+static errno_t lookup_inner(
+    struct vfs_path root,
+    struct vfs_path start,
+    const char* path,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    size_t* symlink_depth,
+    struct vfs_path* out
+) {
+    errno_t status = 0;
+
+    if (path[0] == '\0')
+        return ENOENT;
+
+    // If a path starts with '/', it's an absolute path.
+    // In that case, skip the first character and use the current root as a starting point.
+    struct vfs_path current;
+    if (path[0] == '/') {
+        current = vfs_path_clone(root);
+        path++;
+    } else {
+        current = vfs_path_clone(start);
+    }
+
+    // Parse each component.
+    while (true) {
+        while (*path == '/')
+            path++;
+        if (*path == '\0')
+            break;
+
+        const char* component = path;
+        size_t component_len = 0;
+        while (path[component_len] != '/' && path[component_len] != '\0')
+            component_len++;
+        path += component_len;
+
+        if (component_len > __NAME_MAX) {
+            status = ENAMETOOLONG;
+            goto fail;
+        }
+
+        status = resolve_symlink(root, &current, identity, flags, symlink_depth);
+        if (status)
+            goto fail;
+
+        struct vfs_node* node = vfs_entry_get_inode(current.entry);
+        if (!node) {
+            status = ENOENT;
+            goto fail;
+        }
+        if (node->type != VNODE_DIRECTORY) {
+            vfs_node_unref(node);
+            status = ENOTDIR;
+            goto fail;
+        }
+
+        status = vfs_node_try_access(node, identity, X_OK, flags & VFS_LOOKUP_REAL_ID);
+        vfs_node_unref(node);
+        if (status)
+            goto fail;
+
+        struct vfs_path next;
+        status = vfs_path_lookup_child(current, &root, component, component_len, &next);
+        if (status)
+            goto fail;
+        vfs_path_drop(current);
+        current = next;
+    }
+
+    if (flags & VFS_LOOKUP_FOLLOW_SYMLINKS) {
+        status = resolve_symlink(root, &current, identity, flags, symlink_depth);
+        if (status)
+            goto fail;
+    }
+
+    struct vfs_node* node = vfs_entry_get_inode(current.entry);
+    if (node)
+        vfs_node_unref(node);
+
+    if ((flags & VFS_LOOKUP_MUST_EXIST) && !node) {
+        status = ENOENT;
+        goto fail;
+    }
+    if ((flags & VFS_LOOKUP_MUST_NOT_EXIST) && node) {
+        status = EEXIST;
+        goto fail;
+    }
+
+    *out = current;
+    return 0;
+
+fail:
+    vfs_path_drop(current);
+    return status;
+}
+
+errno_t vfs_path_lookup(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    struct vfs_path* out
+) {
+    size_t symlink_depth = 0;
+    return lookup_inner(root, cwd, path, identity, flags, &symlink_depth, out);
+}
+
+errno_t vfs_path_absolute(struct vfs_path path, struct vfs_path* root, char* buf, size_t len, size_t* out_len) {
+    if (len == 0)
+        return ENAMETOOLONG;
+
+    struct vfs_path global_root;
+    if (!root) {
+        global_root = vfs_get_root();
+        root = &global_root;
+    }
+
+    errno_t status = 0;
+    struct vfs_mount* mount = path.mount;
+    struct vfs_entry* entry = path.entry;
+
+    // Build the path back to front, leaving room for the NUL terminator.
+    size_t cursor = len - 1;
+    buf[cursor] = '\0';
+
+    while (entry != root->entry || mount != root->mount) {
+        // At the root of a mounted file system we must step into the parent file system.
+        if (entry == mount->root) {
+            if (!mount->has_mount_point) {
+                // Global root with no parent.
+                status = ENOENT;
+                goto out;
+            }
+            entry = mount->mount_point.entry;
+            mount = mount->mount_point.mount;
+            continue;
+        }
+
+        if (entry->name_len + 1 > cursor) {
+            status = ENAMETOOLONG;
+            goto out;
+        }
+        cursor -= entry->name_len;
+        memcpy(buf + cursor, entry->name, entry->name_len);
+        buf[--cursor] = '/';
+
+        if (!entry->parent) {
+            status = ENOENT;
+            goto out;
+        }
+        entry = entry->parent;
+    }
+
+    // The root itself resolves to "/".
+    if (cursor == len - 1) {
+        if (cursor == 0) {
+            status = ENAMETOOLONG;
+            goto out;
+        }
+        buf[--cursor] = '/';
+    }
+
+    size_t result_len = len - 1 - cursor;
+    memmove(buf, buf + cursor, result_len + 1);
+    if (out_len)
+        *out_len = result_len;
+
+out:
+    if (root == &global_root)
+        vfs_path_drop(global_root);
+    return status;
+}

fs/devtmpfs.c

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/chardev.h>
+#include <zinnia/ident.h>
+#include <zinnia/vfs.h>
+
+static struct vfs_mount* dev_mount;
+
+struct vfs_path devtmpfs_root() {
+    return vfs_path_clone((struct vfs_path){.mount = dev_mount, .entry = dev_mount->root});
+}
+
+static errno_t devtmpfs_mount(
+    const struct vfs_filesystem* fs,
+    enum mount_flags flags,
+    void* arg,
+    struct vfs_mount** out
+) {
+    return vfs_mount_new(flags, dev_mount->root, out);
+}
+
+static const struct vfs_filesystem devtmpfs = {
+    .name = "devtmpfs",
+    .mount = devtmpfs_mount,
+};
+
+static void devtmpfs_stage_init(struct boot_info* info) {
+    errno_t status = vfs_mount("tmpfs", 0, nullptr, &dev_mount);
+    assert(status == 0, "Unable to create the devtmpfs (%i)", status);
+
+    status = vfs_register_filesystem(&devtmpfs);
+    assert(status == 0, "Unable to register the devtmpfs (%i)", status);
+}
+
+INIT_TASK(devtmpfs_stage, "fs.devtmpfs", devtmpfs_stage_init);
+INIT_DEPENDS(devtmpfs_stage, vfs_stage);
+
+static void devtmpfs_mount_stage_init(struct boot_info* info) {
+    // Create /dev.
+    struct vfs_path root = vfs_get_root();
+    errno_t status = vfs_mkdir(root, root, "/dev", 0755, ident_kernel());
+    assert(status == 0 || status == EEXIST, "Unable to create /dev (%i)", status);
+
+    // Get /dev.
+    struct vfs_path target;
+    status = vfs_path_lookup(root, root, "/dev", ident_kernel(), VFS_LOOKUP_MUST_EXIST, &target);
+    vfs_path_drop(root);
+    assert(status == 0, "Unable to look up /dev (%i)", status);
+
+    // Prepare a devtmpfs for mounting.
+    struct vfs_mount* mount;
+    status = vfs_mount("devtmpfs", 0, nullptr, &mount);
+    assert(status == 0, "Unable to mount the devtmpfs (%i)", status);
+
+    // Mount it on /dev.
+    status = vfs_path_mount(target, mount);
+    assert(status == 0, "Unable to mount the devtmpfs on /dev (%i)", status);
+    vfs_mount_unref(mount);
+    vfs_path_drop(target);
+}
+
+INIT_TASK(devtmpfs_mount_stage, "fs.devtmpfs.mount", devtmpfs_mount_stage_init);
+INIT_DEPENDS(devtmpfs_mount_stage, devtmpfs_stage);

fs/exec/elf.c

@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/elf.h>
+#include <zinnia/errno.h>
+#include <zinnia/exec.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/mman.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vfs.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+#include <bits/elf.h>
+#include <stdckdint.h>
+#include <uapi/limits.h>
+
+constexpr uintptr_t ELF_DYN_BASE = 0x10000;
+
+struct elf_image {
+    uintptr_t base;
+    uintptr_t entry;
+    uintptr_t phdr;
+    size_t phnum;
+    size_t phent;
+};
+
+static bool elf_identify(struct vmo* obj) {
+    elf_ehdr_t ehdr = {};
+    size_t read;
+    if (vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read) || read != sizeof(ehdr))
+        return false;
+
+    return memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) == 0;
+}
+
+static errno_t elf_read_interp(struct vmo* obj, const elf_phdr_t* phdr, char** out) {
+    if (phdr->p_filesz < 2 || phdr->p_filesz > __PATH_MAX)
+        return ENOEXEC;
+
+    char* path = mem_alloc(phdr->p_filesz, ALLOC_NOZERO);
+    if (!path)
+        return ENOMEM;
+
+    size_t read;
+    errno_t err = vmo_read(obj, phdr->p_offset, path, phdr->p_filesz, &read);
+    if (!err && (read != phdr->p_filesz || path[phdr->p_filesz - 1] != '\0'))
+        err = ENOEXEC;
+    if (err) {
+        mem_free(path);
+        return err;
+    }
+    *out = path;
+    return 0;
+}
+
+static errno_t elf_map(
+    struct vma* space,
+    struct vmo* obj,
+    uintptr_t dyn_base,
+    struct elf_image* out,
+    char** out_interp
+) {
+    errno_t err;
+
+    size_t read;
+    elf_ehdr_t ehdr = {};
+    err = vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read);
+    if (err)
+        return err;
+    if (read != sizeof(ehdr))
+        return ENOEXEC;
+
+    // Check if the file is an ELF.
+    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
+        return EINVAL;
+    if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
+        return EINVAL;
+    if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS)
+        return EINVAL;
+    if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
+        return EINVAL;
+    if (ehdr.e_machine != ELF_ARCH_MACHINE)
+        return EINVAL;
+    if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN)
+        return EINVAL;
+
+    const uintptr_t base = ehdr.e_type == ET_DYN ? dyn_base : 0;
+    uintptr_t entry;
+    if (ckd_add(&entry, ehdr.e_entry, base) || !arch_mem_addr_is_user(entry))
+        return ENOEXEC;
+
+    const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
+    uintptr_t phdrs_end;
+    if (ehdr.e_phentsize < sizeof(elf_phdr_t) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
+        return ENOEXEC;
+    if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size))
+        return ENOEXEC;
+
+    const size_t page_size = arch_mem_page_size();
+    uintptr_t phdr_addr = 0;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        elf_phdr_t phdr = {};
+        err = vmo_read(obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
+        if (err)
+            return err;
+        if (read != sizeof(phdr))
+            return ENOEXEC;
+
+        if (phdr.p_type == PT_PHDR)
+            phdr_addr = phdr.p_vaddr + base;
+
+        if (phdr.p_type == PT_INTERP && out_interp) {
+            if (*out_interp)
+                return ENOEXEC;
+            err = elf_read_interp(obj, &phdr, out_interp);
+            if (err)
+                return err;
+        }
+
+        if (phdr.p_type == PT_LOAD) {
+            if (phdr.p_memsz == 0)
+                continue;
+            if (phdr.p_filesz > phdr.p_memsz)
+                return ENOEXEC;
+
+            uintptr_t vaddr, seg_end;
+            if (ckd_add(&vaddr, phdr.p_vaddr, base) || ckd_add(&seg_end, vaddr, phdr.p_memsz))
+                return ENOEXEC;
+            if (!arch_mem_addr_is_user(vaddr) || !arch_mem_addr_is_user(seg_end - 1))
+                return ENOEXEC;
+
+            if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
+                return ENOEXEC;
+
+            if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
+                phdr_addr = vaddr + (ehdr.e_phoff - phdr.p_offset);
+
+            enum prot_flags prot = 0;
+            if (phdr.p_flags & PF_R)
+                prot |= PROT_READ;
+            if (phdr.p_flags & PF_W)
+                prot |= PROT_WRITE;
+            if (phdr.p_flags & PF_X)
+                prot |= PROT_EXEC;
+
+            const uintptr_t misalign = vaddr & (page_size - 1);
+            const uintptr_t map_address = vaddr - misalign;
+            const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
+            const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
+
+            if (backed_map_size) {
+                err = vma_map(space, obj, map_address, backed_map_size, prot, 0, phdr.p_offset - misalign);
+                if (err)
+                    return err;
+            }
+            if (total_map_size > backed_map_size) {
+                err = vma_map(
+                    space,
+                    nullptr,
+                    map_address + backed_map_size,
+                    total_map_size - backed_map_size,
+                    prot,
+                    0,
+                    0
+                );
+                if (err)
+                    return err;
+            }
+
+            const uintptr_t file_end = vaddr + phdr.p_filesz;
+            if (phdr.p_memsz > phdr.p_filesz && file_end % page_size != 0) {
+                const uintptr_t zero_end = MIN(ALIGN_UP(file_end, page_size), seg_end);
+                err = vma_zero(space, file_end, zero_end - file_end);
+                if (err)
+                    return err;
+            }
+        }
+    }
+
+    *out = (struct elf_image){
+        .base = base,
+        .entry = entry,
+        .phdr = phdr_addr,
+        .phnum = ehdr.e_phnum,
+        .phent = ehdr.e_phentsize,
+    };
+    return 0;
+}
+
+static errno_t elf_load(struct process* proc, struct exec_info* info, uintptr_t* out_ip, uintptr_t* out_sp) {
+    struct elf_image exe = {};
+    char* interp_path = nullptr;
+    errno_t err = elf_map(info->space, info->file_obj, ELF_DYN_BASE, &exe, &interp_path);
+
+    uintptr_t entry = exe.entry;
+    uintptr_t interp_base = 0;
+    if (!err && interp_path) {
+        struct vmo* interp;
+        err = exec_open(proc, interp_path, &interp);
+        if (!err) {
+            struct elf_image ld;
+            if (!elf_identify(interp))
+                err = ELIBBAD;
+            else
+                err = elf_map(info->space, interp, (uintptr_t)1 << (mem_high_shift() - 2), &ld, nullptr);
+            if (!err) {
+                entry = ld.entry;
+                interp_base = ld.base;
+            }
+            vmo_unref(interp);
+        }
+    }
+    mem_free(interp_path);
+    if (err)
+        return err;
+
+    const size_t page_size = arch_mem_page_size();
+
+    const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
+    const size_t stack_size = 2 * 1024 * 1024; // 2MiB
+    const uintptr_t stack_start = highest - stack_size;
+
+    err = vma_map(info->space, nullptr, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0);
+    if (err)
+        return err;
+
+    // Fill in stack info.
+    uintptr_t stack_off = stack_size;
+
+    uintptr_t execfn = 0;
+    if (info->exec_path) {
+        const char nul = 0;
+        stack_off -= 1;
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
+
+        const size_t len = strlen(info->exec_path);
+        stack_off -= len;
+        vma_write(info->space, stack_start + stack_off, info->exec_path, len);
+        execfn = stack_start + stack_off;
+    }
+
+    // TODO: Instead of allocating, calculate it on the fly.
+    size_t num_envp = 0;
+    for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
+        num_envp++;
+    }
+    size_t num_argv = 0;
+    for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
+        num_argv++;
+    }
+
+    size_t strings_size = 0;
+    for (size_t env = 0; env < num_envp; env++)
+        strings_size += strlen(info->envp[env]) + 1;
+    for (size_t arg = 0; arg < num_argv; arg++)
+        strings_size += strlen(info->argv[arg]) + 1;
+
+    uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
+    uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
+    if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
+        mem_free(envp_offsets);
+        mem_free(argv_offsets);
+        return ENOMEM;
+    }
+
+    for (size_t env = 0; env < num_envp; env++) {
+        const char nul = 0;
+        stack_off -= 1;
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
+
+        const size_t len = strlen(info->envp[env]);
+        stack_off -= len;
+        vma_write(info->space, stack_start + stack_off, info->envp[env], len);
+
+        envp_offsets[env] = stack_start + stack_off;
+    }
+
+    for (size_t arg = 0; arg < num_argv; arg++) {
+        const char nul = 0;
+        stack_off -= 1;
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
+
+        const size_t len = strlen(info->argv[arg]);
+        stack_off -= len;
+        vma_write(info->space, stack_start + stack_off, info->argv[arg], len);
+
+        argv_offsets[arg] = stack_start + stack_off;
+    }
+
+    stack_off = ALIGN_DOWN(stack_off, 16);
+
+    // Align the stack if argc + argv + envp does not add up to 16 byte alignment.
+    if ((1 + num_argv + num_envp) & 1) {
+        stack_off -= sizeof(uintptr_t);
+        uintptr_t zero = 0;
+        vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
+    }
+
+#define WRITE_AUXV(auxv, value) \
+    do { \
+        stack_off -= sizeof(uintptr_t); \
+        uintptr_t auxv_val = value; \
+        vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \
+        stack_off -= sizeof(uintptr_t); \
+        auxv_val = auxv; \
+        vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \
+    } while (0)
+
+    // Write auxiliary values.
+    WRITE_AUXV(AT_NULL, 0);
+    WRITE_AUXV(AT_SECURE, 0);
+    WRITE_AUXV(AT_PAGESZ, page_size);
+    WRITE_AUXV(AT_PHNUM, exe.phnum);
+    WRITE_AUXV(AT_PHENT, exe.phent);
+    WRITE_AUXV(AT_ENTRY, exe.entry);
+    if (exe.phdr)
+        WRITE_AUXV(AT_PHDR, exe.phdr);
+    if (interp_base)
+        WRITE_AUXV(AT_BASE, interp_base);
+    if (execfn)
+        WRITE_AUXV(AT_EXECFN, execfn);
+
+    // envp pointers.
+    stack_off -= sizeof(uintptr_t);
+    const uintptr_t zero = 0;
+    vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
+    for (size_t env = num_envp; env > 0; env--) {
+        stack_off -= sizeof(uintptr_t);
+        vma_write(info->space, stack_start + stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]));
+    }
+
+    // argv pointers.
+    stack_off -= sizeof(uintptr_t);
+    vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
+    for (size_t arg = num_argv; arg > 0; arg--) {
+        stack_off -= sizeof(uintptr_t);
+        vma_write(info->space, stack_start + stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]));
+    }
+
+    mem_free(envp_offsets);
+    mem_free(argv_offsets);
+
+    // argc
+    stack_off -= sizeof(uintptr_t);
+    uintptr_t argc = num_argv;
+    vma_write(info->space, stack_start + stack_off, &argc, sizeof(argc));
+
+    *out_ip = entry;
+    *out_sp = highest - stack_size + stack_off;
+    return 0;
+}
+
+static struct exec_format elf_format = {
+    .name = "elf",
+    .identify = elf_identify,
+    .load = elf_load,
+};
+
+static void elf_stage_init(struct boot_info*) {
+    exec_register(&elf_format);
+}
+INIT_TASK(elf_stage, "fs.exec.elf", elf_stage_init);
+INIT_DEPENDS(elf_stage, mem_stage);
+INIT_ENTAILS(elf_stage, vfs_stage);

fs/exec/exec.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/exec.h>
+#include <zinnia/mutex.h>
+#include <zinnia/print.h>
+#include <zinnia/process.h>
+#include <zinnia/tailq.h>
+#include <zinnia/vfs.h>
+
+static mutex_t formats_lock;
+static tailq_head(struct exec_format) formats = {nullptr, &formats.tqh_first};
+
+struct exec_format* exec_identify(struct vmo* obj) {
+    mutex_lock(&formats_lock);
+    struct exec_format* format = tailq_first(&formats);
+    while (format != nullptr && !format->identify(obj))
+        format = tailq_next(format, link);
+    mutex_unlock(&formats_lock);
+    return format;
+}
+
+void exec_register(struct exec_format* format) {
+    mutex_lock(&formats_lock);
+    tailq_insert_tail(&formats, format, link);
+    mutex_unlock(&formats_lock);
+
+    log("exec: Registered new exec format \"%s\"\n", format->name);
+}
+
+errno_t exec_open(struct process* proc, const char* path, struct vmo** out) {
+    mutex_lock(&proc->lock);
+    struct vfs_path root = vfs_path_clone(proc->root);
+    struct vfs_path cwd = vfs_path_clone(proc->cwd);
+    mutex_unlock(&proc->lock);
+
+    struct vfs_file* file;
+    errno_t status = vfs_file_open(root, cwd, path, O_RDONLY | O_EXEC, 0, &proc->identity, &file);
+    vfs_path_drop(root);
+    vfs_path_drop(cwd);
+    if (status)
+        return status;
+
+    status = exec_file_object(proc, file, out);
+    vfs_file_unref(file);
+    return status;
+}
+
+errno_t exec_file_object(struct process* proc, struct vfs_file* file, struct vmo** out) {
+    errno_t status = vfs_node_check_exec(file->vnode, &proc->identity);
+    if (!status)
+        status = vfs_file_get_object(file, out);
+    return status;
+}

fs/exec/meson.build

@@ -0,0 +1,4 @@
+kernel_sources += files(
+    'elf.c',
+    'exec.c',
+)

fs/file.c

@@ -0,0 +1,628 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/print.h>
+#include <zinnia/vfs.h>
+#include <stdckdint.h>
+#include <string.h>
+#include "internal.h"
+
+errno_t vfs_file_new(struct vfs_node* node, const struct vfs_path* path, enum open_flags flags, struct vfs_file** out) {
+    assert(node != nullptr, "Every open file must be backed by an inode");
+
+    struct vfs_file* file = mem_alloc(sizeof(*file), 0);
+    if (!file)
+        return ENOMEM;
+
+    refcount_init(&file->rc);
+    file->ops = node->file_ops;
+    file->vnode = node;
+    vfs_node_ref(node);
+    atomic_init(&file->flags, flags);
+    mutex_init(&file->lock);
+
+    switch (node->type) {
+    case VNODE_REGULAR:
+    case VNODE_DIRECTORY:
+        file->position_kind = VFS_POSITION_ATOMIC;
+        break;
+    default:
+        file->position_kind = VFS_POSITION_POSITION;
+        break;
+    }
+
+    if (path) {
+        file->path = vfs_path_clone(*path);
+        file->has_path = true;
+    }
+
+    *out = file;
+    return 0;
+}
+
+void vfs_stream_open(struct vfs_file* file) {
+    file->position_kind = VFS_POSITION_STREAM;
+}
+
+// Resolves the absolute path of `path` as seen from `root` into a fitting allocation.
+static char* make_absolute_path(struct vfs_path path, struct vfs_path* root) {
+    char* buf = mem_alloc(__PATH_MAX, ALLOC_NOZERO);
+    if (!buf)
+        return nullptr;
+
+    size_t len;
+    char* result = nullptr;
+    if (vfs_path_absolute(path, root, buf, __PATH_MAX, &len) == 0) {
+        result = mem_alloc(len + 1, ALLOC_NOZERO);
+        if (result)
+            memcpy(result, buf, len + 1);
+    }
+
+    mem_free(buf);
+    return result;
+}
+
+static errno_t open_special(
+    struct vfs_path* root,
+    struct vfs_path* file_path,
+    struct vfs_node* node,
+    enum open_flags flags,
+    struct vfs_file** out
+) {
+    if ((node->type == VNODE_CHARDEV || node->type == VNODE_BLOCKDEV) && (!node->ops || !node->ops->device.open))
+        return ENXIO;
+
+    struct vfs_file* file;
+    errno_t status = vfs_file_new(node, file_path, flags, &file);
+    if (status)
+        return status;
+    file->absolute_path = make_absolute_path(*file_path, root);
+
+    status = node->type == VNODE_FIFO ? vfs_fifo_open(node, file) : node->ops->device.open(node, file);
+    if (status) {
+        file->ops = nullptr;
+        vfs_file_unref(file);
+        return status;
+    }
+
+    *out = file;
+    return 0;
+}
+
+static enum access_flags open_access(enum open_flags flags) {
+    if (flags & O_PATH)
+        return F_OK;
+
+    enum access_flags access = F_OK;
+    if (flags & O_RDONLY)
+        access |= R_OK;
+    if (flags & (O_WRONLY | O_TRUNC))
+        access |= W_OK;
+    return access;
+}
+
+static errno_t do_open_inode(
+    struct vfs_path* root,
+    struct vfs_path* file_path,
+    struct vfs_node* node,
+    enum open_flags flags,
+    bool created,
+    const struct ident* identity,
+    struct vfs_file** out
+) {
+    // If we want to open as a directory, make sure this is actually a directory.
+    if ((flags & O_DIRECTORY) && node->type != VNODE_DIRECTORY)
+        return ENOTDIR;
+
+    if (node->type == VNODE_DIRECTORY && (flags & (O_WRONLY | O_CREAT | O_TRUNC)))
+        return EISDIR;
+
+    errno_t status = created ? 0 : vfs_node_try_access(node, identity, open_access(flags), false);
+    if (status)
+        return status;
+
+    switch (node->type) {
+    case VNODE_REGULAR: {
+        if (flags & O_TRUNC) {
+            if (!(flags & O_WRONLY))
+                return EINVAL;
+            if (!node->ops->regular.truncate)
+                return EPERM;
+            status = node->ops->regular.truncate(node, 0);
+            if (status)
+                return status;
+        }
+
+        struct vfs_file* file;
+        status = vfs_file_new(node, file_path, flags, &file);
+        if (status)
+            return status;
+        file->absolute_path = make_absolute_path(*file_path, root);
+        *out = file;
+        return 0;
+    }
+    case VNODE_DIRECTORY:
+        if (node->ops->dir.open)
+            return node->ops->dir.open(node, file_path, flags, identity, out);
+        return vfs_file_new(node, file_path, flags, out);
+    case VNODE_SYMLINK:
+        return ELOOP;
+    case VNODE_CHARDEV:
+    case VNODE_BLOCKDEV:
+    case VNODE_FIFO:
+        return open_special(root, file_path, node, flags, out);
+    case VNODE_SOCKET:
+        return ENXIO;
+    default:
+        return ENOTSUP;
+    }
+}
+
+errno_t vfs_file_open(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    enum open_flags flags,
+    mode_t mode,
+    const struct ident* identity,
+    struct vfs_file** out
+) {
+    if ((flags & O_DIRECTORY) && (flags & (O_CREAT | O_TMPFILE)))
+        return EINVAL;
+
+    const bool exclusive = (flags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL);
+
+    enum vfs_lookup_flags lookup_flags = 0;
+    // If we don't want to create a file when opening, the lookup must succeed.
+    // If we want to create a temporary unnamed file, `path` is the directory of the owning FS.
+    // In that case the path also has to exist.
+    if (!(flags & O_CREAT) || (flags & O_TMPFILE))
+        lookup_flags |= VFS_LOOKUP_MUST_EXIST;
+    if (exclusive)
+        lookup_flags |= VFS_LOOKUP_MUST_NOT_EXIST;
+    if (!(flags & O_NOFOLLOW) && !exclusive)
+        lookup_flags |= VFS_LOOKUP_FOLLOW_SYMLINKS;
+
+    struct vfs_path file_path;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, lookup_flags, &file_path);
+    if (status)
+        return status;
+
+    bool created = false;
+    struct vfs_node* node = vfs_entry_get_inode(file_path.entry);
+    if (!node) {
+        // If the lookup was successful, we expect that the entry is positive.
+        if (!(flags & O_CREAT)) {
+            log_warn("vfs: Tried opening a file without O_CREAT and backing inode, this is a bug!\n");
+            status = ENOENT;
+            goto out;
+        }
+
+        struct vfs_path parent_path;
+        status = vfs_path_lookup_parent(file_path, &parent_path);
+        if (status)
+            goto out;
+        struct vfs_node* parent = vfs_entry_get_inode(parent_path.entry);
+        vfs_path_drop(parent_path);
+        if (!parent) {
+            status = ENOENT;
+            goto out;
+        }
+
+        status = parent->type == VNODE_DIRECTORY ? 0 : ENOTDIR;
+        if (!status)
+            status = vfs_node_try_access(parent, identity, W_OK | X_OK, false);
+        if (!status)
+            status = parent->ops->dir.create ? parent->ops->dir.create(parent, file_path.entry, mode, identity) : EPERM;
+        vfs_node_unref(parent);
+        if (status)
+            goto out;
+
+        node = vfs_entry_get_inode(file_path.entry);
+        if (!node) {
+            status = EIO;
+            goto out;
+        }
+        created = true;
+    }
+
+    status = do_open_inode(&root, &file_path, node, flags, created, identity, out);
+    vfs_node_unref(node);
+
+out:
+    vfs_path_drop(file_path);
+    return status;
+}
+
+static errno_t advance_offset(off_t* offset, ssize_t amount) {
+    if (amount > 0 && ckd_add(offset, *offset, amount))
+        return EOVERFLOW;
+    return 0;
+}
+
+errno_t vfs_file_read(struct vfs_file* file, iovec_iter_t* buffer, ssize_t* out_read) {
+    *out_read = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (!file->ops || !file->ops->read)
+        return EINVAL;
+
+    errno_t status;
+    ssize_t read = 0;
+    switch (file->position_kind) {
+    case VFS_POSITION_STREAM:
+        status = file->ops->read(file, buffer, 0, &read);
+        break;
+    case VFS_POSITION_ATOMIC:
+        mutex_lock(&file->lock);
+        status = file->ops->read(file, buffer, file->position, &read);
+        if (!status)
+            status = advance_offset(&file->position, read);
+        mutex_unlock(&file->lock);
+        break;
+    case VFS_POSITION_POSITION: {
+        mutex_lock(&file->lock);
+        off_t pos = file->position;
+        mutex_unlock(&file->lock);
+
+        status = file->ops->read(file, buffer, pos, &read);
+        if (!status)
+            status = advance_offset(&pos, read);
+        if (!status) {
+            mutex_lock(&file->lock);
+            file->position = pos;
+            mutex_unlock(&file->lock);
+        }
+        break;
+    }
+    default:
+        return EINVAL;
+    }
+
+    if (status)
+        return status;
+    *out_read = read;
+    return 0;
+}
+
+errno_t vfs_file_pread(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    *out_read = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return ESPIPE;
+    if (!file->ops || !file->ops->read)
+        return EINVAL;
+
+    return file->ops->read(file, buffer, offset, out_read);
+}
+
+static errno_t check_write_progress(ssize_t written) {
+    if (written == 0) {
+        log_warn("vfs: write reported zero progress for a non-empty buffer\n");
+        return EIO;
+    }
+    return 0;
+}
+
+static errno_t sync_after_write(struct vfs_file* file, ssize_t written) {
+    if (written <= 0)
+        return 0;
+
+    if ((atomic_load(&file->flags, ATOMIC_RELAXED) & O_SYNC) == O_SYNC)
+        return vfs_file_sync(file, false);
+    if ((atomic_load(&file->flags, ATOMIC_RELAXED) & O_DSYNC) == O_DSYNC)
+        return vfs_file_sync(file, true);
+    return 0;
+}
+
+errno_t vfs_file_write(struct vfs_file* file, iovec_iter_t* buffer, ssize_t* out_written) {
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (!file->ops || !file->ops->write)
+        return EINVAL;
+
+    // TODO: Appending writes through different files of the same inode aren't serialized.
+    const bool append = atomic_load(&file->flags, ATOMIC_RELAXED) & O_APPEND;
+
+    errno_t status;
+    ssize_t written = 0;
+    switch (file->position_kind) {
+    case VFS_POSITION_STREAM:
+        status = file->ops->write(file, buffer, 0, &written);
+        break;
+    case VFS_POSITION_ATOMIC:
+        mutex_lock(&file->lock);
+        if (append)
+            file->position = file->vnode->size;
+        status = file->ops->write(file, buffer, file->position, &written);
+        if (!status)
+            status = advance_offset(&file->position, written);
+        mutex_unlock(&file->lock);
+        break;
+    case VFS_POSITION_POSITION: {
+        mutex_lock(&file->lock);
+        off_t pos = append ? (off_t)file->vnode->size : file->position;
+        mutex_unlock(&file->lock);
+
+        status = file->ops->write(file, buffer, pos, &written);
+        if (!status)
+            status = advance_offset(&pos, written);
+        if (!status) {
+            mutex_lock(&file->lock);
+            file->position = pos;
+            mutex_unlock(&file->lock);
+        }
+        break;
+    }
+    default:
+        return EINVAL;
+    }
+
+    if (!status)
+        status = check_write_progress(written);
+    if (!status)
+        status = sync_after_write(file, written);
+    if (status)
+        return status;
+
+    *out_written = written;
+    return 0;
+}
+
+errno_t vfs_file_pwrite(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return ESPIPE;
+    if (!file->ops || !file->ops->write)
+        return EINVAL;
+
+    ssize_t written = 0;
+    errno_t status = file->ops->write(file, buffer, offset, &written);
+    if (!status)
+        status = check_write_progress(written);
+    if (!status)
+        status = sync_after_write(file, written);
+    if (status)
+        return status;
+
+    *out_written = written;
+    return 0;
+}
+
+errno_t vfs_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    if (!file->ops || !file->ops->ioctl)
+        return ENOTTY;
+    return file->ops->ioctl(file, request, arg, out_result);
+}
+
+errno_t vfs_file_sync(struct vfs_file* file, bool data_only) {
+    if (!file->ops || !file->ops->sync)
+        return 0;
+    return file->ops->sync(file, data_only);
+}
+
+errno_t vfs_file_seek(struct vfs_file* file, enum vfs_seek_anchor anchor, off_t offset, off_t* out_position) {
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return ESPIPE;
+
+    errno_t status = 0;
+    mutex_lock(&file->lock);
+
+    off_t base;
+    switch (anchor) {
+    case VFS_SEEK_START:
+        base = 0;
+        break;
+    case VFS_SEEK_CURRENT:
+        base = file->position;
+        break;
+    case VFS_SEEK_END:
+        base = file->vnode->size;
+        break;
+    default:
+        status = EINVAL;
+        goto out;
+    }
+
+    off_t new_position;
+    if (ckd_add(&new_position, base, offset)) {
+        status = EOVERFLOW;
+        goto out;
+    }
+    if (new_position < 0) {
+        status = EINVAL;
+        goto out;
+    }
+
+    file->position = new_position;
+    if (out_position)
+        *out_position = new_position;
+
+out:
+    mutex_unlock(&file->lock);
+    return status;
+}
+
+off_t vfs_file_tell(struct vfs_file* file) {
+    mutex_lock(&file->lock);
+    off_t position = file->position;
+    mutex_unlock(&file->lock);
+    return position;
+}
+
+errno_t vfs_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    if (!file->ops || !file->ops->mmap)
+        return ENODEV;
+    return file->ops->mmap(file, vma, addr, len, prot, flags, offset, out_addr);
+}
+
+enum poll_flags vfs_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    const enum poll_flags always = POLLERR | POLLHUP | POLLNVAL;
+    if (!file->ops || !file->ops->poll)
+        return events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM);
+    return file->ops->poll(file, events | POLLERR | POLLHUP, table) & (events | always);
+}
+
+errno_t vfs_file_get_object(struct vfs_file* file, struct vmo** out) {
+    struct vfs_node* node = file->vnode;
+    if (node->type != VNODE_REGULAR || !node->ops || !node->ops->regular.get_vmo)
+        return ENODEV;
+    return node->ops->regular.get_vmo(node, out);
+}
+
+static uint8_t dirent_type(enum vfs_node_type type) {
+    switch (type) {
+    case VNODE_REGULAR:
+        return __DT_REG;
+    case VNODE_DIRECTORY:
+        return __DT_DIR;
+    case VNODE_SYMLINK:
+        return __DT_LNK;
+    case VNODE_FIFO:
+        return __DT_FIFO;
+    case VNODE_BLOCKDEV:
+        return __DT_BLK;
+    case VNODE_CHARDEV:
+        return __DT_CHR;
+    case VNODE_SOCKET:
+        return __DT_SOCK;
+    default:
+        // Should be unreachable, but just as a failsafe in case someone messes up.
+        return __DT_UNKNOWN;
+    }
+}
+
+errno_t vfs_dir_get_cached_entries(
+    struct vfs_node* node,
+    struct vfs_entry* entry,
+    off_t offset,
+    struct __dirent* buffer,
+    size_t buffer_len,
+    const struct ident* identity,
+    size_t* out_count
+) {
+    size_t read = 0;
+    off_t current = 0;
+
+    mutex_lock(&entry->lock);
+    for (struct vfs_entry* child = tailq_first(&entry->children); child && read < buffer_len;
+         child = tailq_next(child, child_link)) {
+        struct vfs_node* child_node = vfs_entry_get_inode(child);
+        if (!child_node)
+            continue;
+
+        if (current < offset) {
+            current++;
+            vfs_node_unref(child_node);
+            continue;
+        }
+
+        struct __dirent* d = &buffer[read];
+        d->d_ino = child_node->id;
+        d->d_off = current;
+        d->d_reclen = sizeof(*d);
+        d->d_type = dirent_type(child_node->type);
+        memcpy(d->d_name, child->name, child->name_len);
+        d->d_name[child->name_len] = '\0';
+        vfs_node_unref(child_node);
+
+        read++;
+        current++;
+    }
+    mutex_unlock(&entry->lock);
+
+    *out_count = read;
+    return 0;
+}
+
+static void fill_dot_entry(struct __dirent* d, ino_t ino, off_t off, const char* name) {
+    d->d_ino = ino;
+    d->d_off = off;
+    d->d_reclen = sizeof(*d);
+    d->d_type = __DT_DIR;
+    memset(d->d_name, 0, sizeof(d->d_name));
+    memcpy(d->d_name, name, strlen(name));
+}
+
+errno_t vfs_get_dir_entries(
+    struct vfs_file* file,
+    struct __dirent* buffer,
+    size_t buffer_len,
+    const struct ident* identity,
+    size_t* out_count
+) {
+    *out_count = 0;
+    if (buffer_len == 0)
+        return 0;
+
+    struct vfs_node* node = file->vnode;
+    if (!file->has_path || node->type != VNODE_DIRECTORY)
+        return ENOTDIR;
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return EBADF;
+
+    ino_t parent_ino = node->id;
+    struct vfs_path parent;
+    if (vfs_path_lookup_parent(file->path, &parent) == 0) {
+        struct vfs_node* parent_node = vfs_entry_get_inode(parent.entry);
+        if (parent_node) {
+            parent_ino = parent_node->id;
+            vfs_node_unref(parent_node);
+        }
+        vfs_path_drop(parent);
+    }
+
+    errno_t status = 0;
+    mutex_lock(&file->lock);
+
+    const off_t start_offset = file->position;
+    size_t num_read = 0;
+
+    // Read `.` and `..` entries.
+    if (start_offset == 0)
+        fill_dot_entry(&buffer[num_read++], node->id, 0, ".");
+    if ((start_offset == 1 || num_read == 1) && num_read < buffer_len)
+        fill_dot_entry(&buffer[num_read++], parent_ino, 1, "..");
+
+    // Read normal dir entries.
+    if (num_read < buffer_len && node->ops->dir.get_dir_entries) {
+        size_t count = 0;
+        status = node->ops->dir.get_dir_entries(
+            node,
+            file->path.entry,
+            start_offset > 2 ? start_offset - 2 : 0,
+            buffer + num_read,
+            buffer_len - num_read,
+            identity,
+            &count
+        );
+        if (status)
+            goto out;
+        num_read += count;
+    }
+
+    file->position += num_read;
+    for (size_t i = 0; i < num_read; i++)
+        buffer[i].d_off = start_offset + i + 1;
+
+    *out_count = num_read;
+
+out:
+    mutex_unlock(&file->lock);
+    return status;
+}

fs/ident.c

@@ -0,0 +1,21 @@
+#include <zinnia/ident.h>
+
+static const struct ident kernel_identity = {0};
+
+const struct ident* ident_kernel() {
+    return &kernel_identity;
+}
+
+bool ident_in_group(const struct ident* identity, gid_t gid, bool use_real) {
+    if (gid == (use_real ? identity->gid : identity->egid))
+        return true;
+    vec_foreach(&identity->groups, i) {
+        if (vec_elem_get(&identity->groups, i) == gid)
+            return true;
+    }
+    return false;
+}
+
+bool ident_is_privileged(const struct ident* identity, bool use_real) {
+    return (use_real ? identity->uid : identity->euid) == 0;
+}