vfs interface

.clangd

@@ -1,2 +1,2 @@
 Style:
-  AngledHeaders: [ "zinnia/.*", " kernel/.*" ]
+  AngledHeaders: [ "zinnia/.*", "uapi/.*" ]

.gitignore

@@ -0,0 +1,8 @@
+.config*
+build/
+.cache/
+.wraplock
+*.o
+*.a
+*.so
+__pycache__/

Kconfig

@@ -0,0 +1,9 @@
+mainmenu "Zinnia Kernel Configuration"
+
+config KERNEL_FLAVOR
+  string "Kernel flavor name"
+  default "vanilla"
+
+source "arch/$(ARCH)/Kconfig"
+source "core/Kconfig"
+source "dev/Kconfig"

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2026 Marvin Friedrich
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

LICENSE

@@ -0,0 +1 @@
+LICENSES/ISC.txt

LICENSES/ISC.txt

@@ -0,0 +1,7 @@
+ISC License
+
+Copyright 2026 Marvin Friedrich and contributors
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

LICENSES/Unlicense.txt

@@ -0,0 +1,10 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
+
+In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

README.md

@@ -1,20 +1,27 @@
+<img src="logo.svg" width="10%"/>
+
 # Zinnia
 
-![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=blue)
+![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=green)
+![GitHub top language](https://img.shields.io/github/languages/top/zinnia-os/zinnia)
 ![GitHub Repo stars](https://img.shields.io/github/stars/zinnia-os/zinnia?style=flat)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/zinnia-os/zinnia/ci.yml)
 ![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/zinnia-os/zinnia?style=flat)
+![Discord](https://img.shields.io/discord/1305893510704267327?logo=discord&logoColor=white&label=discord)
 
-Zinnia is a lightweight Unix-like microkernel targeting 64-bit devices.
+Zinnia is a lightweight Unix-like monolithic kernel targeting 64-bit devices.
 
 ## Building Zinnia
 
-To build the kernel and the servers you need `meson` and a C23-compatible
-GNU-like compiler targeting Zinnia.
-Currently supported are GCC and Clang toolchains.
+To build the kernel you need:
+- `meson`
+- a `kconfig` frontend (like `kconfiglib` via `pip`)
+- a GNU-like C compiler with support for C23 (e.g. `gcc` or `clang`)
 
 To configure, run:
 ```sh
-meson setup -Dbuild_kernel=true -Dbuild_servers=true $build_dir
+alldefconfig
+meson setup $build_dir
 ```
 
 And to build:

arch/x86_64/Kconfig

@@ -0,0 +1,16 @@
+menu "x86_64 Architecture options"
+
+config X86_64
+  bool
+  default y
+  select ACPI
+  select HAVE_MMU
+  select HAVE_MMU_GENERIC
+  help
+    64-bit x86 CPU support for AMD and Intel CPUs.
+
+config TSC
+  bool "Invariant Timestamp Counter support"
+  default y
+
+endmenu

arch/x86_64/apic.c

@@ -1,11 +1,10 @@
-#include <zinnia/mem.h>
-#include <kernel/clock.h>
-#include <kernel/mmio.h>
-#include <kernel/percpu.h>
-#include <kernel/pmap.h>
-#include <kernel/vas.h>
-#include <kernel/virt.h>
-#include <assert.h>
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/vma.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
@@ -48,17 +47,21 @@ void lapic_init(struct local_apic* lapic) {
         apic_msr |= 1 << 10;
     else {
         uintptr_t vaddr = vm_alloc(0x1000);
-        pmap_map(&kernel_vas.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        pmap_map(&kernel_vma.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
         lapic->xapic_regs = (void*)vaddr;
         assert(lapic->xapic_regs, "Failed to allocate virtual memory!\n");
     }
 
     asm_wrmsr(0x1B, apic_msr);
 
+    lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
+    if (lapic->xapic_regs)
+        lapic->id >>= 24;
+
     // Reset the TPR.
     lapic_write_reg(lapic, APIC_REG_TPR, 0);
     // Enable APIC bit in the SIVR.
-    lapic_write_reg(lapic, APIC_REG_SIVR, lapic_read_reg(lapic, APIC_REG_SIVR) | 0x100);
+    lapic_write_reg(lapic, APIC_REG_SIVR, (lapic_read_reg(lapic, APIC_REG_SIVR) & ~0xFF) | 0x100 | IDT_SPURIOUS);
 
     if (lapic->xapic_regs) {
         lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
@@ -85,10 +88,38 @@ void lapic_eoi(struct local_apic* lapic) {
     lapic_write_reg(lapic, APIC_REG_EOI, 0);
 }
 
-#define PIC1_COMMAND_PORT 0x20
-#define PIC1_DATA_PORT    0x21
-#define PIC2_COMMAND_PORT 0xA0
-#define PIC2_DATA_PORT    0xA1
+void lapic_send_ipi(
+    struct local_apic* lapic,
+    uint32_t target,
+    uint8_t vector,
+    enum apic_delivery_mode delivery_mode,
+    enum apic_dest_mode dest_mode,
+    enum apic_level level,
+    enum apic_trigger_mode trigger_mode
+) {
+    uint64_t icr = vector;
+    icr |= (uint64_t)delivery_mode << 8;
+    icr |= (uint64_t)dest_mode << 11;
+    icr |= (uint64_t)level << 14;
+    icr |= (uint64_t)trigger_mode << 15;
+
+    if (lapic->xapic_regs)
+        icr |= (uint64_t)(uint8_t)target << 56;
+    else
+        icr |= (uint64_t)target << 32;
+
+    lapic_write_reg(lapic, APIC_REG_ICR, icr);
+
+    if (lapic->xapic_regs) {
+        while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12))
+            asm volatile("pause");
+    }
+}
+
+constexpr uint16_t PIC1_COMMAND_PORT = 0x20;
+constexpr uint16_t PIC1_DATA_PORT = 0x21;
+constexpr uint16_t PIC2_COMMAND_PORT = 0xA0;
+constexpr uint16_t PIC2_DATA_PORT = 0xA1;
 
 void pic_disable() {
     // We initialize the PIC properly, but completely disable it and use the APIC in favor of it.

arch/x86_64/archctl.c

@@ -0,0 +1,32 @@
+#include <zinnia/errno.h>
+#include <zinnia/syscall.h>
+#include <arch/archctl.h>
+#include <arch/mem.h>
+#include <uapi/archctl.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+
+struct syscall_ret arch_archctl(uintptr_t op, uintptr_t arg) {
+    struct syscall_ret ret = {};
+
+    switch (op) {
+    case __ARCHCTL_SET_FSBASE:
+        if (!arch_mem_addr_is_user(arg)) {
+            ret.error = EINVAL;
+            return ret;
+        }
+        asm_wrmsr(MSR_FS_BASE, arg);
+        return ret;
+    case __ARCHCTL_SET_GSBASE:
+        if (!arch_mem_addr_is_user(arg)) {
+            ret.error = EINVAL;
+            return ret;
+        }
+        // Update the inactive GSBASE MSR, not ours.
+        asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
+        return ret;
+    default:
+        ret.error = EINVAL;
+        return ret;
+    }
+}

arch/x86_64/gdt.c

@@ -1,14 +1,14 @@
-#include <kernel/percpu.h>
-#include <kernel/syscalls.h>
+#include <zinnia/percpu.h>
+#include <zinnia/string.h>
 #include <stddef.h>
-#include <string.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
+#include <x86_64/percpu.h>
 #include <x86_64/tss.h>
 
-static const struct gdt main_gdt = {
-    .null = {.val = 0},
+const struct gdt gdt_base = {
+    .null = {0},
     .kernel_code32 = {.val = 0x00cf9b000000ffff},
     .kernel_data32 = {.val = 0x00cf93000000ffff},
     .kernel_code64 = {.val = 0x00a09b0000000000},
@@ -18,17 +18,17 @@ static const struct gdt main_gdt = {
     .user_code64 = {.val = 0x00a0fb0000000000},
     .tss = {
         .limit0 = sizeof(struct tss) - 1,
-        .access = 0x89, // Present | DPL = 0 | System | 64-bit TSS
+        .access = 0x89, // Present | DPL=0 | System | 64-bit TSS
         .limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F,
         .reserved = 0,
     },
 };
 
 void gdt_load() {
-    struct gdt* gdt = &percpu_get()->arch.gdt;
-    memcpy(gdt, &main_gdt, sizeof(main_gdt));
+    struct gdt* gdt = &percpu_ref(arch_data)->gdt;
+    memcpy(gdt, &gdt_base, sizeof(gdt_base));
 
-    struct tss* tss = &percpu_get()->arch.tss;
+    struct tss* tss = &percpu_ref(arch_data)->tss;
     tss->iopb = sizeof(struct tss);
 
     uintptr_t tss_addr = (uintptr_t)tss;

arch/x86_64/hpet.c

@@ -1,11 +1,11 @@
-#include <common/utils.h>
-#include <kernel/clock.h>
-#include <kernel/cmdline.h>
-#include <kernel/pmap.h>
-#include <kernel/print.h>
-#include <kernel/vas.h>
-#include <kernel/virt.h>
-#include <assert.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/mmio.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
 #include <uacpi/acpi.h>
 #include <uacpi/status.h>
 #include <uacpi/tables.h>
@@ -15,27 +15,27 @@ static bool use_hpet = true;
 static void hpet_option(bool use) {
     use_hpet = use;
 }
-CMDLINE_OPTION("hpet", hpet_option);
+CMDLINE_OPTION("x86_64.hpet", "Enable High Precision Event Timer clock", hpet_option);
 
 struct hpet {
     struct clock clock;
-    volatile uint64_t* regs;
+    void* regs;
     uint32_t period;
 };
 
-#define HPET_CAP     (0x0 / sizeof(uint64_t))
-#define HPET_CFG     (0x10 / sizeof(uint64_t))
-#define HPET_COUNTER (0xF0 / sizeof(uint64_t))
+constexpr uint32_t HPET_CAP = 0x0;
+constexpr uint32_t HPET_CFG = 0x10;
+constexpr uint32_t HPET_COUNTER = 0xF0;
 
 static void hpet_reset(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    hpet->regs[HPET_COUNTER] = 0;
+    mmio_write64(hpet->regs + HPET_COUNTER, 0);
 }
 
 static uint64_t hpet_get_elapsed_ns(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    uint64_t counter = hpet->regs[HPET_COUNTER];
-    return (counter * (uint64_t)hpet->period / 1'000'000);
+    uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
+    return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
 }
 
 static struct hpet hpet_clock = {
@@ -60,15 +60,15 @@ void hpet_init() {
     struct acpi_hpet* hpet = table.ptr;
 
     uintptr_t vaddr = vm_alloc(0x1000);
-    pmap_map(&kernel_vas.pmap, vaddr, hpet->address.address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+    pmap_map(&kernel_vma.pmap, vaddr, hpet->address.address, PTE_READ | PTE_WRITE, CACHE_MMIO);
     hpet_clock.regs = (void*)vaddr;
 
     uacpi_table_unref(&table);
-    kprintf("HPET regs at %p\n", hpet_clock.regs);
+    log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
 
-    hpet_clock.regs[HPET_CFG] = hpet_clock.regs[HPET_CFG] | 1;
+    mmio_write64(hpet_clock.regs + HPET_CFG, mmio_read64(hpet_clock.regs + HPET_CFG) | 1);
 
-    uint64_t caps = hpet_clock.regs[HPET_CAP];
+    uint64_t caps = mmio_read64(hpet_clock.regs + HPET_CAP);
     hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
 
     assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");

arch/x86_64/idt.c

@@ -1,16 +1,21 @@
-#include <common/compiler.h>
-#include <kernel/irq.h>
-#include <kernel/panic.h>
-#include <kernel/percpu.h>
-#include <kernel/print.h>
-#include <kernel/sched.h>
-#include <assert.h>
+#include <zinnia/assert.h>
+#include <zinnia/compiler.h>
+#include <zinnia/irq.h>
+#include <zinnia/panic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <arch/sched.h>
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
+#include <x86_64/percpu.h>
 
 struct [[__packed]] idt_entry {
     uint16_t base0;
@@ -33,31 +38,69 @@ static_assert(sizeof(struct idtr) == 10);
 
 void interrupt_handler(struct arch_context* ctx) {
     struct percpu* cpu = percpu_get();
-    bool old = irq_set_interrupted(true);
-
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
     uint64_t isr = ctx->isr;
+
     switch (isr) {
     case IDT_PF:
         uint64_t cr2;
         asm volatile("mov %0, cr2" : "=r"(cr2));
-        panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
-    case IDT_IPI_RESCHED:
-        arch_sched_preempt_disable();
-        if (arch_sched_preempt_enable()) {
-            lapic_eoi(&cpu->arch.lapic);
-            sched_reschedule(&cpu->sched);
+
+        const struct vma_fault_info info = {
+            .ip = ctx->rip,
+            .addr = cr2,
+            .caused_by_user = (ctx->cs & CPL_USER) == CPL_USER,
+            .caused_by_write = ctx->error & (1 << 1),
+            .caused_by_fetch = ctx->error & (1 << 4),
+            .page_was_present = ctx->error & (1 << 0),
+        };
+
+        struct task* current = cpu->sched.current;
+        const bool irqs_were_on = ctx->rflags & RFLAGS_IF;
+        if (irqs_were_on)
+            irq_set_state(true);
+
+        const bool handled = current && current->space && vma_page_fault(current->space, &info);
+
+        if (irqs_were_on)
+            irq_set_state(false);
+
+        if (handled)
+            break;
+
+        if (current && current->usercopy_region) {
+            struct usercopy_region* region = current->usercopy_region;
+            if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) {
+                ctx->rip = (uintptr_t)region->fault_ip;
+                break;
+            }
         }
+
+        panic(
+            "Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n",
+            info.caused_by_write       ? "write to"
+                : info.caused_by_fetch ? "execute on"
+                                       : "read from",
+            info.page_was_present ? "present" : "non-present",
+            info.addr,
+            info.ip
+        );
+    case IDT_SPURIOUS:
         break;
-    case 0x20 ...(0x20 + ARRAY_SIZE(cpu->arch.irq_lines)):
-        struct irq_line* line = cpu->arch.irq_lines[isr - 0x20];
-        assert(line, "Unhandled interrupt on ISR %lu\n", isr);
-        // TODO: irq_raise(line);
+    case IDT_IPI_RESCHED:
+        lapic_eoi(&arch_cpu->lapic);
+        arch_sched_preempt_disable();
+        if (arch_sched_preempt_enable())
+            sched_reschedule(&cpu->sched);
         break;
     default:
-        panic("Unhandled exception on ISR %lu\n", isr);
+        if (isr < 0x20)
+            panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
+        assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu\n", isr);
+        struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
+        assert(line, "Unhandled interrupt on ISR %lu\n", isr);
+        irq_line_raise(line);
     }
-
-    irq_set_interrupted(old);
 }
 
 #define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))

arch/x86_64/include/bits/elf.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <zinnia/elf.h>
+
+#define ELF_ARCH_CLASS   ELFCLASS64
+#define ELF_ARCH_DATA    ELFDATA2LSB
+#define ELF_ARCH_MACHINE EM_X86_64
+
+#define ELF_R_SYM   ELF64_R_SYM
+#define ELF_R_TYPE  ELF64_R_TYPE
+#define ELF_ST_BIND ELF64_ST_BIND
+#define ELF_ST_TYPE ELF64_ST_TYPE
+
+#define elf_ehdr elf64_ehdr
+#define elf_phdr elf64_phdr
+#define elf_dyn  elf64_dyn
+#define elf_addr elf64_addr
+#define elf_off  elf64_off
+#define elf_nhdr elf64_nhdr
+#define elf_auxv elf64_auxv
+#define elf_sym  elf64_sym
+#define elf_rela elf64_rela
+#define elf_rel  elf64_rel
+
+enum {
+    R_NONE = 0,
+    R_COPY = 5,
+    R_GLOB_DAT = 6,
+    R_JUMP_SLOT = 7,
+    R_RELATIVE = 8,
+    R_IRELATIVE = 37,
+};

arch/x86_64/include/bits/sched.h

@@ -1,36 +1,32 @@
 #pragma once
 
-#include <bits/types.h>
+#include <zinnia/types.h>
 #include <stdint.h>
 
 struct arch_task_context {
     uint64_t rsp;
     uint8_t* fpu_region;
+    phys_t fpu_phys;
     uint16_t ds, es, fs, gs;
     uint64_t fs_base, gs_base;
 };
 
-#define ARCH_ABI(typ, reg, ...) \
-    union { \
-        typ reg, __VA_ARGS__; \
-    }
-
 struct arch_context {
     uint64_t r15;
     uint64_t r14;
-    ARCH_ABI(uint64_t, r13, a7);
-    ARCH_ABI(uint64_t, r12, a6);
+    uint64_t r13;
+    uint64_t r12;
     uint64_t r11;
-    ARCH_ABI(uint64_t, r10, a5);
-    ARCH_ABI(uint64_t, r9, a3);
-    ARCH_ABI(uint64_t, r8, a4);
-    ARCH_ABI(uint64_t, rsi, a1);
-    ARCH_ABI(uint64_t, rdi, a0);
+    uint64_t r10;
+    uint64_t r9;
+    uint64_t r8;
+    uint64_t rsi;
+    uint64_t rdi;
     uint64_t rbp;
-    ARCH_ABI(uint64_t, rdx, a2, ret1);
+    uint64_t rdx;
     uint64_t rcx;
     uint64_t rbx;
-    ARCH_ABI(uint64_t, rax, num, ret0);
+    uint64_t rax;
     uint64_t isr;   // Pushed onto the stack by the interrupt handler stubs.
     uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code.
     uint64_t rip;   // The rest is pushed onto the stack by the CPU during an interrupt.
@@ -40,16 +36,12 @@ struct arch_context {
     uint64_t ss;
 };
 
-static_assert(sizeof(struct arch_context) == 22 * sizeof(uint64_t));
-
 #define ARCH_CTX_NUM  rax
 #define ARCH_CTX_A0   rdi
 #define ARCH_CTX_A1   rsi
 #define ARCH_CTX_A2   rdx
-#define ARCH_CTX_A3   r9
+#define ARCH_CTX_A3   r10
 #define ARCH_CTX_A4   r8
-#define ARCH_CTX_A5   r10
-#define ARCH_CTX_A6   r12
-#define ARCH_CTX_A7   r13
+#define ARCH_CTX_A5   r9
 #define ARCH_CTX_RET0 rax
 #define ARCH_CTX_RET1 rdx

arch/x86_64/include/bits/types.h

@@ -3,4 +3,3 @@
 #include <stdint.h>
 
 typedef uint64_t phys_t;
-typedef uint64_t reg_t;

arch/x86_64/include/x86_64/apic.h

@@ -55,4 +55,15 @@ enum apic_regs {
 
 void lapic_init(struct local_apic* lapic);
 void lapic_eoi(struct local_apic* lapic);
+
+void lapic_send_ipi(
+    struct local_apic* lapic,
+    uint32_t target,
+    uint8_t vector,
+    enum apic_delivery_mode delivery_mode,
+    enum apic_dest_mode dest_mode,
+    enum apic_level level,
+    enum apic_trigger_mode trigger_mode
+);
+
 void pic_disable();

arch/x86_64/include/x86_64/asm.h

@@ -55,13 +55,13 @@ static inline void asm_fxrstor(const void* mem) {
 // Saves the FPU state to a region of memory using XSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_xsave(void* mem) {
-    asm volatile("xsave [%0]" ::"r"(mem) : "memory");
+    asm volatile("xsave [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
 }
 
 // Restores the FPU state from a region of memory using XRSTOR.
 // Pointer must be 16-byte aligned.
 static inline void asm_xrstor(const void* mem) {
-    asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
+    asm volatile("xrstor [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
 }
 
 static inline uint8_t asm_inb(uint16_t port) {

arch/x86_64/include/x86_64/defs.h

@@ -185,6 +185,7 @@
 #define MSR_EFER_FFXSR (1 << 14)
 #define MSR_EFER_TCE   (1 << 15)
 
+#define MSR_PAT            (0x277)
 #define MSR_EFER           (0xC0000080)
 #define MSR_STAR           (0xC0000081)
 #define MSR_LSTAR          (0xC0000082)
@@ -194,8 +195,16 @@
 #define MSR_GS_BASE        (0xC0000101)
 #define MSR_KERNEL_GS_BASE (0xC0000102)
 
-#define CPL_USER   (0b11)
-#define CPL_KERNEL (0b00)
+#define PAT_UC  (0x0)
+#define PAT_WC  (0x1)
+#define PAT_WT  (0x4)
+#define PAT_WP  (0x5)
+#define PAT_WB  (0x6)
+#define PAT_UCM (0x7)
+
+#define CPL_USER   (0x3)
+#define CPL_KERNEL (0x0)
 
 #define IDT_PF          (0xE)
+#define IDT_SPURIOUS    (0xFE)
 #define IDT_IPI_RESCHED (0xFF)

arch/x86_64/include/x86_64/gdt.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <common/compiler.h>
+#include <zinnia/compiler.h>
 #include <stdint.h>
 
 union gdt_desc {
@@ -50,7 +50,7 @@ struct [[__packed]] gdtr {
     struct gdt* base;
 };
 
-struct percpu;
+extern const struct gdt gdt_base;
 
 // Initializes a GDT on the local CPU.
 void gdt_load();

arch/x86_64/include/x86_64/idt.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <kernel/init.h>
+#include <zinnia/init.h>
 
 [[__init]]
 void idt_init();

arch/x86_64/include/x86_64/percpu.h

@@ -1,6 +1,7 @@
 #pragma once
 
-#include <kernel/irq.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <stddef.h>
 #include <x86_64/apic.h>
 #include <x86_64/gdt.h>
@@ -16,3 +17,8 @@ struct arch_percpu {
     void (*fpu_restore)(const void*);
     bool can_smap;
 };
+
+extern struct arch_percpu arch_data;
+
+// Initializes the calling CPU and makes `cpu` its per-CPU block.
+void x86_64_cpu_setup(struct percpu* cpu);

arch/x86_64/include/x86_64/pmap.h

@@ -0,0 +1,19 @@
+#pragma once
+#include <zinnia/pmap.h>
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};

arch/x86_64/include/x86_64/smp.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <zinnia/compiler.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <x86_64/gdt.h>
+
+struct [[__packed]] smp_info {
+    struct gdt gdt;
+    struct gdtr gdtr;
+    uint32_t farjmp_offset;
+    uint32_t farjmp_segment;
+    uint32_t temp_stack;
+    uint32_t temp_cr3;
+    uint64_t entry;
+    uint64_t hhdm_offset;
+    uint64_t percpu;
+    uint32_t lapic_id;
+    _Atomic(uint8_t) booted;
+};
+
+// Discovers all APs and starts them.
+void x86_64_smp_init();

arch/x86_64/include/x86_64/tss.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <common/compiler.h>
 #include <stdint.h>
+#include <zinnia/compiler.h>
 
 struct [[__packed]] tss {
     uint32_t reserved0;

arch/x86_64/irq.c

@@ -1,3 +1,4 @@
+#include <arch/irq.h>
 #include <stdint.h>
 #include <x86_64/defs.h>
 
@@ -7,11 +8,13 @@ bool arch_irq_get_state() {
     return flags & RFLAGS_IF;
 }
 
-void arch_irq_set_state(bool state) {
+bool arch_irq_set_state(bool state) {
+    bool old = arch_irq_get_state();
     if (state)
-        asm volatile("sti");
+        asm volatile("sti" ::: "memory");
     else
-        asm volatile("cli");
+        asm volatile("cli" ::: "memory");
+    return old;
 }
 
 void arch_irq_wait() {

arch/x86_64/mem.c

@@ -1,4 +1,5 @@
-#include <kernel/types.h>
+#include <zinnia/types.h>
+#include <arch/mem.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -29,3 +30,7 @@ uintptr_t arch_mem_pfndb_addr() {
 uintptr_t arch_mem_mapping_addr() {
     return 0xFFFF'C000'0000'0000;
 }
+
+bool arch_mem_addr_is_user(uintptr_t addr) {
+    return addr < 0x0000'7FFF'FFFF'F000;
+}

arch/x86_64/meson.build

@@ -1,11 +1,12 @@
 kernel_c_args += [
     '-mgeneral-regs-only',
     '-mno-red-zone',
-    '-mcmodel=kernel',
+    '-masm=intel',
 ]
 
 kernel_sources += files(
     'apic.c',
+    'archctl.c',
     'gdt.c',
     'hpet.c',
     'idt.c',
@@ -13,12 +14,12 @@ kernel_sources += files(
     'mem.c',
     'percpu.c',
     'pmap.c',
+    'pmap.c',
     'sched.c',
     'serial.c',
     'smp.c',
     'sys.c',
     'syscall.c',
-    'trampoline.S',
     'tsc.c',
     'usercopy.c',
 )

arch/x86_64/percpu.c

@@ -1,5 +1,6 @@
-#include <kernel/percpu.h>
-#include <kernel/print.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <config.h>
 #include <stddef.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
@@ -7,6 +8,8 @@
 #include <x86_64/gdt.h>
 #include <x86_64/hpet.h>
 #include <x86_64/idt.h>
+#include <x86_64/percpu.h>
+#include <x86_64/smp.h>
 #include <x86_64/syscall.h>
 #include <x86_64/tsc.h>
 
@@ -38,14 +41,27 @@ union ia32_star {
 
 static_assert(sizeof(union ia32_star) == 8);
 
-static void setup_cpu(struct percpu* cpu) {
+[[__percpu]]
+struct arch_percpu arch_data = {0};
+
+void x86_64_cpu_setup(struct percpu* cpu) {
     asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
     asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
     asm_wrmsr(MSR_FS_BASE, 0);
 
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+
     gdt_load();
+
+    // TODO: not if FRED
     idt_load();
 
+    asm_wrmsr(
+        MSR_PAT,
+        PAT_WB | PAT_WT << 8 | PAT_UCM << 16 | PAT_UC << 24 | (uint64_t)PAT_WP << 32 | (uint64_t)PAT_WC << 40 |
+            (uint64_t)PAT_UCM << 48 | (uint64_t)PAT_UC << 56
+    );
+
     // Syscall extension.
     asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
     union ia32_star star = {0};
@@ -85,13 +101,13 @@ static void setup_cpu(struct percpu* cpu) {
 
         asm_wrxcr(0, xcr0);
 
-        cpu->arch.fpu_size = cpuid13.ecx;
-        cpu->arch.fpu_save = asm_xsave;
-        cpu->arch.fpu_restore = asm_xrstor;
+        arch_cpu->fpu_size = cpuid13.ecx;
+        arch_cpu->fpu_save = asm_xsave;
+        arch_cpu->fpu_restore = asm_xrstor;
     } else {
-        cpu->arch.fpu_size = 512;
-        cpu->arch.fpu_save = asm_fxsave;
-        cpu->arch.fpu_restore = asm_fxrstor;
+        arch_cpu->fpu_size = 512;
+        arch_cpu->fpu_save = asm_fxsave;
+        arch_cpu->fpu_restore = asm_fxrstor;
     }
 
     if (cpuid7.ecx & CPUID_7C_UMIP)
@@ -102,25 +118,28 @@ static void setup_cpu(struct percpu* cpu) {
 
     if (cpuid7.ebx & CPUID_7B_SMAP) {
         cr4 |= CR4_SMAP;
-        cpu->arch.can_smap = true;
+        arch_cpu->can_smap = true;
     }
 
     if (cpuid7.ebx & CPUID_7B_FSGSBASE)
         cr4 |= CR4_FSGSBASE;
 
-    asm volatile("mov %0, cr0" : "=r"(cr0));
-    asm volatile("mov %0, cr4" : "=r"(cr4));
+    asm volatile("mov cr0, %0" ::"r"(cr0));
+    asm volatile("mov cr4, %0" ::"r"(cr4));
 
-    lapic_init(&cpu->arch.lapic);
+    lapic_init(&arch_cpu->lapic);
     cpu->online = true;
 }
 
 void arch_percpu_init() {
     hpet_init();
+#if CONFIG_TSC
     tsc_init();
+#endif
 
     // Init the BSP.
-    setup_cpu(&percpu_bsp);
+    x86_64_cpu_setup(&percpu_bsp);
 
     // Init any APs.
+    x86_64_smp_init();
 }

arch/x86_64/pmap.c

@@ -1,27 +1,15 @@
-#include <kernel/pmap.h>
-#include <kernel/types.h>
-
-enum : pte_t {
-    ARCH_FLAG_PRESENT = 1 << 0,
-    ARCH_FLAG_READ_WRITE = 1 << 1,
-    ARCH_FLAG_USER_MODE = 1 << 2,
-    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
-    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
-    ARCH_FLAG_ACCESSED = 1 << 5,
-    ARCH_FLAG_DIRTY = 1 << 6,
-    ARCH_FLAG_SIZE = 1 << 7,
-    ARCH_FLAG_GLOBAL = 1 << 8,
-    ARCH_FLAG_AVAILABLE = 1 << 9,
-    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
-    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
-
-    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
-};
+#include <zinnia/pmap.h>
+#include <zinnia/types.h>
+#include <x86_64/pmap.h>
 
 void pte_clear(pte_t* pte) {
     *pte = 0;
 }
 
+void pte_flush(uintptr_t vaddr) {
+    asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
+}
+
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
 
@@ -35,6 +23,21 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
             result |= ARCH_FLAG_READ_WRITE;
         if (!(flags & PTE_EXEC))
             result |= ARCH_FLAG_EXECUTE_DISABLE;
+
+        switch (cache) {
+        case CACHE_ANY:
+        case CACHE_WRITE_BACK:
+            break;
+        case CACHE_WRITE_THROUGH:
+            result |= ARCH_FLAG_WRITE_THROUGH;
+            break;
+        case CACHE_MMIO:
+            result |= ARCH_FLAG_CACHE_DISABLE | ARCH_FLAG_WRITE_THROUGH;
+            break;
+        case CACHE_WRITE_COMBINE:
+            result |= ARCH_FLAG_ATTRIBUTE_TABLE | ARCH_FLAG_WRITE_THROUGH;
+            break;
+        }
     }
     return result;
 }

arch/x86_64/sched.c

@@ -1,10 +1,12 @@
-#include <zinnia/status.h>
-#include <common/compiler.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/percpu.h>
-#include <kernel/sched.h>
-#include <kernel/virt.h>
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/sched.h>
+#include <zinnia/vma.h>
+#include <arch/sched.h>
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -12,6 +14,7 @@
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
 #include <x86_64/idt.h>
+#include <x86_64/percpu.h>
 
 // The task frame consists of registers that the C ABI marks as callee-saved.
 // If we don't save them, these registers are lost during a context switch.
@@ -20,46 +23,45 @@ struct task_frame {
 };
 
 [[__naked]]
-static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) {
+static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) {
     asm volatile(
         "sub rsp, 0x30\n" // Make room for all regs (except RIP).
-        "mov [rsp + %c0], rbx\n"
-        "mov [rsp + %c1], rbp\n"
-        "mov [rsp + %c2], r12\n"
-        "mov [rsp + %c3], r13\n"
-        "mov [rsp + %c4], r14\n"
-        "mov [rsp + %c5], r15\n"
+        "mov [rsp + %c[rbx]], rbx\n"
+        "mov [rsp + %c[rbp]], rbp\n"
+        "mov [rsp + %c[r12]], r12\n"
+        "mov [rsp + %c[r13]], r13\n"
+        "mov [rsp + %c[r14]], r14\n"
+        "mov [rsp + %c[r15]], r15\n"
         "mov [rdi], rsp\n" // rdi = old_rsp
         "mov rsp, rsi\n"   // rsi = new_rsp
-        "mov rbx, [rsp + %c0]\n"
-        "mov rbp, [rsp + %c1]\n"
-        "mov r12, [rsp + %c2]\n"
-        "mov r13, [rsp + %c3]\n"
-        "mov r14, [rsp + %c4]\n"
-        "mov r15, [rsp + %c5]\n"
+        "mov rbx, [rsp + %c[rbx]]\n"
+        "mov rbp, [rsp + %c[rbp]]\n"
+        "mov r12, [rsp + %c[r12]]\n"
+        "mov r13, [rsp + %c[r13]]\n"
+        "mov r14, [rsp + %c[r14]]\n"
+        "mov r15, [rsp + %c[r15]]\n"
         "add rsp, 0x30\n"
-        "call %c6\n"
+        "mov rax, rdx\n"
         "ret" // This will conveniently move us to the RIP we put at this stack entry.
         :
-        : "i"(offsetof(struct task_frame, rbx)),
-          "i"(offsetof(struct task_frame, rbp)),
-          "i"(offsetof(struct task_frame, r12)),
-          "i"(offsetof(struct task_frame, r13)),
-          "i"(offsetof(struct task_frame, r14)),
-          "i"(offsetof(struct task_frame, r15)),
-          "i"(irq_unlock)
+        : [rbx] "i"(offsetof(struct task_frame, rbx)),
+          [rbp] "i"(offsetof(struct task_frame, rbp)),
+          [r12] "i"(offsetof(struct task_frame, r12)),
+          [r13] "i"(offsetof(struct task_frame, r13)),
+          [r14] "i"(offsetof(struct task_frame, r14)),
+          [r15] "i"(offsetof(struct task_frame, r15))
         : "memory"
     );
 }
 
-void arch_sched_switch(struct task* from, struct task* to) {
-    struct arch_percpu* cpu = &percpu_get()->arch;
-    struct arch_task_context* from_ctx = &from->task_ctx;
-    struct arch_task_context* to_ctx = &to->task_ctx;
+struct task* arch_sched_switch(struct task* from, struct task* to) {
+    struct arch_percpu* cpu = percpu_ref(arch_data);
+    struct arch_task_context* from_ctx = &from->context;
+    struct arch_task_context* to_ctx = &to->context;
 
-    cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
+    cpu->tss.rsp0 = kernel_stack_top(&to->kernel_stack);
 
-    if (from->user_stack) {
+    if (from_ctx->fpu_region != nullptr) {
         cpu->fpu_save(from_ctx->fpu_region);
         from_ctx->ds = asm_read_ds();
         from_ctx->es = asm_read_es();
@@ -69,7 +71,7 @@ void arch_sched_switch(struct task* from, struct task* to) {
         from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
     }
 
-    if (to->user_stack) {
+    if (to_ctx->fpu_region != nullptr) {
         cpu->fpu_restore(to_ctx->fpu_region);
         asm_write_ds(to_ctx->ds);
         asm_write_es(to_ctx->es);
@@ -86,32 +88,36 @@ void arch_sched_switch(struct task* from, struct task* to) {
         asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base);
     }
 
-    perform_switch(&from_ctx->rsp, to_ctx->rsp);
+    return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
 }
 
+// Calls task_entry with the correct parameters.
 [[__naked]]
 void task_entry_thunk() {
     asm volatile(
         "mov rdi, rbx\n"
         "mov rsi, r12\n"
-        "jmp %c0"
+        "mov rdx, rax\n"
+        "jmp %0"
         :
-        : "i"(task_entry)
-        : "memory"
+        : "r"(task_entry)
+        : "memory", "rdi", "rsi", "rdx"
     );
 }
 
-zn_status_t arch_task_init(
+errno_t arch_task_init(
     struct arch_task_context* context,
     void* entry,
     uintptr_t arg0,
     uintptr_t stack_start,
     bool is_user
 ) {
-    struct arch_percpu* cpu = &percpu_get()->arch;
+    struct arch_percpu* cpu = percpu_ref(arch_data);
+
+    assert((stack_start & 0xF) == 0, "Task stack is not 16-byte aligned!\n");
 
     // Prepare a dummy stack with an entry point function to return to.
-    struct task_frame* frame = (struct task_frame*)stack_start - 1;
+    struct task_frame* frame = (struct task_frame*)(stack_start - sizeof(uint64_t)) - 1;
     frame->rbx = (uint64_t)entry;
     frame->r12 = arg0;
     frame->rip = (uint64_t)task_entry_thunk;
@@ -122,18 +128,17 @@ zn_status_t arch_task_init(
         const size_t fpu_size = cpu->fpu_size;
         uintptr_t fpu_base = vm_alloc(fpu_size);
         if (!fpu_base)
-            return ZN_ERR_NO_MEMORY;
-        for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
-            phys_t page;
-            zn_status_t s = mem_phys_alloc(1, 0, &page);
-            if (s) {
-                vm_free(fpu_base, fpu_size);
-                return s;
-            }
-
-            pmap_map(&kernel_vas.pmap, fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
+            return ENOMEM;
+        phys_t pages;
+        errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages);
+        if (s) {
+            vm_free(fpu_base, fpu_size);
+            return s;
         }
+        for (size_t i = 0; i < fpu_size; i += arch_mem_page_size())
+            pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
         context->fpu_region = (void*)fpu_base;
+        context->fpu_phys = pages;
 
         context->ds = asm_read_ds();
         context->es = asm_read_es();
@@ -143,7 +148,22 @@ zn_status_t arch_task_init(
         context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
     }
 
-    return ZN_OK;
+    return 0;
+}
+
+void arch_task_fini(struct arch_task_context* context) {
+    if (context->fpu_region == nullptr)
+        return;
+
+    const size_t page_size = arch_mem_page_size();
+    const size_t fpu_size = percpu_ref(arch_data)->fpu_size;
+    const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
+    for (size_t i = 0; i < fpu_size; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, fpu_base + i);
+    mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
+    vm_free(fpu_base, fpu_size);
+
+    context->fpu_region = nullptr;
 }
 
 [[__naked]]
@@ -173,7 +193,7 @@ bool arch_sched_preempt_enable() {
 void arch_sched_jump_to_context(struct arch_context* context) {
     asm volatile(
         "mov rsp, rdi\n"
-        "jmp %c0\n" ::"i"(interrupt_return)
+        "jmp %0\n" ::"r"(interrupt_return)
     );
 }
 

arch/x86_64/serial.c

@@ -1,4 +1,4 @@
-#include <kernel/console.h>
+#include <zinnia/console.h>
 #include <stdint.h>
 #include <x86_64/asm.h>
 

arch/x86_64/smp.c

@@ -0,0 +1,352 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/compiler.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/string.h>
+#include <zinnia/types.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stdatomic.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <uacpi/acpi.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+#include <x86_64/pmap.h>
+#include <x86_64/smp.h>
+
+extern uint8_t smp_trampoline_start[];
+extern uint8_t smp_trampoline_data[];
+extern uint8_t smp_trampoline_end[];
+
+[[__naked, __used]]
+static void smp_trampoline() {
+    asm volatile(
+        ".pushsection .rodata\n"
+        ".code16\n"
+        ".global smp_trampoline_start\n"
+        "smp_trampoline_start:\n"
+        "    cli\n"
+        "    cld\n"
+        "    jmp .Lmode16\n"
+
+        ".global smp_trampoline_data\n"
+        "smp_trampoline_data:\n"
+        "    .skip %c[info_size]\n"
+
+        ".set .Ldata_offset, (smp_trampoline_data - smp_trampoline_start)\n"
+        ".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
+        ".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
+        ".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
+        ".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
+        ".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
+        ".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
+
+        ".Lmode16:\n"
+        "    mov bx, cs\n"
+        "    shl ebx, 4\n"
+
+        "    lidtd cs:(.Linvalid_idtr - smp_trampoline_start)\n"
+        "    lgdtd cs:.Lgdtr_offset\n"
+
+        "    lea eax, [ebx + (.Lmode32 - smp_trampoline_start)]\n"
+        "    mov dword ptr cs:.Lfarjmp_offset, eax\n"
+
+        "    mov eax, %c[cr0]\n"
+        "    mov cr0, eax\n"
+        "    jmp fword ptr cs:.Lfarjmp_offset\n"
+
+        ".code32\n"
+        ".Lmode32:\n"
+        "    mov ax, %c[kernel32_ds]\n"
+        "    mov ds, ax\n"
+        "    mov es, ax\n"
+        "    mov fs, ax\n"
+        "    mov gs, ax\n"
+        "    mov ss, ax\n"
+
+        "    xor eax, eax\n"
+        "    lldt ax\n"
+
+        "    mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n"
+
+        "    xor eax, eax\n"
+        "    mov cr4, eax\n"
+
+        "    or eax, %c[cr4_pae]\n"
+        "    mov cr4, eax\n"
+
+        "    mov ecx, %c[msr_efer]\n"
+        "    mov eax, %c[efer_lme]\n"
+        "    xor edx, edx\n"
+        "    wrmsr\n"
+
+        "    mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n"
+        "    mov cr3, eax\n"
+
+        "    mov eax, cr0\n"
+        "    or eax, %c[cr0_pg]\n"
+        "    mov cr0, eax\n"
+
+        "    lea eax, [ebx + (.Lmode64 - smp_trampoline_start)]\n"
+        "    push %c[kernel64_cs]\n"
+        "    push eax\n"
+        "    retf\n"
+
+        ".code64\n"
+        ".Lmode64:\n"
+        "    mov ax, %c[kernel64_ds]\n"
+        "    mov ds, ax\n"
+        "    mov es, ax\n"
+        "    mov fs, ax\n"
+        "    mov gs, ax\n"
+        "    mov ss, ax\n"
+
+        "    mov ecx, %c[msr_efer]\n"
+        "    rdmsr\n"
+        "    or eax, %c[efer_nxe]\n"
+        "    wrmsr\n"
+
+        "    xor ebp, ebp\n"
+
+        "    lea rdi, [rbx + .Ldata_offset]\n"
+        "    mov rax, [rbx + .Lentry_offset]\n"
+        "    add rsp, [rbx + .Lhhdm_offset]\n"
+        "    jmp rax\n"
+
+        ".Linvalid_idtr:\n"
+        "    .word 0\n"
+        "    .quad 0\n"
+
+        ".global smp_trampoline_end\n"
+        "smp_trampoline_end:\n"
+        ".popsection\n"
+        "    ud2\n"
+        :
+        : [info_size] "i"(sizeof(struct smp_info)),
+          [gdtr] "i"(offsetof(struct smp_info, gdtr)),
+          [farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
+          [temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
+          [temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)),
+          [entry] "i"(offsetof(struct smp_info, entry)),
+          [hhdm] "i"(offsetof(struct smp_info, hhdm_offset)),
+          [kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)),
+          [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
+          [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
+          [cr0] "i"(CR0_PE | CR0_ET),
+          [cr0_pg] "i"(CR0_PG),
+          [cr4_pae] "i"(CR4_PAE),
+          [msr_efer] "i"(MSR_EFER),
+          [efer_lme] "i"(MSR_EFER_LME),
+          [efer_nxe] "i"(MSR_EFER_NXE)
+    );
+}
+
+constexpr size_t AP_BOOT_STACK_SIZE = 0x8000;
+constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000;
+
+[[noreturn]]
+static void ap_entry(phys_t info_addr) {
+    // We're still running on the temporary page table.
+    pmap_set(&kernel_vma.pmap);
+
+    struct smp_info* info = HHDM_PTR(info_addr);
+    struct percpu* cpu = (struct percpu*)info->percpu;
+
+    x86_64_cpu_setup(cpu);
+    irq_lock();
+
+    // This task keeps running on the boot stack, but is never scheduled again after the first yield.
+    sched_init(&cpu->sched);
+
+    // Let the BSP know that we're alive.
+    // `info` must not be touched after this point.
+    atomic_store_explicit(&info->booted, 1, memory_order_release);
+
+    log("smp: CPU %zu online\n", cpu->id);
+
+    sched_yield(&cpu->sched);
+    unreachable();
+}
+
+static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
+    const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
+
+    phys_t stack;
+    if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
+        log("smp: Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
+        return false;
+    }
+
+    struct percpu* cpu = percpu_new();
+    if (!cpu) {
+        log("smp: Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
+        mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
+        return false;
+    }
+
+    log("smp: Starting CPU %zu\n", cpu->id);
+
+    memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
+
+    const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start);
+    struct smp_info* info = HHDM_PTR(info_addr);
+    memcpy(&info->gdt, &gdt_base, sizeof(info->gdt));
+    info->gdtr.limit = sizeof(struct gdt) - 1;
+    info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt));
+    info->farjmp_offset = 0; // Filled in by the trampoline.
+    info->farjmp_segment = offsetof(struct gdt, kernel_code32);
+    info->temp_stack = stack + AP_BOOT_STACK_SIZE;
+    info->temp_cr3 = temp_cr3;
+    info->entry = (uint64_t)ap_entry;
+    info->hhdm_offset = mem_hhdm_base;
+    info->percpu = (uint64_t)cpu;
+    info->lapic_id = lapic_id;
+    atomic_init(&info->booted, 0);
+
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+
+    lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE);
+    clock_spin_ns(10'000'000);
+
+    lapic_send_ipi(
+        lapic,
+        lapic_id,
+        trampoline >> 12,
+        APIC_DELIVERY_STARTUP,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    clock_spin_ns(10'000'000);
+
+    for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) {
+        if (waited >= AP_BOOT_TIMEOUT_NS) {
+            // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
+            log("smp: CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
+            return false;
+        }
+        clock_spin_ns(1'000'000);
+    }
+
+    return true;
+}
+
+static errno_t build_temp_table(phys_t* out) {
+    const size_t pgsz = arch_mem_page_size();
+
+    phys_t mem;
+    errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem);
+    if (status)
+        return status;
+
+    uint64_t* l4 = HHDM_PTR(mem);
+    uint64_t* l3 = HHDM_PTR(mem + pgsz);
+    const phys_t l2_base = mem + 2 * pgsz;
+
+    // Use 2 MiB pages.
+    l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    for (size_t i = 0; i < 4; i++) {
+        uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz);
+        l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        for (size_t j = 0; j < 512; j++)
+            l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE;
+    }
+
+    // Copy over the higher half from the kernel page table.
+    const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root);
+    for (size_t i = 256; i < 512; i++)
+        l4[i] = kernel_l4[i];
+
+    *out = mem;
+    return 0;
+}
+
+void x86_64_smp_init() {
+    const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
+    assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page\n");
+
+    const size_t max_cpus = percpu_max_cpus();
+    if (max_cpus <= 1)
+        return;
+
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log("smp: No MADT found, not starting any APs\n");
+        return;
+    }
+
+    phys_t trampoline;
+    if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
+        log("smp: Failed to allocate 20-bit memory for the AP trampoline\n");
+        uacpi_table_unref(&table);
+        return;
+    }
+
+    phys_t temp_table;
+    if (build_temp_table(&temp_table)) {
+        log("smp: Failed to allocate a temporary page table\n");
+        mem_phys_free(trampoline, 1);
+        uacpi_table_unref(&table);
+        return;
+    }
+
+    const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id;
+    bool all_booted = true;
+    size_t num_cpus = 1;
+
+    const struct acpi_madt* madt = table.ptr;
+    const uint8_t* entry = (const uint8_t*)madt->entries;
+    const uint8_t* end = (const uint8_t*)madt + madt->hdr.length;
+
+    while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) {
+        const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry;
+
+        // Don't trust malformed firmware tables.
+        if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
+            log("smp: MADT contains a malformed entry\n");
+            break;
+        }
+
+        uint32_t lapic_id = 0;
+        bool enabled = false;
+        switch (hdr->type) {
+        case ACPI_MADT_ENTRY_TYPE_LAPIC:
+            const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr;
+            lapic_id = lapic->id;
+            enabled = lapic->flags & ACPI_PIC_ENABLED;
+            break;
+        case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC:
+            const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr;
+            lapic_id = x2apic->id;
+            enabled = x2apic->flags & ACPI_PIC_ENABLED;
+            break;
+        }
+
+        if (enabled && lapic_id != bsp_id) {
+            if (start_ap(trampoline, temp_table, lapic_id))
+                num_cpus++;
+            else
+                all_booted = false;
+        }
+
+        entry += hdr->length;
+    }
+
+    uacpi_table_unref(&table);
+
+    // An AP which didn't respond might still be using these.
+    if (all_booted) {
+        mem_phys_free(temp_table, 6);
+        mem_phys_free(trampoline, 1);
+    }
+
+    log("smp: %zu CPU(s) online\n", num_cpus);
+}

arch/x86_64/sys.c

@@ -0,0 +1,19 @@
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/init.h>
+#include <zinnia/percpu.h>
+#include <uapi/archctl.h>
+#include <x86_64/apic.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+
+[[__init, __naked, __used]]
+void _start() {
+    asm volatile(
+        "mov rsp, %0\n"
+        "jmp %1"
+        :
+        : "r"(__ld_stack_top), "r"(kernel_entry)
+    );
+}

arch/x86_64/syscall.c

@@ -0,0 +1,71 @@
+#include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
+#include <zinnia/syscall.h>
+#include <bits/sched.h>
+#include <stddef.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+
+// TODO: Refactor to only push callee-saved registers.
+
+[[__naked]]
+void arch_syscall_stub() {
+    asm volatile(
+        "    swapgs\n"
+        "    mov gs:%c[user_stack], rsp\n"
+        "    mov rsp, gs:%c[kernel_stack]\n"
+        "    cld\n"
+        "1:\n"
+        "    push %c[user_data]\n" // ss
+        "    push gs:%c[user_stack]\n"
+        "    push r11\n"             // rflags
+        "    push %c[user_code64]\n" // cs
+        "    push rcx\n"             // rip
+        "    push 0x00\n"            // arch_context::error
+        "    push 0x00\n"            // arch_context::isr
+        "    push rax\n"
+        "    push rbx\n"
+        "    push rcx\n"
+        "    push rdx\n"
+        "    push rbp\n"
+        "    push rdi\n"
+        "    push rsi\n"
+        "    push r8\n"
+        "    push r9\n"
+        "    push r10\n"
+        "    push r11\n"
+        "    push r12\n"
+        "    push r13\n"
+        "    push r14\n"
+        "    push r15\n"
+        "    xor rbp, rbp\n"
+        "    mov rdi, rsp\n"          // SysV ABI first argument.
+        "    call syscall_dispatch\n" // Call syscall handler.
+        "    cli\n"
+        "    pop r15\n"
+        "    pop r14\n"
+        "    pop r13\n"
+        "    pop r12\n"
+        "    pop r11\n"
+        "    pop r10\n"
+        "    pop r9\n"
+        "    pop r8\n"
+        "    pop rsi\n"
+        "    pop rdi\n"
+        "    pop rbp\n"
+        "    pop rdx\n"
+        "    pop rcx\n"
+        "    pop rbx\n"
+        "    pop rax\n"
+        "    add rsp, 0x10\n" // Skip .error and .isr fields
+        "2:\n"
+        "    mov rsp, gs:%c[user_stack]\n"
+        "    swapgs\n"
+        "    sysretq\n"
+        :
+        : [user_stack] "i"(offsetof(struct percpu, user_stack)),
+          [kernel_stack] "i"(offsetof(struct percpu, kernel_stack)),
+          [user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER),
+          [user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER)
+    );
+}

arch/x86_64/tsc.c

@@ -0,0 +1,85 @@
+#include <zinnia/clock.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <stdint.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+
+static bool use_tsc = true;
+static void tsc_option(bool opt) {
+    use_tsc = opt;
+}
+CMDLINE_OPTION("x86_64.tsc", "Enable timestamp counter", tsc_option);
+
+constexpr uint64_t nanos_per_sec = 1'000'000'000;
+static uint64_t tsc_freq = 0;
+static uint64_t tsc_base_cycles = 0;
+
+static uint64_t cycles_to_ns(uint64_t cycles, uint64_t freq) {
+    uint64_t seconds = cycles / freq;
+    uint64_t remaining_cycles = cycles % freq;
+    uint64_t fractional_ns = (uint64_t)((__uint128_t)remaining_cycles * (__uint128_t)nanos_per_sec / (__uint128_t)freq);
+    uint64_t ns = seconds * nanos_per_sec + fractional_ns;
+
+    return MIN(ns, UINT64_MAX);
+}
+
+static void tsc_reset(struct clock* clock) {
+    tsc_base_cycles = asm_rdtsc();
+}
+
+static uint64_t tsc_get_elapsed_ns(struct clock* clock) {
+    uint64_t cycles = asm_rdtsc() - tsc_base_cycles;
+    return cycles_to_ns(cycles, tsc_freq);
+}
+
+static struct clock tsc_clock = {
+    .name = "tsc",
+    .priority = 255,
+    .reset = tsc_reset,
+    .get_elapsed_ns = tsc_get_elapsed_ns,
+};
+
+void tsc_init() {
+    if (!use_tsc)
+        return;
+
+    if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
+        log("x86_64/tsc: No invariant TSC detected!\n");
+        return;
+    }
+
+    // Check for the TSC info leaf.
+    bool has_tsc_leaf = false;
+    struct cpuid tsc_leaf = {0};
+    if (asm_cpuid(0, 0).eax >= 0x15) {
+        tsc_leaf = asm_cpuid(0x15, 0);
+        has_tsc_leaf = true;
+    }
+
+    if (clock_available()) {
+        // Calibrate using existing clock.
+        uint64_t t1 = asm_rdtsc();
+        clock_spin_ns(10'000'000);
+        uint64_t t2 = asm_rdtsc();
+        tsc_freq = (t2 - t1) * 100;
+    } else if (has_tsc_leaf) {
+        // If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency.
+        // On a normal system, this should usually never be called and is a last resort
+        // since at this point we have at least the HPET timer.
+        if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
+            log("x86_64/tsc: Calibrating TSC using CPUID 0x15");
+            tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
+        } else {
+            log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information");
+            return;
+        }
+    } else {
+        log("x86_64/tsc: Unable to calibrate the TSC without a clock or static frequency info!\n");
+        return;
+    }
+
+    log("x86_64/tsc: Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
+    clock_switch(&tsc_clock);
+}

arch/x86_64/usercopy.c

@@ -0,0 +1,142 @@
+#include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
+#include <arch/usercopy.h>
+#include <x86_64/percpu.h>
+
+void arch_usercopy_allow(bool state) {
+    if (!percpu_ref(arch_data)->can_smap)
+        return;
+
+    if (state) {
+        asm volatile("stac" ::: "memory");
+    } else {
+        asm volatile("clac" ::: "memory");
+    }
+}
+
+extern void x86_64_read_start();
+extern void x86_64_read_end();
+extern void x86_64_read_fault();
+
+static const struct usercopy_region read_region = {
+    .start_ip = x86_64_read_start,
+    .end_ip = x86_64_read_end,
+    .fault_ip = x86_64_read_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) {
+    asm volatile(
+        // Setup regs for `rep movsb`.
+        "    xchg rcx, rdx\n"
+        "    lea rax, [rip + %0]\n"
+        "    xchg [rdx], rax\n"
+
+        ".global x86_64_read_start\n"
+        "    x86_64_read_start:\n"
+        "    rep movsb\n"
+
+        ".global x86_64_read_end\n"
+        "x86_64_read_end:\n"
+        "    xor rax, rax\n"
+        "    xchg [rdx], rax\n"
+        "    mov rax, 1\n"
+        "    ret\n"
+
+        ".global x86_64_read_fault\n"
+        "x86_64_read_fault:\n"
+        "    xor rax, rax\n"
+        "    mov [rdx], rax\n"
+        "    ret\n"
+        :
+        : "i"(&read_region)
+        : "memory"
+    );
+}
+
+extern void x86_64_write_start();
+extern void x86_64_write_end();
+extern void x86_64_write_fault();
+
+static const struct usercopy_region write_region = {
+    .start_ip = x86_64_write_start,
+    .end_ip = x86_64_write_end,
+    .fault_ip = x86_64_write_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) {
+    asm volatile(
+        // Setup regs for `rep movsb`.
+        "    xchg rcx, rdx\n"
+        "    lea rax, [rip + %0]\n"
+        "    mov [rdx], rax\n"
+
+        ".global x86_64_write_start\n"
+        "    x86_64_write_start:\n"
+        "    rep movsb\n"
+
+        ".global x86_64_write_end\n"
+        "    x86_64_write_end:\n"
+        "    xor rax, rax\n"
+        "    xchg [rdx], rax\n"
+        "    mov rax, 1\n"
+        "    ret\n"
+
+        ".global x86_64_write_fault\n"
+        "x86_64_write_fault:\n"
+        "    xor rax, rax\n"
+        "    mov [rdx], rax\n"
+        "    ret\n"
+        :
+        : "i"(&write_region)
+        : "memory"
+    );
+}
+
+extern void x86_64_strlen_start();
+extern void x86_64_strlen_end();
+extern void x86_64_strlen_fault();
+
+static const struct usercopy_region strlen_region = {
+    .start_ip = x86_64_strlen_start,
+    .end_ip = x86_64_strlen_end,
+    .fault_ip = x86_64_strlen_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
+    asm volatile(
+        "    lea rax, [rip + %0]\n"
+        "    xchg [rcx], rax\n"
+
+        ".global x86_64_strlen_start\n"
+        "    x86_64_strlen_start:\n"
+        "    xor r8, r8\n"
+        ".Lloop:\n"
+        "    cmp r8, rsi\n"
+        "    je .Lleave\n"
+        "    cmp byte ptr [rdi + r8], 0\n"
+        "    je .Lleave\n"
+        "    inc r8\n"
+        "    jmp .Lloop\n"
+        ".Lleave:\n"
+        "    mov [rdx], r8\n"
+
+        ".global x86_64_strlen_end\n"
+        "x86_64_strlen_end:\n"
+        "    xor rax, rax\n"
+        "    xchg [rcx], rax\n"
+        "    mov rax, 1\n"
+        "    ret\n"
+
+        ".global x86_64_strlen_fault\n"
+        "x86_64_strlen_fault:\n"
+        "    xor rax, rax\n"
+        "    mov [rcx], rax\n"
+        "    ret\n"
+        :
+        : "i"(&strlen_region)
+        : "memory"
+    );
+}

boot/cmdline.c

@@ -1,8 +1,8 @@
-#include <kernel/cmdline.h>
-#include <kernel/init.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/init.h>
 #include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
+#include <zinnia/stdlib.h>
+#include <zinnia/string.h>
 
 // Defined in linker script.
 extern const struct cmdline_option* __ld_cmdline_start[];
@@ -12,6 +12,9 @@ static char cmdline_buf[0x1000];
 
 [[__init]]
 void cmdline_parse(const char* cmdline) {
+    if (cmdline == nullptr)
+        cmdline = "";
+
     const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
     size_t idx = 0;
     memcpy(cmdline_buf, cmdline, len);
@@ -62,6 +65,8 @@ void cmdline_parse(const char* cmdline) {
         while (opt < __ld_cmdline_end) {
             const struct cmdline_option* current = *opt;
             if (!strcmp(current->name, name)) {
+                if (value == nullptr)
+                    break;
                 switch (current->option_type) {
                 case CMDLINE_STRING:
                     current->func_str(value);

boot/kernel.ld

@@ -1,4 +1,4 @@
-ENTRY(arch_start)
+ENTRY(_start)
 
 PHDRS {
     text    PT_LOAD     FLAGS(5);
@@ -23,18 +23,22 @@ SECTIONS
         __ld_text_end = .;
     } :text
 
-    .rodata ALIGN(CONSTANT(MAXPAGESIZE)) : {
+    .cmdline ALIGN(CONSTANT(MAXPAGESIZE)) : {
         __ld_rodata_start = .;
-
         __ld_cmdline_start = .;
         KEEP(*(.cmdline))
         __ld_cmdline_end = .;
+    } :rodata
 
-        __ld_vdso_start = .;
-        KEEP(*(.vdso))
-        __ld_vdso_end = .;
-
+    .rodata : {
         *(.rodata .rodata.*)
+
+        /* Builtin module entry points */
+        . = ALIGN(8);
+        __ld_mod_entry_start = .;
+        KEEP(*(.mod.entry .mod.entry.*))
+        __ld_mod_entry_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -43,44 +47,49 @@ SECTIONS
     .data ALIGN(CONSTANT(MAXPAGESIZE)) : {
         __ld_data_start = .;
 
-        __ld_console_start = .;
-        KEEP(*(.console))
-        __ld_console_end = .;
-
         KEEP(*(SORT_BY_NAME(.init.data.*)))
         KEEP(*(.init.data))
         *(.data .data.*)
     } :data
 
+    .console : {
+        __ld_console_start = .;
+        KEEP(*(.console))
+        __ld_console_end = .;
+    } :data
+
     .dynsym : {
         __ld_dynsym_start = .;
-        *(.dynsym)
+        KEEP(*(.dynsym*))
         __ld_dynsym_end = .;
     } :data
 
     .dynstr : {
         __ld_dynstr_start = .;
-        *(.dynstr)
+        KEEP(*(.dynstr*))
         __ld_dynstr_end = .;
     } :data
 
     .dynamic : {
-        *(.dynamic)
+        *(.dynamic*)
     } :data :dynamic
 
-    .bss : {
+    .bss ALIGN(CONSTANT(MAXPAGESIZE)) (NOLOAD) : {
+        *(.*bss .*bss.*)
+        *(COMMON)
+
         /* Small stack to bootstrap the allocator */
+        . = ALIGN(16);
         __ld_stack_bottom = .;
         . += 0x2000;
         __ld_stack_top = .;
-        *(.*bss .*bss.*)
-        *(COMMON)
     } :data
 
     /* Per-CPU data block. MUST always come last. */
     .percpu ALIGN(CONSTANT(MAXPAGESIZE)) : {
         __ld_percpu_start = .;
-        KEEP(*(.percpu))
+        KEEP(*(.percpu.init .percpu.init*))
+        KEEP(*(.percpu.data .percpu.data*))
         . = ALIGN(CONSTANT(MAXPAGESIZE));
         __ld_percpu_end = .;
 

boot/limine/entry.c

@@ -1,15 +1,15 @@
-#include <common/utils.h>
-#include <kernel/alloc.h>
-#include <kernel/cmdline.h>
-#include <kernel/init.h>
-#include <kernel/print.h>
-#include <string.h>
+#include <zinnia/alloc.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/init.h>
+#include <zinnia/print.h>
+#include <zinnia/string.h>
+#include <zinnia/utils.h>
 #include "limine.h"
 
 [[__initdata_sorted("limine.0")]]
 static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
 [[__initdata_sorted("limine.1")]]
-static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(4);
+static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
 [[__initdata_sorted("limine.2")]]
 static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
 
@@ -39,7 +39,7 @@ static struct boot_file files[8];
 void kernel_entry() {
     kernel_early_init();
 
-    struct boot_info info;
+    struct boot_info info = {};
 
     struct limine_memmap_response* mm_res = memmap_request.response;
     struct limine_module_response* module_res = module_request.response;
@@ -76,7 +76,8 @@ void kernel_entry() {
         for (size_t i = 0; i < info.num_files; i++) {
             files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset;
             files[i].length = module_res->modules[i]->size;
-            strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path));
+            strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1);
+            files[i].path[sizeof(files[i].path) - 1] = 0;
         }
     }
 
@@ -85,9 +86,13 @@ void kernel_entry() {
     info.virt_base = (uintptr_t)exec_res->virtual_base;
     info.hhdm_base = (uintptr_t)hhdm_request.response->offset;
 
-    info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base;
+#if CONFIG_ACPI
+    if (rsdp_request.response) {
+        info.rsdp = (phys_t)rsdp_request.response->address;
+        if (info.rsdp >= info.hhdm_base)
+            info.rsdp -= info.hhdm_base;
+    }
+#endif
 
     kernel_main(&info);
 }
-
-phys_t rsdp_addr;

boot/limine/limine.h

@@ -0,0 +1,669 @@
+/* SPDX-License-Identifier: 0BSD */
+
+/* Copyright (C) 2022-2026 Mintsuki and contributors.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef LIMINE_H
+#define LIMINE_H 1
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Misc */
+
+#ifdef LIMINE_NO_POINTERS
+#  define LIMINE_PTR(TYPE) uint64_t
+#else
+#  define LIMINE_PTR(TYPE) TYPE
+#endif
+
+#define LIMINE_REQUESTS_START_MARKER { 0xf6b8f4b39de7d1ae, 0xfab91a6940fcb9cf, \
+                                       0x785c6ed015d3e316, 0x181e920a7852b9d9 }
+#define LIMINE_REQUESTS_END_MARKER { 0xadc0e0531bb10d03, 0x9572709f31764c62 }
+
+#define LIMINE_BASE_REVISION(N) { 0xf9562b2d5c95a6c8, 0x6a7b384944536bdc, (N) }
+
+#define LIMINE_BASE_REVISION_SUPPORTED(VAR) ((VAR)[2] == 0)
+
+#define LIMINE_LOADED_BASE_REVISION_VALID(VAR) ((VAR)[1] != 0x6a7b384944536bdc)
+#define LIMINE_LOADED_BASE_REVISION(VAR) ((VAR)[1])
+
+#define LIMINE_COMMON_MAGIC 0xc7b1dd30df4c8b88, 0x0a82e883a194f07b
+
+struct limine_uuid {
+    uint32_t a;
+    uint16_t b;
+    uint16_t c;
+    uint8_t d[8];
+};
+
+#define LIMINE_MEDIA_TYPE_GENERIC 0
+#define LIMINE_MEDIA_TYPE_OPTICAL 1
+#define LIMINE_MEDIA_TYPE_TFTP 2
+
+struct limine_file {
+    uint64_t revision;
+    LIMINE_PTR(void *) address;
+    uint64_t size;
+    LIMINE_PTR(char *) path;
+    LIMINE_PTR(char *) string;
+    uint32_t media_type;
+    uint32_t unused;
+    uint8_t tftp_ipv4[4];
+    uint32_t tftp_port;
+    uint32_t partition_index;
+    uint32_t mbr_disk_id;
+    struct limine_uuid gpt_disk_uuid;
+    struct limine_uuid gpt_part_uuid;
+    struct limine_uuid part_uuid;
+};
+
+/* Boot info */
+
+#define LIMINE_BOOTLOADER_INFO_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xf55038d8e2a1202f, 0x279426fcf5f59740 }
+
+struct limine_bootloader_info_response {
+    uint64_t revision;
+    LIMINE_PTR(char *) name;
+    LIMINE_PTR(char *) version;
+};
+
+struct limine_bootloader_info_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_bootloader_info_response *) response;
+};
+
+/* Executable command line */
+
+#define LIMINE_EXECUTABLE_CMDLINE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x4b161536e598651e, 0xb390ad4a2f1f303a }
+
+struct limine_executable_cmdline_response {
+    uint64_t revision;
+    LIMINE_PTR(char *) cmdline;
+};
+
+struct limine_executable_cmdline_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_executable_cmdline_response *) response;
+};
+
+/* Firmware type */
+
+#define LIMINE_FIRMWARE_TYPE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8c2f75d90bef28a8, 0x7045a4688eac00c3 }
+
+#define LIMINE_FIRMWARE_TYPE_X86BIOS 0
+#define LIMINE_FIRMWARE_TYPE_EFI32 1
+#define LIMINE_FIRMWARE_TYPE_EFI64 2
+#define LIMINE_FIRMWARE_TYPE_SBI 3
+
+struct limine_firmware_type_response {
+    uint64_t revision;
+    uint64_t firmware_type;
+};
+
+struct limine_firmware_type_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_firmware_type_response *) response;
+};
+
+/* Stack size */
+
+#define LIMINE_STACK_SIZE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x224ef0460a8e8926, 0xe1cb0fc25f46ea3d }
+
+struct limine_stack_size_response {
+    uint64_t revision;
+};
+
+struct limine_stack_size_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_stack_size_response *) response;
+    uint64_t stack_size;
+};
+
+/* HHDM */
+
+#define LIMINE_HHDM_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x48dcf1cb8ad2b852, 0x63984e959a98244b }
+
+struct limine_hhdm_response {
+    uint64_t revision;
+    uint64_t offset;
+};
+
+struct limine_hhdm_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_hhdm_response *) response;
+};
+
+/* Framebuffer */
+
+#define LIMINE_FRAMEBUFFER_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9d5827dcd881dd75, 0xa3148604f6fab11b }
+
+#define LIMINE_FRAMEBUFFER_RGB 1
+
+struct limine_video_mode {
+    uint64_t pitch;
+    uint64_t width;
+    uint64_t height;
+    uint16_t bpp;
+    uint8_t memory_model;
+    uint8_t red_mask_size;
+    uint8_t red_mask_shift;
+    uint8_t green_mask_size;
+    uint8_t green_mask_shift;
+    uint8_t blue_mask_size;
+    uint8_t blue_mask_shift;
+};
+
+struct limine_framebuffer {
+    LIMINE_PTR(void *) address;
+    uint64_t width;
+    uint64_t height;
+    uint64_t pitch;
+    uint16_t bpp;
+    uint8_t memory_model;
+    uint8_t red_mask_size;
+    uint8_t red_mask_shift;
+    uint8_t green_mask_size;
+    uint8_t green_mask_shift;
+    uint8_t blue_mask_size;
+    uint8_t blue_mask_shift;
+    uint8_t unused[7];
+    uint64_t edid_size;
+    LIMINE_PTR(void *) edid;
+    /* Response revision 1 */
+    uint64_t mode_count;
+    LIMINE_PTR(struct limine_video_mode **) modes;
+};
+
+struct limine_framebuffer_response {
+    uint64_t revision;
+    uint64_t framebuffer_count;
+    LIMINE_PTR(struct limine_framebuffer **) framebuffers;
+};
+
+struct limine_framebuffer_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_framebuffer_response *) response;
+};
+
+/* Flanterm FB init params */
+
+#define LIMINE_FLANTERM_FB_INIT_PARAMS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3259399fe7c5f126, 0xe01c1c8c5db9d1a9 }
+
+#define LIMINE_FLANTERM_FB_ROTATE_0 0
+#define LIMINE_FLANTERM_FB_ROTATE_90 1
+#define LIMINE_FLANTERM_FB_ROTATE_180 2
+#define LIMINE_FLANTERM_FB_ROTATE_270 3
+
+struct limine_flanterm_fb_init_params {
+    LIMINE_PTR(uint32_t *) canvas;
+    uint64_t canvas_size;
+    uint32_t ansi_colours[8];
+    uint32_t ansi_bright_colours[8];
+    uint32_t default_bg;
+    uint32_t default_fg;
+    uint32_t default_bg_bright;
+    uint32_t default_fg_bright;
+    LIMINE_PTR(void *) font;
+    uint64_t font_width;
+    uint64_t font_height;
+    uint64_t font_spacing;
+    uint64_t font_scale_x;
+    uint64_t font_scale_y;
+    uint64_t margin;
+    uint64_t rotation;
+};
+
+struct limine_flanterm_fb_init_params_response {
+    uint64_t revision;
+    uint64_t entry_count;
+    LIMINE_PTR(struct limine_flanterm_fb_init_params **) entries;
+};
+
+struct limine_flanterm_fb_init_params_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_flanterm_fb_init_params_response *) response;
+};
+
+/* Paging mode */
+
+#define LIMINE_PAGING_MODE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95c1a0edab0944cb, 0xa4e5cb3842f7488a }
+
+#define LIMINE_PAGING_MODE_X86_64_4LVL 0
+#define LIMINE_PAGING_MODE_X86_64_5LVL 1
+#define LIMINE_PAGING_MODE_X86_64_MIN LIMINE_PAGING_MODE_X86_64_4LVL
+#define LIMINE_PAGING_MODE_X86_64_DEFAULT LIMINE_PAGING_MODE_X86_64_4LVL
+
+#define LIMINE_PAGING_MODE_AARCH64_4LVL 0
+#define LIMINE_PAGING_MODE_AARCH64_5LVL 1
+#define LIMINE_PAGING_MODE_AARCH64_MIN LIMINE_PAGING_MODE_AARCH64_4LVL
+#define LIMINE_PAGING_MODE_AARCH64_DEFAULT LIMINE_PAGING_MODE_AARCH64_4LVL
+
+#define LIMINE_PAGING_MODE_RISCV_SV39 0
+#define LIMINE_PAGING_MODE_RISCV_SV48 1
+#define LIMINE_PAGING_MODE_RISCV_SV57 2
+#define LIMINE_PAGING_MODE_RISCV_MIN LIMINE_PAGING_MODE_RISCV_SV39
+#define LIMINE_PAGING_MODE_RISCV_DEFAULT LIMINE_PAGING_MODE_RISCV_SV48
+
+#define LIMINE_PAGING_MODE_LOONGARCH_4LVL 0
+#define LIMINE_PAGING_MODE_LOONGARCH_MIN LIMINE_PAGING_MODE_LOONGARCH_4LVL
+#define LIMINE_PAGING_MODE_LOONGARCH_DEFAULT LIMINE_PAGING_MODE_LOONGARCH_4LVL
+
+struct limine_paging_mode_response {
+    uint64_t revision;
+    uint64_t mode;
+};
+
+struct limine_paging_mode_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_paging_mode_response *) response;
+    uint64_t mode;
+    uint64_t max_mode;
+    uint64_t min_mode;
+};
+
+/* MP */
+
+#define LIMINE_MP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95a67b819a1b857e, 0xa0b61b723b6a73e0 }
+
+struct limine_mp_info;
+
+typedef void (*limine_goto_address)(struct limine_mp_info *);
+
+#if defined (__x86_64__) || defined (__i386__)
+
+#define LIMINE_MP_RESPONSE_X86_64_X2APIC (1 << 0)
+
+struct limine_mp_info {
+    uint32_t processor_id;
+    uint32_t lapic_id;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint32_t flags;
+    uint32_t bsp_lapic_id;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#elif defined (__aarch64__)
+
+struct limine_mp_info {
+    uint32_t processor_id;
+    uint32_t reserved1;
+    uint64_t mpidr;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint64_t flags;
+    uint64_t bsp_mpidr;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#elif defined (__riscv) && (__riscv_xlen == 64)
+
+struct limine_mp_info {
+    uint64_t processor_id;
+    uint64_t hartid;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint64_t flags;
+    uint64_t bsp_hartid;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#elif defined (__loongarch__) && (__loongarch_grlen == 64)
+
+struct limine_mp_info {
+    uint64_t processor_id;
+    uint64_t phys_id;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint64_t flags;
+    uint64_t bsp_phys_id;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#else
+#error Unknown architecture
+#endif
+
+#define LIMINE_MP_REQUEST_X86_64_X2APIC (1 << 0)
+
+struct limine_mp_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_mp_response *) response;
+    uint64_t flags;
+};
+
+/* Memory map */
+
+#define LIMINE_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x67cf3d9d378a806f, 0xe304acdfc50c3c62 }
+
+#define LIMINE_MEMMAP_USABLE                 0
+#define LIMINE_MEMMAP_RESERVED               1
+#define LIMINE_MEMMAP_ACPI_RECLAIMABLE       2
+#define LIMINE_MEMMAP_ACPI_NVS               3
+#define LIMINE_MEMMAP_BAD_MEMORY             4
+#define LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE 5
+#define LIMINE_MEMMAP_EXECUTABLE_AND_MODULES 6
+#define LIMINE_MEMMAP_FRAMEBUFFER            7
+#define LIMINE_MEMMAP_RESERVED_MAPPED        8
+
+struct limine_memmap_entry {
+    uint64_t base;
+    uint64_t length;
+    uint64_t type;
+};
+
+struct limine_memmap_response {
+    uint64_t revision;
+    uint64_t entry_count;
+    LIMINE_PTR(struct limine_memmap_entry **) entries;
+};
+
+struct limine_memmap_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_memmap_response *) response;
+};
+
+/* Entry point */
+
+#define LIMINE_ENTRY_POINT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x13d86c035a1cd3e1, 0x2b0caa89d8f3026a }
+
+typedef void (*limine_entry_point)(void);
+
+struct limine_entry_point_response {
+    uint64_t revision;
+};
+
+struct limine_entry_point_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_entry_point_response *) response;
+    LIMINE_PTR(limine_entry_point) entry;
+};
+
+/* Executable File */
+
+#define LIMINE_EXECUTABLE_FILE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xad97e90e83f1ed67, 0x31eb5d1c5ff23b69 }
+
+struct limine_executable_file_response {
+    uint64_t revision;
+    LIMINE_PTR(struct limine_file *) executable_file;
+};
+
+struct limine_executable_file_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_executable_file_response *) response;
+};
+
+/* Module */
+
+#define LIMINE_MODULE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3e7e279702be32af, 0xca1c4f3bd1280cee }
+
+#define LIMINE_INTERNAL_MODULE_REQUIRED (1 << 0)
+#define LIMINE_INTERNAL_MODULE_COMPRESSED (1 << 1)
+
+struct limine_internal_module {
+    LIMINE_PTR(const char *) path;
+    LIMINE_PTR(const char *) string;
+    uint64_t flags;
+};
+
+struct limine_module_response {
+    uint64_t revision;
+    uint64_t module_count;
+    LIMINE_PTR(struct limine_file **) modules;
+};
+
+struct limine_module_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_module_response *) response;
+
+    /* Request revision 1 */
+    uint64_t internal_module_count;
+    LIMINE_PTR(struct limine_internal_module **) internal_modules;
+};
+
+/* RSDP */
+
+#define LIMINE_RSDP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xc5e77b6b397e7b43, 0x27637845accdcf3c }
+
+struct limine_rsdp_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) address;
+};
+
+struct limine_rsdp_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_rsdp_response *) response;
+};
+
+/* SMBIOS */
+
+#define LIMINE_SMBIOS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9e9046f11e095391, 0xaa4a520fefbde5ee }
+
+struct limine_smbios_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) entry_32;
+    LIMINE_PTR(void *) entry_64;
+};
+
+struct limine_smbios_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_smbios_response *) response;
+};
+
+/* EFI system table */
+
+#define LIMINE_EFI_SYSTEM_TABLE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x5ceba5163eaaf6d6, 0x0a6981610cf65fcc }
+
+struct limine_efi_system_table_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) address;
+};
+
+struct limine_efi_system_table_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_efi_system_table_response *) response;
+};
+
+/* TPM event log */
+
+#define LIMINE_TPM_EVENT_LOG_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x98e094fc7e76e979, 0xee8d8775c54e1d1f }
+
+#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_1_2 1
+#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_2   2
+
+struct limine_tpm_event_log_response {
+    uint64_t revision;
+    uint64_t format;
+    uint64_t size;
+    LIMINE_PTR(void *) address;
+};
+
+struct limine_tpm_event_log_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_tpm_event_log_response *) response;
+};
+
+/* EFI memory map */
+
+#define LIMINE_EFI_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x7df62a431d6872d5, 0xa4fcdfb3e57306c8 }
+
+struct limine_efi_memmap_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) memmap;
+    uint64_t memmap_size;
+    uint64_t desc_size;
+    uint64_t desc_version;
+};
+
+struct limine_efi_memmap_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_efi_memmap_response *) response;
+};
+
+/* Date at boot */
+
+#define LIMINE_DATE_AT_BOOT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x502746e184c088aa, 0xfbc5ec83e6327893 }
+
+struct limine_date_at_boot_response {
+    uint64_t revision;
+    int64_t timestamp;
+};
+
+struct limine_date_at_boot_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_date_at_boot_response *) response;
+};
+
+/* Executable address */
+
+#define LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x71ba76863cc55f63, 0xb2644a48c516a487 }
+
+struct limine_executable_address_response {
+    uint64_t revision;
+    uint64_t physical_base;
+    uint64_t virtual_base;
+};
+
+struct limine_executable_address_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_executable_address_response *) response;
+};
+
+/* Device Tree Blob */
+
+#define LIMINE_DTB_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xb40ddb48fb54bac7, 0x545081493f81ffb7 }
+
+struct limine_dtb_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) dtb_ptr;
+};
+
+struct limine_dtb_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_dtb_response *) response;
+};
+
+/* RISC-V Boot Hart ID */
+
+#define LIMINE_RISCV_BSP_HARTID_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x1369359f025525f9, 0x2ff2a56178391bb6 }
+
+struct limine_riscv_bsp_hartid_response {
+    uint64_t revision;
+    uint64_t bsp_hartid;
+};
+
+struct limine_riscv_bsp_hartid_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_riscv_bsp_hartid_response *) response;
+};
+
+/* Bootloader Performance */
+
+#define LIMINE_BOOTLOADER_PERFORMANCE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x6b50ad9bf36d13ad, 0xdc4c7e88fc759e17 }
+
+struct limine_bootloader_performance_response {
+    uint64_t revision;
+    uint64_t reset_usec;
+    uint64_t init_usec;
+    uint64_t exec_usec;
+};
+
+struct limine_bootloader_performance_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_bootloader_performance_response *) response;
+};
+
+#define LIMINE_X86_64_KEEP_IOMMU_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8ebaabe51f490179, 0x2aa86a59ffb4ab0f }
+
+struct limine_x86_64_keep_iommu_response {
+    uint64_t revision;
+};
+
+struct limine_x86_64_keep_iommu_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_x86_64_keep_iommu_response *) response;
+};
+
+/* TSC (Timestamp Counter) Frequency */
+
+#define LIMINE_TSC_FREQUENCY_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x10f2ee1d87d195e4, 0xf747a2b78f6ddb31 }
+
+struct limine_tsc_frequency_response {
+    uint64_t revision;
+    uint64_t frequency;
+};
+
+struct limine_tsc_frequency_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_tsc_frequency_response *) response;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

boot/main.c

@@ -0,0 +1,102 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/compiler.h>
+#include <zinnia/console.h>
+#include <zinnia/elf.h>
+#include <zinnia/init.h>
+#include <zinnia/irq.h>
+#include <zinnia/module.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+#include <config.h>
+#include <stdint.h>
+
+#if CONFIG_ACPI
+#include <zinnia/acpi.h>
+#endif
+
+static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
+                                    ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
+
+static void kernel_main_task(uintptr_t arg0) {
+    struct boot_info* info = (struct boot_info*)arg0;
+
+    log("main: Hello from main task!\n");
+
+    // Initialize built-in modules.
+    module_init();
+
+    assert(info->num_files >= 1, "No init executable provided");
+
+    struct vma* init_space;
+    assert(vma_new(&init_space) == EOK, "");
+
+    struct boot_file* file_data = &info->files[0];
+    struct paged_vmo* init_file;
+    assert(vmo_new_phys(&init_file) == EOK, "");
+    vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
+
+    // Load init.
+    const char* init_argv[] = {"init", nullptr};
+    const char* init_envp[] = {nullptr};
+
+    struct exec_info exec = {
+        .space = init_space,
+        .file_obj = &init_file->object,
+        .argv = init_argv,
+        .envp = init_envp,
+    };
+
+    struct task* init_task = nullptr;
+    errno_t status = elf_load(&exec, &init_task);
+    vmo_unref(&init_file->object);
+
+    assert(status == 0, "Failed to load init process (%i)\n", status);
+    sched_add_task(&percpu_get()->sched, init_task);
+
+    while (true) {
+        irq_wait();
+    }
+}
+
+[[__init]]
+void kernel_early_init() {
+    percpu_bsp_init();
+    percpu_get()->online = true;
+    irq_lock();
+}
+
+[[noreturn]]
+void kernel_main(struct boot_info* info) {
+    // TODO: Initialize in-memory log ring.
+    kprintf("%s\n", zinnia_banner); // Say hello!
+    kprintf("Command line: %s\n", info->cmdline);
+
+    cmdline_parse(info->cmdline);
+    console_init();
+    kprintf("\ec");
+
+    mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
+
+#if CONFIG_ACPI
+    acpi_init(info);
+#endif
+
+    percpu_init();
+    sched_init(&percpu_get()->sched);
+
+    struct task* main_task;
+    task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
+    sched_add_task(&percpu_get()->sched, main_task);
+
+    irq_unlock();
+    sched_yield(&percpu_get()->sched);
+
+    unreachable();
+}

core/Kconfig

@@ -0,0 +1,18 @@
+menu "Kernel Core options"
+
+source "core/mem/Kconfig"
+
+choice SCHED_ALG
+  prompt "Scheduler algorithm"
+  default SCHED_DUMB
+  help
+    The algorithm to use for scheduling.
+
+config SCHED_DUMB
+  bool "Dumb Scheduler"
+  help
+    Very basic scheduler, without any priorities.
+
+endchoice
+
+endmenu

core/clock.c

@@ -0,0 +1,65 @@
+#include <zinnia/clock.h>
+#include <zinnia/compiler.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <stdint.h>
+
+static struct clock* active_clock = nullptr;
+static uint64_t counter_base = 0;
+static struct spinlock lock = {};
+
+static uint64_t clock_get_elapsed_ns_locked() {
+    if (__unlikely(!active_clock))
+        return 0;
+
+    return counter_base + active_clock->get_elapsed_ns(active_clock);
+}
+
+bool clock_available() {
+    spin_lock(&lock);
+    bool available = active_clock != nullptr;
+    spin_unlock(&lock);
+    return available;
+}
+
+uint64_t clock_get_elapsed_ns() {
+    spin_lock(&lock);
+    uint64_t elapsed = clock_get_elapsed_ns_locked();
+    spin_unlock(&lock);
+    return elapsed;
+}
+
+bool clock_switch(struct clock* clock) {
+    spin_lock(&lock);
+
+    if (active_clock) {
+        if (clock->priority <= active_clock->priority) {
+            spin_unlock(&lock);
+            return false;
+        }
+    }
+
+    uint64_t elapsed = clock_get_elapsed_ns_locked();
+    clock->reset(clock);
+
+    counter_base = elapsed;
+    active_clock = clock;
+
+    spin_unlock(&lock);
+
+    log("clock: Switching to clock source \"%s\"\n", clock->name);
+
+    return true;
+}
+
+void clock_spin_ns(uint64_t ns) {
+    if (!clock_available()) {
+        log("clock: Unable to sleep for %lu ns, no clock available!\n", ns);
+        return;
+    }
+
+    uint64_t target = clock_get_elapsed_ns() + ns;
+    while (clock_get_elapsed_ns() < target) {
+        // TODO: Maybe pause?
+    }
+}

core/console.c

@@ -1,8 +1,7 @@
-#include <kernel/cmdline.h>
-#include <kernel/console.h>
-#include <kernel/print.h>
-#include <kernel/syscalls.h>
-#include <string.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/console.h>
+#include <zinnia/print.h>
+#include <zinnia/string.h>
 
 extern struct console* const __ld_console_start[];
 extern struct console* const __ld_console_end[];
@@ -28,7 +27,7 @@ static void console_setup(const char* value) {
     }
 }
 
-CMDLINE_OPTION("console", console_setup);
+CMDLINE_OPTION("console", "Set a default console", console_setup);
 
 void console_init() {
     // If we have no console set and at least one set, use that as a default.
@@ -36,9 +35,11 @@ void console_init() {
         active = __ld_console_start[0];
 
     if (active) {
-        if (active->init(active) == false)
+        if (active->init(active) == false) {
             active = nullptr;
-        else
-            kprintf("Logging on console \"%s\"\n", active->name);
+            return;
+        }
+
+        log("console: Logging on console \"%s\"\n", active->name);
     }
 }

core/dpc.c

@@ -0,0 +1,3 @@
+#include <zinnia/dpc.h>
+
+// TODO

core/irq.c

@@ -0,0 +1,69 @@
+#include <zinnia/assert.h>
+#include <zinnia/irq.h>
+#include <zinnia/list.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <arch/irq.h>
+
+[[__percpu]]
+struct irq_percpu irq_data;
+
+bool irq_set_state(bool state) {
+    return arch_irq_set_state(state);
+}
+
+bool irq_get_state() {
+    return arch_irq_get_state();
+}
+
+void irq_wait() {
+    arch_irq_wait();
+}
+
+void irq_lock() {
+    bool prev = arch_irq_set_state(false);
+    struct irq_percpu* state = percpu_ref(irq_data);
+    if (state->depth++ == 0)
+        state->saved_state = prev;
+}
+
+void irq_unlock() {
+    struct irq_percpu* state = percpu_ref(irq_data);
+    if (--state->depth == 0 && state->saved_state)
+        arch_irq_set_state(true);
+}
+
+void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
+    irq_lock();
+    line->mode = line->set_config(line, trigger, polarity);
+    irq_unlock();
+}
+
+void irq_line_raise(struct irq_line* line) {
+    assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
+
+    irq_lock();
+    line->is_busy = true;
+
+    bool claimed = false;
+    struct irq_handler* handler;
+    list_foreach(handler, &line->handlers, next) {
+        if (handler->raise(handler) == IRQ_STATUS_HANDLED)
+            claimed = true;
+    }
+
+    if (!claimed)
+        log("irq: Spurious interrupt\n");
+
+    if ((line->mode & IRQ_MODE_EOI) && line->eoi)
+        line->eoi(line);
+
+    line->is_busy = false;
+    irq_unlock();
+}
+
+void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
+    irq_lock();
+    list_insert_head(&line->handlers, handler, next);
+    irq_unlock();
+}

core/lib/meson.build

@@ -1,5 +1,7 @@
 # Freestanding libc functions that are needed in the kernel
 
 kernel_sources += files(
+    'runtime.c',
+    'stdlib.c',
     'string.c',
 )

core/lib/runtime.c

@@ -0,0 +1,93 @@
+// Compiler runtime support.
+
+#include <stdint.h>
+
+uint64_t __udivdi3(uint64_t dividend, uint64_t divisor) {
+    if (divisor == 0)
+        return 0;
+
+    if (dividend < divisor)
+        return 0;
+
+    unsigned shift = __builtin_clzll(divisor) - __builtin_clzll(dividend);
+
+    divisor <<= shift;
+
+    uint64_t quotient = 0;
+
+    for (;;) {
+        if (dividend >= divisor) {
+            dividend -= divisor;
+            quotient |= (uint64_t)1 << shift;
+        }
+
+        if (shift == 0)
+            break;
+
+        shift--;
+        divisor >>= 1;
+    }
+
+    return quotient;
+}
+
+static unsigned clz128(__uint128_t value) {
+    uint64_t high = (uint64_t)(value >> 64);
+
+    if (high != 0)
+        return (unsigned)__builtin_clzll(high);
+
+    return 64 + (unsigned)__builtin_clzll((uint64_t)value);
+}
+
+__uint128_t __udivmodti4(__uint128_t dividend, __uint128_t divisor, __uint128_t* remainder) {
+    if (divisor == 0) {
+        if (remainder)
+            *remainder = dividend;
+
+        return 0;
+    }
+
+    if (dividend < divisor) {
+        if (remainder)
+            *remainder = dividend;
+
+        return 0;
+    }
+
+    unsigned shift = clz128(divisor) - clz128(dividend);
+
+    divisor <<= shift;
+
+    __uint128_t quotient = 0;
+
+    for (;;) {
+        if (dividend >= divisor) {
+            dividend -= divisor;
+            quotient |= (__uint128_t)1 << shift;
+        }
+
+        if (shift == 0)
+            break;
+
+        shift--;
+        divisor >>= 1;
+    }
+
+    if (remainder)
+        *remainder = dividend;
+
+    return quotient;
+}
+
+__uint128_t __udivti3(__uint128_t dividend, __uint128_t divisor) {
+    return __udivmodti4(dividend, divisor, nullptr);
+}
+
+__uint128_t __umodti3(__uint128_t dividend, __uint128_t divisor) {
+    __uint128_t remainder;
+
+    __udivmodti4(dividend, divisor, &remainder);
+
+    return remainder;
+}

core/lib/stdlib.c

@@ -0,0 +1,102 @@
+#include <stdint.h>
+#include <zinnia/stdlib.h>
+#include <zinnia/string.h>
+
+static void reverse(char* s) {
+    int i, j;
+    char c;
+
+    for (i = 0, j = strlen(s) - 1; i < j; i++, j--) {
+        c = s[i];
+        s[i] = s[j];
+        s[j] = c;
+    }
+}
+
+static int char_to_digit(char c) {
+    if (c >= '0' && c <= '9')
+        return c - '0';
+    if (c >= 'a' && c <= 'z')
+        return c - 'a' + 10;
+    if (c >= 'A' && c <= 'Z')
+        return c - 'A' + 10;
+    return -1;
+}
+
+unsigned long atolu(const char* str, int base) {
+    unsigned long result = 0;
+
+    if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
+        str += 2;
+    }
+
+    while (*str) {
+        int digit = char_to_digit(*str++);
+        if (digit < 0 || digit >= base)
+            break;
+
+        result = result * base + digit;
+    }
+
+    return result;
+}
+
+long atol(const char* str, int base) {
+    int neg = 0;
+
+    if (*str == '-') {
+        neg = 1;
+        str++;
+    }
+
+    unsigned long val = atolu(str, base);
+    return neg ? -(long)val : (long)val;
+}
+
+char* lutoa(uint64_t value, char* str, int base) {
+    if (base < 2 || base > 36)
+        return nullptr;
+
+    size_t i = 0;
+
+    do {
+        uint64_t digit = value % base;
+        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
+        value /= base;
+    } while (value);
+
+    str[i] = '\0';
+    reverse(str);
+    return str;
+}
+
+char* ltoa(int64_t value, char* str, int base) {
+    if (base < 2 || base > 36)
+        return nullptr;
+
+    uint64_t magnitude;
+    int negative = 0;
+
+    if (value < 0) {
+        negative = 1;
+        magnitude = (uint64_t)(-(value + 1)) + 1;
+        // avoids INT64_MIN overflow
+    } else {
+        magnitude = (uint64_t)value;
+    }
+
+    size_t i = 0;
+
+    do {
+        uint64_t digit = magnitude % base;
+        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
+        magnitude /= base;
+    } while (magnitude);
+
+    if (negative)
+        str[i++] = '-';
+
+    str[i] = '\0';
+    reverse(str);
+    return str;
+}

core/lib/string.c

@@ -1,8 +1,8 @@
-#include <common/compiler.h>
-#include <common/utils.h>
+#include <zinnia/compiler.h>
+#include <zinnia/utils.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <string.h>
+#include <zinnia/string.h>
 
 [[__hot]]
 size_t strlen(const char* str) {
@@ -24,15 +24,14 @@ size_t strnlen(const char* str, size_t len) {
 
 [[__hot]]
 int memcmp(const void* s1, const void* s2, size_t size) {
-    int diff = 0;
-    char* s1ptr = (char*)s1;
-    char* s2ptr = (char*)s2;
+    const uint8_t* s1ptr = s1;
+    const uint8_t* s2ptr = s2;
 
     for (size_t i = 0; i < size; i++) {
         if (s1ptr[i] != s2ptr[i])
-            diff++;
+            return s1ptr[i] - s2ptr[i];
     }
-    return diff;
+    return 0;
 }
 
 [[__hot]]
@@ -103,7 +102,7 @@ void* memset(void* dest, int value, size_t n) {
     }
 
     size_t* qword_dest = (size_t*)d;
-    size_t qword_value = value;
+    size_t qword_value = (uint8_t)value;
     qword_value |= (qword_value << 8);
     qword_value |= (qword_value << 16);
     qword_value |= (qword_value << 32);
@@ -126,8 +125,10 @@ void* memset(void* dest, int value, size_t n) {
 }
 
 char* strncpy(char* restrict dst, const char* restrict src, size_t len) {
-    size_t src_len = strnlen(src, len) + 1;
-    return memcpy(dst, src, MIN(len, src_len));
+    size_t src_len = strnlen(src, len);
+    memcpy(dst, src, src_len);
+    memset(dst + src_len, 0, len - src_len);
+    return dst;
 }
 
 int strncmp(const char* str1, const char* str2, size_t len) {

core/mem/Kconfig

@@ -0,0 +1,8 @@
+menu "Memory Management options"
+
+config HAVE_MMU
+  bool
+config HAVE_MMU_GENERIC
+  bool
+
+endmenu

core/mem/init.c

@@ -1,15 +1,11 @@
-// SPDX-License-Identifier: MIT
-
-#include <common/utils.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/init.h>
-#include <kernel/panic.h>
-#include <kernel/print.h>
-#include <kernel/vas.h>
-#include <kernel/virt.h>
-#include <assert.h>
-#include <string.h>
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/init.h>
+#include <zinnia/panic.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <zinnia/string.h>
 
 extern uint8_t __ld_text_start[];
 extern uint8_t __ld_text_end[];
@@ -19,7 +15,7 @@ extern uint8_t __ld_data_start[];
 extern uint8_t __ld_data_end[];
 extern uint8_t __ld_kernel_start[];
 
-struct vas kernel_vas = {0};
+struct vma kernel_vma = {0};
 
 [[__init]]
 void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) {
@@ -27,7 +23,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
 
     const size_t pgsz = arch_mem_page_size();
 
-    kprintf("Memory map:\n");
+    log("mem: Memory map:\n");
     for (size_t i = 0; i < map_len; i++) {
         if (map[i].length == 0)
             continue;
@@ -48,7 +44,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
             break;
         }
 
-        kprintf("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
+        log("mem: [%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
     }
 
     // Set up the bootstrap allocator.
@@ -58,80 +54,80 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
             largest = &map[i];
     }
     mem_phys_bootstrap(largest);
-    kprintf(
-        "Using region [%p - %p] for bootstrap allocator\n",
+    log("mem: Using region [%p - %p] for bootstrap allocator\n",
         (void*)largest->address,
-        (void*)(largest->address + largest->length - 1)
-    );
+        (void*)(largest->address + largest->length - 1));
 
     // Set the HHDM base address to the address given by the loader.
     // We must not keep any virtual addresses to this region,
     // since we're likely going to map it at a different base address.
     mem_hhdm_base = tmp_hhdm;
 
-    assert(pmap_new_kernel(&kernel_vas.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
+    assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
+    tailq_init(&kernel_vma.mappings);
 
     // text
-    kprintf("Mapping text segment at %p\n", __ld_text_start);
-    for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
-        zn_status_t status = pmap_map(
-            &kernel_vas.pmap,
+    log("mem: Mapping text segment at %p\n", __ld_text_start);
+    for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
+        errno_t status = pmap_map(
+            &kernel_vma.pmap,
             (uintptr_t)p,
             (phys_t)(p - __ld_kernel_start + kernel_phys),
             PTE_READ | PTE_EXEC,
-            CACHE_NONE
+            CACHE_ANY
         );
         assert(!status, "Failed to map %p with error %i\n", p, status);
     }
 
     // rodata
-    kprintf("Mapping rodata segment at %p\n", __ld_rodata_start);
+    log("mem: Mapping rodata segment at %p\n", __ld_rodata_start);
     for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
-        zn_status_t status = pmap_map(
-            &kernel_vas.pmap,
+        errno_t status = pmap_map(
+            &kernel_vma.pmap,
             (uintptr_t)p,
             (phys_t)(p - __ld_kernel_start + kernel_phys),
             PTE_READ,
-            CACHE_NONE
+            CACHE_ANY
         );
         assert(!status, "Failed to map %p with error %i\n", p, status);
     }
 
     // data
-    kprintf("Mapping data segment at %p\n", __ld_data_start);
+    log("mem: Mapping data segment at %p\n", __ld_data_start);
     for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
-        zn_status_t status = pmap_map(
-            &kernel_vas.pmap,
+        errno_t status = pmap_map(
+            &kernel_vma.pmap,
             (uintptr_t)p,
             (phys_t)(p - __ld_kernel_start + kernel_phys),
             PTE_READ | PTE_WRITE,
-            CACHE_NONE
+            CACHE_ANY
         );
         assert(!status, "Failed to map %p with error %i\n", p, status);
     }
 
-    // The kernel address space is divided into 3 segments (plus kernel).
-    // On x86_64 for example, they live at:
+    // The kernel address space is divided into 4 segments.
+    // For example, they could live at:
     // HHDM     FFFF'8000'0000'0000
     // PFNDB    FFFF'A000'0000'0000
     // Mappings FFFF'C000'0000'0000
+    // Kernel   FFFF'FFFF'8000'0000
 
     // Map all physical memory to the HHDM address.
     tmp_hhdm = arch_mem_hhdm_addr();
     for (size_t i = 0; i < map_len; i++) {
         if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
             continue;
-        for (size_t p = 0; p <= map[i].length; p += pgsz) {
+        for (size_t p = 0; p < map[i].length; p += pgsz) {
             uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
             phys_t paddr = (phys_t)(map[i].address + p);
-            zn_status_t status = pmap_map(&kernel_vas.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_NONE);
+            errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
             assert(!status, "Failed to map HHDM page %p to %p with error %i\n", (void*)vaddr, (void*)paddr, status);
         }
     }
     mem_hhdm_base = tmp_hhdm;
 
     // Switch to our own page table.
-    pmap_set(&kernel_vas.pmap);
+    pmap_set(&kernel_vma.pmap);
 
     // We record metadata for every single page of available memory in a large array.
     // This array is contiguous in virtual memory, but is sparsely populated.
@@ -154,7 +150,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
         for (size_t page = 0; page < length; page += pgsz) {
             phys_t paddr;
             assert(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!\n");
-            pmap_map(&kernel_vas.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_NONE);
+            pmap_map(&kernel_vma.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
         }
     }
     vm_pfndb = (struct page*)arch_mem_pfndb_addr();
@@ -165,7 +161,5 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
 
     slab_init();
 
-    // TODO: Free `__init` memory.
-
-    kprintf("Memory initialization complete\n");
+    log("mem: Memory initialization complete\n");
 }

core/mem/meson.build

@@ -0,0 +1,17 @@
+kernel_sources += files(
+    'fault.c',
+    'init.c',
+    'pmm.c',
+    'slab.c',
+    'stack.c',
+    'usercopy.c',
+    'virt.c',
+    'vma.c',
+    'vmo.c',
+)
+
+if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y'
+    kernel_sources += files(
+        'pmap.c',
+    )
+endif

core/mem/pmap.c

@@ -1,21 +1,32 @@
-// SPDX-License-Identifier: MIT
+#include <zinnia/alloc.h>
+#include <zinnia/errno.h>
+#include <zinnia/pmap.h>
+#include <zinnia/spin.h>
+#include <zinnia/string.h>
+#include <zinnia/vma.h>
+#include <config.h>
 
-#include <zinnia/status.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/pmap.h>
-#include <kernel/spin.h>
-#include <kernel/vas.h>
-#include <string.h>
+// TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
+static_assert(CONFIG_HAVE_MMU_GENERIC);
 
-zn_status_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
+errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
     flags &= ~ALLOC_NOZERO;
 
     phys_t addr;
-    zn_status_t status = mem_phys_alloc(1, flags, &addr);
+    errno_t status = mem_phys_alloc(1, flags, &addr);
     if (status)
         return status;
 
+    pte_t* root = HHDM_PTR(addr);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+    for (size_t i = num_entries / 2; i < num_entries; i++) {
+        phys_t dir;
+        status = mem_phys_alloc(1, flags, &dir);
+        if (status)
+            return status;
+        root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
+    }
+
     struct pmap result = {
         .root = addr,
         .lock = (struct spinlock){0},
@@ -23,17 +34,17 @@ zn_status_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
     };
 
     *pt = result;
-    return ZN_OK;
+    return 0;
 }
 
-zn_status_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
+errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
     phys_t user_l1;
-    zn_status_t status = mem_phys_alloc(1, 0, &user_l1);
+    errno_t status = mem_phys_alloc(1, 0, &user_l1);
     if (status)
         return status;
 
     void* user_l1_ptr = HHDM_PTR(user_l1);
-    void* kernel_l1_ptr = HHDM_PTR(kernel_vas.pmap.root);
+    void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
     memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
 
     struct pmap result = {
@@ -43,7 +54,7 @@ zn_status_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
     };
 
     *pt = result;
-    return ZN_OK;
+    return 0;
 }
 
 // Gets a reference to the PTE at the given virtual address.
@@ -70,7 +81,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
 
         if (pte_is_present(pte)) {
             // Get the next level.
-            *pte = pte_build(pte_address(pte), level_flags, CACHE_NONE);
+            *pte |= pte_build(0, level_flags, CACHE_ANY);
             current_head = HHDM_PTR(pte_address(pte));
         } else {
             // If the current level isn't present, we can skip the rest.
@@ -78,10 +89,10 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
                 return nullptr;
 
             phys_t addr;
-            if (mem_phys_alloc(1, 0, &addr) != ZN_OK)
+            if (mem_phys_alloc(1, 0, &addr) != 0)
                 return nullptr;
 
-            *pte = pte_build(addr, level_flags, CACHE_NONE);
+            *pte = pte_build(addr, level_flags, CACHE_ANY);
             current_head = HHDM_PTR(addr);
         }
     }
@@ -89,13 +100,13 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
     return &current_head[index];
 }
 
-zn_status_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
+errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
     spin_lock(&pt->lock);
 
-    zn_status_t status = ZN_OK;
+    errno_t status = 0;
     pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
     if (pte == nullptr) {
-        status = ZN_ERR_NO_MEMORY;
+        status = ENOMEM;
         goto fail;
     }
 
@@ -105,3 +116,21 @@ fail:
     spin_unlock(&pt->lock);
     return status;
 }
+
+errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte)) {
+        status = ENOENT;
+        goto fail;
+    }
+
+    pte_clear(pte);
+    pte_flush(vaddr);
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}

core/mem/pmm.c

@@ -1,35 +1,32 @@
-// SPDX-License-Identifier: MIT
-
-#include <common/compiler.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/panic.h>
-#include <kernel/print.h>
-#include <kernel/spin.h>
-#include <kernel/virt.h>
+#include <zinnia/alloc.h>
+#include <zinnia/compiler.h>
+#include <zinnia/panic.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/string.h>
+#include <zinnia/virt.h>
 #include <stdint.h>
-#include <string.h>
 
 uintptr_t mem_hhdm_base = 0;
 
-static zn_status_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
-static zn_status_t (*free_fn)(phys_t start, size_t num_pages);
+static errno_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
+static errno_t (*free_fn)(phys_t start, size_t num_pages);
 
-zn_status_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
+errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
     return alloc_fn(num_pages, flags, out);
 }
 
-zn_status_t mem_phys_free(phys_t start, size_t num_pages) {
+errno_t mem_phys_free(phys_t start, size_t num_pages) {
     return free_fn(start, num_pages);
 }
 
 static struct phys_mem bump_mem = {0};
 static struct phys_mem* bump_region = nullptr;
 
-static zn_status_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
+static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
     const size_t bytes = num_pages * arch_mem_page_size();
-    if (((intptr_t)bump_mem.length - bytes) <= 0)
-        return ZN_ERR_NO_MEMORY;
+    if (bytes > bump_mem.length)
+        return ENOMEM;
 
     *out = bump_mem.address;
     bump_mem.length -= bytes;
@@ -42,7 +39,7 @@ static zn_status_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
     return 0;
 }
 
-static zn_status_t bump_free(phys_t addr, size_t num_pages) {
+static errno_t bump_free(phys_t addr, size_t num_pages) {
     // We don't free bootstrap memory.
     panic("Attempted to free bootstrap memory!");
 }
@@ -62,7 +59,7 @@ static struct page* pmm_head = nullptr;
 static struct spinlock pmm_lock = {0};
 static size_t pmm_total_free = 0;
 
-static zn_status_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
+static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
     spin_lock(&pmm_lock);
 
     const size_t bytes = num_pages * arch_mem_page_size();
@@ -79,7 +76,7 @@ static zn_status_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys
     while (it) {
         const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
         // If the region is too high up or doesn't have enough pages left, continue searching.
-        if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
+        if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) {
             prev_it = it;
             it = it->freelist.next;
             continue;
@@ -87,11 +84,10 @@ static zn_status_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys
 
         // If the current block has exactly enough pages left, consume it entirely.
         if (__unlikely(it->freelist.count == num_pages)) {
-            if (prev_it) {
+            if (prev_it)
                 prev_it->freelist.next = it->freelist.next;
-            } else {
+            else
                 pmm_head = it->freelist.next;
-            }
 
             it->freelist.next = nullptr;
             it->freelist.count = 0;
@@ -100,17 +96,32 @@ static zn_status_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys
                 *out = page_addr;
         }
         // If not, we can just shrink the block.
-        else {
+        else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) {
             it->freelist.count -= num_pages;
 
             if (__likely(out))
                 *out = page_addr + (it->freelist.count * arch_mem_page_size());
+        } else {
+            struct page* rest = it + num_pages;
+            rest->freelist.count = it->freelist.count - num_pages;
+            rest->freelist.next = it->freelist.next;
+
+            if (prev_it)
+                prev_it->freelist.next = rest;
+            else
+                pmm_head = rest;
+
+            it->freelist.next = nullptr;
+            it->freelist.count = 0;
+
+            if (__likely(out))
+                *out = page_addr;
         }
         goto success;
     }
 
     spin_unlock(&pmm_lock);
-    return ZN_ERR_NO_MEMORY;
+    return ENOMEM;
 
 success:
     // If the NOZERO flag is *not* specified, zero memory.
@@ -122,13 +133,41 @@ success:
     return 0;
 }
 
-static zn_status_t freelist_free(phys_t addr, size_t num_pages) {
+static void freelist_insert(struct page* page, size_t num_pages) {
+    struct page* prev = nullptr;
+    struct page* next = pmm_head;
+    while (next && next < page) {
+        prev = next;
+        next = next->freelist.next;
+    }
+
+    page->freelist.count = num_pages;
+    page->freelist.next = next;
+
+    if (next && page + page->freelist.count == next) {
+        page->freelist.count += next->freelist.count;
+        page->freelist.next = next->freelist.next;
+        next->freelist.next = nullptr;
+        next->freelist.count = 0;
+    }
+
+    if (prev && prev + prev->freelist.count == page) {
+        prev->freelist.count += page->freelist.count;
+        prev->freelist.next = page->freelist.next;
+        page->freelist.next = nullptr;
+        page->freelist.count = 0;
+    } else if (prev) {
+        prev->freelist.next = page;
+    } else {
+        pmm_head = page;
+    }
+}
+
+static errno_t freelist_free(phys_t addr, size_t num_pages) {
     spin_lock(&pmm_lock);
 
-    struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
-    page->freelist.count = num_pages;
-    page->freelist.next = pmm_head;
-    pmm_head = page;
+    freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages);
+    pmm_total_free += num_pages;
 
     spin_unlock(&pmm_lock);
     return 0;
@@ -145,13 +184,11 @@ void mem_phys_init(struct phys_mem* map, size_t length) {
         if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
             continue;
 
-        struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
-        page->freelist.count = map[i].length / arch_mem_page_size();
-        page->freelist.next = pmm_head;
-        pmm_head = page;
-        pmm_total_free += page->freelist.count;
+        const size_t num_pages = map[i].length / arch_mem_page_size();
+        freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages);
+        pmm_total_free += num_pages;
     }
 
     const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
-    kprintf("Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
+    log("pmm: Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
 }

core/mem/slab.c

@@ -1,11 +1,8 @@
-// SPDX-License-Identifier: MIT
-
-#include <zinnia/status.h>
-#include <common/compiler.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/spin.h>
-#include <string.h>
+#include <zinnia/alloc.h>
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/spin.h>
+#include <zinnia/string.h>
 
 typedef struct {
     size_t num_pages; // Amount of pages connected to this slab.
@@ -26,12 +23,10 @@ typedef struct {
 
 static struct slab slabs[8] = {0};
 
-static zn_status_t slab_new(struct slab* slab, size_t size) {
-    slab->lock = (struct spinlock){0};
-
+static errno_t slab_new(struct slab* slab, size_t size) {
     // Allocate a new page for the head.
     phys_t addr;
-    zn_status_t status = mem_phys_alloc(1, 0, &addr);
+    errno_t status = mem_phys_alloc(1, 0, &addr);
     if (status)
         return status;
 
@@ -54,7 +49,7 @@ static zn_status_t slab_new(struct slab* slab, size_t size) {
     }
     arr[max * fact] = nullptr;
 
-    return ZN_OK;
+    return 0;
 }
 
 void slab_init(void) {
@@ -71,8 +66,10 @@ void slab_init(void) {
 static void* slab_do_alloc(struct slab* slab) {
     spin_lock(&slab->lock);
 
-    if (__unlikely(slab->head == nullptr))
-        slab_new(slab, slab->ent_size);
+    if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) {
+        spin_unlock(&slab->lock);
+        return nullptr;
+    }
 
     void** old_free = slab->head;
     slab->head = *old_free;
@@ -117,8 +114,8 @@ void* mem_alloc(size_t size, enum alloc_flags flags) {
 
     // Allocate the pages plus an additional page for metadata.
     phys_t ret;
-    zn_status_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
-    if (__unlikely(status != ZN_OK))
+    errno_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
+    if (__unlikely(status != 0))
         return nullptr;
 
     ret = ret + (phys_t)arch_mem_hhdm_addr();

core/mem/stack.c

@@ -0,0 +1,60 @@
+#include <zinnia/alloc.h>
+#include <zinnia/errno.h>
+#include <zinnia/pmap.h>
+#include <zinnia/stack.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+
+errno_t stack_init(struct stack* ks, size_t size) {
+    if (!ks || size == 0)
+        return EINVAL;
+
+    const size_t page_size = arch_mem_page_size();
+    size = ALIGN_UP(size, page_size);
+
+    uintptr_t va = vm_alloc(size + page_size);
+    if (!va)
+        return ENOMEM;
+
+    phys_t phys;
+    errno_t status = mem_phys_alloc(size / page_size, 0, &phys);
+    if (status)
+        goto err_vm;
+
+    const uintptr_t base = va + page_size;
+    size_t mapped = 0;
+    for (; mapped < size; mapped += page_size) {
+        status = pmap_map(&kernel_vma.pmap, base + mapped, phys + mapped, PTE_READ | PTE_WRITE, CACHE_WRITE_BACK);
+        if (status)
+            goto err_map;
+    }
+
+    ks->base = base;
+    ks->phys = phys;
+    ks->size = size;
+    return 0;
+
+err_map:
+    for (size_t i = 0; i < mapped; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, base + i);
+    mem_phys_free(phys, size / page_size);
+err_vm:
+    vm_free(va, size + page_size);
+    return status;
+}
+
+void stack_destroy(struct stack* ks) {
+    if (!ks || !ks->base)
+        return;
+
+    const size_t page_size = arch_mem_page_size();
+    for (size_t i = 0; i < ks->size; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, ks->base + i);
+
+    mem_phys_free(ks->phys, ks->size / page_size);
+    vm_free(ks->base - page_size, ks->size + page_size);
+
+    *ks = (struct stack){0};
+}

core/mem/usercopy.c

@@ -0,0 +1,44 @@
+#include <zinnia/percpu.h>
+#include <zinnia/usercopy.h>
+#include <arch/mem.h>
+#include <arch/usercopy.h>
+
+static bool is_user_range(uintptr_t addr, size_t len) {
+    if (len == 0)
+        return arch_mem_addr_is_user(addr);
+    return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1);
+}
+
+bool usercopy_read(void* dst, const __user void* src, size_t len) {
+    if (!is_user_range((uintptr_t)src, len))
+        return false;
+
+    arch_usercopy_allow(true);
+    bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
+    arch_usercopy_allow(false);
+    return success;
+}
+
+bool usercopy_write(__user void* dst, const void* src, size_t len) {
+    if (!is_user_range((uintptr_t)dst, len))
+        return false;
+
+    arch_usercopy_allow(true);
+    bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
+    arch_usercopy_allow(false);
+    return success;
+}
+
+bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
+    if (!arch_mem_addr_is_user((uintptr_t)str))
+        return false;
+
+    size_t unused;
+    if (len == nullptr)
+        len = &unused;
+
+    arch_usercopy_allow(true);
+    bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
+    arch_usercopy_allow(false);
+    return success;
+}

core/mem/virt.c

@@ -1,9 +1,6 @@
-// SPDX-License-Identifier: MIT
-
-#include <common/utils.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/virt.h>
+#include <zinnia/alloc.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
 #include <stdatomic.h>
 #include <stddef.h>
 
@@ -11,13 +8,14 @@ struct page* vm_pfndb = nullptr;
 
 // TODO: Replace with actual allocator.
 
-static size_t mapping_offset = 0;
+static _Atomic(size_t) mapping_offset = 0;
 
 uintptr_t vm_alloc(size_t length) {
     uintptr_t base = arch_mem_mapping_addr();
-    size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()));
+    size_t offset =
+        atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire);
 
-    return (base + offset);
+    return base + offset;
 }
 
 void vm_free(uintptr_t ptr, size_t length) {
@@ -27,7 +25,3 @@ void vm_free(uintptr_t ptr, size_t length) {
 uintptr_t vm_page_addr(struct page* page) {
     return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
 }
-
-size_t mem_high_shift() {
-    return arch_mem_level_bits() * arch_mem_num_levels() + arch_mem_page_bits();
-}

core/mem/vma.c

@@ -0,0 +1,158 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/errno.h>
+#include <zinnia/mman.h>
+#include <zinnia/pmap.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stdckdint.h>
+
+// Try to create a new, empty virtual address space.
+errno_t vma_new(struct vma** out) {
+    debug_assert(out != nullptr, "Missing out");
+
+    errno_t res;
+    if (!out)
+        return EINVAL;
+
+    struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
+    if (!vmspace)
+        return ENOMEM;
+
+    res = pmap_new_user(&vmspace->pmap, 0);
+    if (res)
+        goto err1;
+
+    tailq_init(&vmspace->mappings);
+
+    *out = vmspace;
+    return 0;
+
+err1:
+    mem_free(vmspace);
+err0:
+    return res;
+}
+
+// Delete a virtual address space.
+void vma_free(struct vma* v) {
+    struct vma_mapping* mapping;
+    while ((mapping = tailq_first(&v->mappings)) != nullptr) {
+        tailq_remove(&v->mappings, mapping, link);
+        vmo_unref(mapping->object);
+        mem_free(mapping);
+    }
+}
+
+errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
+    panic("TODO");
+}
+
+errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
+    panic("TODO");
+}
+
+static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
+    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (page >= m->start_page && page < m->end_page)
+            return m;
+    }
+    return nullptr;
+}
+
+static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) {
+    if (info->caused_by_write)
+        return prot & PROT_WRITE;
+    if (info->caused_by_fetch)
+        return prot & PROT_EXEC;
+    return prot & (PROT_READ | PROT_WRITE);
+}
+
+static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
+    enum pte_flags pf = 0;
+    if (prot & PROT_READ)
+        pf |= PTE_READ;
+    if (prot & PROT_WRITE)
+        pf |= PTE_WRITE;
+    if (prot & PROT_EXEC)
+        pf |= PTE_EXEC;
+    if (v->pmap.is_user)
+        pf |= PTE_USER;
+    return pf;
+}
+
+bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t faulty_page = info->addr / page_size;
+
+    spin_lock(&v->lock);
+    struct vma_mapping* mapping = find_mapping(v, faulty_page);
+    if (!mapping || !can_access(mapping->prot, info)) {
+        spin_unlock(&v->lock);
+        return false;
+    }
+
+    struct vmo* object = mapping->object;
+    const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
+    const enum prot_flags prot = mapping->prot;
+    vmo_ref(object);
+    spin_unlock(&v->lock);
+
+    struct page* page;
+    if (object->get_page(object, object_page, &page)) {
+        vmo_unref(object);
+        return false;
+    }
+
+    spin_lock(&v->lock);
+    mapping = find_mapping(v, faulty_page);
+    const bool still_valid = mapping && mapping->object == object &&
+        (faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info);
+
+    if (still_valid) {
+        errno_t status =
+            pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
+        assert(status == 0, "Failed to map a demand-loaded page\n");
+    }
+    spin_unlock(&v->lock);
+
+    vmo_unref(object);
+    return true;
+}
+
+errno_t vma_map(
+    struct vma* vma,
+    struct vmo* vmo,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags flags,
+    uintptr_t vmo_offset
+) {
+    if (!vma || !vmo)
+        return EINVAL;
+
+    uintptr_t end;
+    if (ckd_add(&end, addr, len))
+        return ENOMEM;
+
+    const size_t page_size = arch_mem_page_size();
+    if (addr % page_size != vmo_offset % page_size)
+        return EINVAL;
+
+    struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
+    if (!mapping)
+        return ENOMEM;
+
+    vmo_ref(vmo);
+    mapping->start_page = addr / page_size;
+    mapping->end_page = ALIGN_UP(end, page_size) / page_size;
+    mapping->offset_page = vmo_offset / page_size;
+    mapping->object = vmo;
+    mapping->prot = flags;
+
+    spin_lock(&vma->lock);
+    tailq_insert_head(&vma->mappings, mapping, link);
+    spin_unlock(&vma->lock);
+
+    return 0;
+}

core/mem/vmo.c

@@ -1,18 +1,23 @@
-// SPDX-License-Identifier: MIT
+#include <zinnia/alloc.h>
+#include <zinnia/errno.h>
+#include <zinnia/list.h>
+#include <zinnia/string.h>
+#include <zinnia/utils.h>
+#include <zinnia/vmo.h>
 
-#include <zinnia/status.h>
-#include <common/utils.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/list.h>
-#include <kernel/virt.h>
-#include <kernel/vmo.h>
-#include <string.h>
+void vmo_ref(struct vmo* obj) {
+    refcount_inc(&obj->rc);
+}
 
-zn_status_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
+void vmo_unref(struct vmo* obj) {
+    if (refcount_dec(&obj->rc))
+        obj->destroy(obj);
+}
+
+errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
     const size_t page_size = arch_mem_page_size();
     size_t progress = 0;
-    zn_status_t status = ZN_OK;
+    errno_t status = 0;
 
     while (progress < len) {
         const size_t misalign = (progress + offset) % page_size;
@@ -21,7 +26,7 @@ zn_status_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, s
 
         struct page* p;
         status = obj->get_page(obj, page_index, &p);
-        if (status != ZN_OK)
+        if (status != 0)
             break;
 
         uintptr_t page_addr = vm_page_addr(p);
@@ -35,10 +40,10 @@ zn_status_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, s
     return status;
 }
 
-zn_status_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
+errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
     const size_t page_size = arch_mem_page_size();
     size_t progress = 0;
-    zn_status_t status = ZN_OK;
+    errno_t status = 0;
 
     while (progress < len) {
         const size_t misalign = (progress + offset) % page_size;
@@ -47,7 +52,7 @@ zn_status_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t
 
         struct page* p;
         status = obj->get_page(obj, page_index, &p);
-        if (status != ZN_OK)
+        if (status != 0)
             break;
 
         uintptr_t page_addr = vm_page_addr(p);
@@ -61,7 +66,7 @@ zn_status_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t
     return status;
 }
 
-zn_status_t vmo_copy(
+errno_t vmo_copy(
     struct vmo* target,
     uintptr_t target_offset,
     struct vmo* src,
@@ -71,7 +76,7 @@ zn_status_t vmo_copy(
 ) {
     const size_t page_size = arch_mem_page_size();
     size_t progress = 0;
-    zn_status_t status = ZN_OK;
+    errno_t status = 0;
 
     while (progress < len) {
         const size_t target_misalign = (progress + target_offset) % page_size;
@@ -83,12 +88,12 @@ zn_status_t vmo_copy(
 
         struct page* tpage;
         status = target->get_page(target, target_page_index, &tpage);
-        if (status != ZN_OK)
+        if (status != 0)
             break;
 
         struct page* spage;
         status = src->get_page(src, src_page_index, &spage);
-        if (status != ZN_OK)
+        if (status != 0)
             break;
 
         uintptr_t taddr = vm_page_addr(tpage);
@@ -103,65 +108,98 @@ zn_status_t vmo_copy(
     return status;
 }
 
-static zn_status_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
+static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
     struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
 
     // Try to find the requested page in the cache.
     struct page_list* iter;
+    spin_lock(&paged->lock);
     list_foreach(iter, &paged->cache, next) {
         if (iter->offset != offset_idx)
             continue;
 
         // We found a cached page!
         *out = iter->value;
-        return ZN_OK;
+        spin_unlock(&paged->lock);
+        return 0;
     }
+    spin_unlock(&paged->lock);
 
     // Get page from out pager if we don't have one already.
     struct page* new_page;
-    zn_status_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
+    errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
     if (status)
         return status;
 
     struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
-    if (!entry)
-        return ZN_ERR_NO_MEMORY;
+    if (!entry) {
+        paged->source.put_page(&paged->source, offset_idx, new_page);
+        return ENOMEM;
+    }
 
     entry->offset = offset_idx;
     entry->value = new_page;
 
+    spin_lock(&paged->lock);
+    list_foreach(iter, &paged->cache, next) {
+        if (iter->offset != offset_idx)
+            continue;
+
+        *out = iter->value;
+        spin_unlock(&paged->lock);
+        mem_free(entry);
+        paged->source.put_page(&paged->source, offset_idx, new_page);
+        return 0;
+    }
+
     // Add page to the object cache.
     list_insert_head(&paged->cache, entry, next);
+    spin_unlock(&paged->lock);
 
     *out = new_page;
-    return ZN_OK;
+    return 0;
 }
 
-zn_status_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
+static void paged_destroy(struct vmo* vmo) {
+    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
+
+    struct page_list* iter = list_first(&paged->cache);
+    while (iter) {
+        struct page_list* next = list_next(iter, next);
+        paged->source.put_page(&paged->source, iter->offset, iter->value);
+        mem_free(iter);
+        iter = next;
+    }
+
+    mem_free(paged);
+}
+
+errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
     struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
     if (!result)
-        return ZN_ERR_NO_MEMORY;
+        return ENOMEM;
 
+    refcount_init(&result->object.rc);
     result->source = pager;
     result->object.get_page = paged_get_page;
+    result->object.destroy = paged_destroy;
 
     *out = result;
-    return ZN_OK;
+    return 0;
 }
 
-static zn_status_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
+static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
     phys_t addr;
-    zn_status_t status = mem_phys_alloc(1, 0, &addr);
-    if (status != ZN_OK)
+    errno_t status = mem_phys_alloc(1, 0, &addr);
+    if (status != 0)
         return status;
     size_t idx = addr / arch_mem_page_size();
     *out = &vm_pfndb[idx];
-    return ZN_OK;
+    return 0;
 }
 
-static zn_status_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
-    // We do nothing here.
-    return ZN_OK;
+static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
+    return mem_phys_free(vm_page_addr(page), 1);
 }
 
 static const struct pager_ops phys_pager = {
@@ -169,31 +207,58 @@ static const struct pager_ops phys_pager = {
     .put_page = phys_put_page,
 };
 
-zn_status_t vmo_new_phys(struct paged_vmo** out) {
+errno_t vmo_new_phys(struct paged_vmo** out) {
     return vmo_new_paged(phys_pager, out);
 }
 
+void vmo_truncate_paged(struct paged_vmo* paged, size_t length) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t keep_pages = ROUND_UP(length, page_size) / page_size;
+
+    spin_lock(&paged->lock);
+    struct page_list** pp = &list_first(&paged->cache);
+    while (*pp) {
+        struct page_list* entry = *pp;
+        if (entry->offset >= keep_pages) {
+            *pp = list_next(entry, next);
+            paged->source.put_page(&paged->source, entry->offset, entry->value);
+            mem_free(entry);
+        } else {
+            pp = &list_next(entry, next);
+        }
+    }
+    spin_unlock(&paged->lock);
+}
+
 struct phys_range_vmo {
     struct vmo object;
     uintptr_t start;
     size_t len;
 };
 
-static zn_status_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
+static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
     struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
+    if (offset_idx >= p->len / arch_mem_page_size())
+        return EINVAL;
     *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
-    return ZN_OK;
+    return 0;
 }
 
-zn_status_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
+static void phys_range_destroy(struct vmo* vmo) {
+    mem_free(containerof(vmo, struct phys_range_vmo, object));
+}
+
+errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
     struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
     if (!vmo)
-        return ZN_ERR_NO_MEMORY;
+        return ENOMEM;
 
+    refcount_init(&vmo->object.rc);
     vmo->object.get_page = phys_range_get_page;
+    vmo->object.destroy = phys_range_destroy;
     vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
     vmo->len = ALIGN_UP(length, arch_mem_page_size());
 
     *out = &vmo->object;
-    return ZN_OK;
+    return 0;
 }

core/meson.build

@@ -1,16 +1,15 @@
 kernel_sources += files(
-    'channel.c',
     'clock.c',
     'console.c',
-    'elf.c',
-    'futex.c',
     'irq.c',
-    'namespace.c',
+    'module.c',
     'panic.c',
     'percpu.c',
     'print.c',
-    'sched.c',
     'spin.c',
-    'uacpi_glue.c',
-    'vdso.c',
+    'wait.c',
 )
+
+subdir('lib')
+subdir('mem')
+subdir('sched')

core/module.c

@@ -0,0 +1,12 @@
+#include <zinnia/assert.h>
+#include <zinnia/module.h>
+
+extern errno_t (*__ld_mod_entry_start[])(struct module*);
+extern errno_t (*__ld_mod_entry_end[])(struct module*);
+
+void module_init() {
+    for (auto start = __ld_mod_entry_start; start < __ld_mod_entry_end; start += 1) {
+        errno_t mod_status = (*start)(nullptr);
+        assert(mod_status == EOK, "Built-in entry point returned %i", mod_status);
+    }
+}

core/panic.c

@@ -0,0 +1,36 @@
+#include <zinnia/irq.h>
+#include <zinnia/panic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <stdarg.h>
+
+#define panic_log(msg, ...) kprintf("[    !!!!    ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
+
+[[noreturn]]
+void panic(const char* msg, ...) {
+    // TODO: Stop all other CPUs with IPI.
+    irq_set_state(false);
+
+    struct percpu* cpu = percpu_get();
+    struct task* current = cpu->sched.current;
+    kprintf(
+        "[    !!!!    ] \e[31m"
+        "Kernel panic on CPU %zu (T%zu, \"%s\"): ",
+        cpu->id,
+        current ? current->id : 0,
+        current ? current->name : nullptr
+    );
+
+    va_list args;
+    va_start(args, msg);
+    kvprintf(msg, args);
+    va_end(args);
+
+    kprintf("\e[0m");
+
+    panic_log("End of panic message\n");
+
+    while (1) {
+        irq_wait();
+    }
+}

core/percpu.c

@@ -0,0 +1,60 @@
+#include <zinnia/alloc.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
+#include <zinnia/string.h>
+#include <zinnia/utils.h>
+#include <arch/percpu.h>
+#include <stdatomic.h>
+
+extern uint8_t __ld_percpu_start[];
+extern uint8_t __ld_percpu_end[];
+
+static _Atomic(size_t) num_cpus = 1;
+
+static size_t max_cpus = SIZE_MAX;
+static void smp_option(uintptr_t value) {
+    max_cpus = value;
+}
+CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option);
+
+[[__used, __section(".percpu.init")]]
+struct percpu percpu_bsp = {
+    .self = &percpu_bsp,
+    .id = 0,
+    .online = false,
+};
+
+struct percpu* percpu_new() {
+    const size_t size = __ld_percpu_end - __ld_percpu_start;
+    const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
+
+    // TODO: Map pages past the end of the kernel.
+    // TODO: Document how allocations work.
+
+    phys_t mem;
+    if (mem_phys_alloc(num_pages, 0, &mem))
+        return nullptr;
+
+    struct percpu* cpu = HHDM_PTR(mem);
+    cpu->self = cpu;
+    cpu->id = atomic_fetch_add_explicit(&num_cpus, 1, memory_order_relaxed);
+    cpu->online = false;
+    return cpu;
+}
+
+size_t percpu_max_cpus() {
+    return max_cpus;
+}
+
+struct percpu* percpu_get() {
+    return arch_percpu_get();
+}
+
+void percpu_bsp_init() {
+    arch_percpu_bsp_init();
+}
+
+void percpu_init() {
+    arch_percpu_init();
+}

core/print.c

@@ -1,113 +1,21 @@
-#include <kernel/clock.h>
-#include <kernel/console.h>
-#include <kernel/panic.h>
-#include <kernel/print.h>
+#include <zinnia/clock.h>
+#include <zinnia/console.h>
+#include <zinnia/panic.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/stdlib.h>
+#include <zinnia/string.h>
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <string.h>
-
-static void reverse(char* s) {
-    int i, j;
-    char c;
-
-    for (i = 0, j = strlen(s) - 1; i < j; i++, j--) {
-        c = s[i];
-        s[i] = s[j];
-        s[j] = c;
-    }
-}
-
-static int char_to_digit(char c) {
-    if (c >= '0' && c <= '9')
-        return c - '0';
-    if (c >= 'a' && c <= 'z')
-        return c - 'a' + 10;
-    if (c >= 'A' && c <= 'Z')
-        return c - 'A' + 10;
-    return -1;
-}
-
-unsigned long atolu(const char* str, int base) {
-    unsigned long result = 0;
-
-    if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
-        str += 2;
-    }
-
-    while (*str) {
-        int digit = char_to_digit(*str++);
-        if (digit < 0 || digit >= base)
-            break;
-
-        result = result * base + digit;
-    }
-
-    return result;
-}
-
-long atol(const char* str, int base) {
-    int neg = 0;
-
-    if (*str == '-') {
-        neg = 1;
-        str++;
-    }
-
-    unsigned long val = atolu(str, base);
-    return neg ? -(long)val : (long)val;
-}
-
-char* lutoa(uint64_t value, char* str, int base) {
-    if (base < 2 || base > 36)
-        return nullptr;
-
-    size_t i = 0;
-
-    do {
-        uint64_t digit = value % base;
-        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
-        value /= base;
-    } while (value);
-
-    str[i] = '\0';
-    reverse(str);
-    return str;
-}
-
-char* ltoa(int64_t value, char* str, int base) {
-    if (base < 2 || base > 36)
-        return nullptr;
-
-    uint64_t magnitude;
-    int negative = 0;
-
-    if (value < 0) {
-        negative = 1;
-        magnitude = (uint64_t)(-(value + 1)) + 1;
-        // avoids INT64_MIN overflow
-    } else {
-        magnitude = (uint64_t)value;
-    }
-
-    size_t i = 0;
-
-    do {
-        uint64_t digit = magnitude % base;
-        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
-        magnitude /= base;
-    } while (magnitude);
-
-    if (negative)
-        str[i++] = '-';
-
-    str[i] = '\0';
-    reverse(str);
-    return str;
-}
 
 typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
 
+static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) {
+    for (size_t i = 0; i < count; i++)
+        callback(ctx, &c, 1);
+}
+
 static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
     // Amount of bytes written.
     int32_t written = 0;
@@ -135,7 +43,7 @@ static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_
         bool blank_sign = false;
         bool write_prefix = false;
         bool pad_zero = false;
-        bool has_width = false;
+        bool left_align = false;
         size_t width = 0;
         bool has_precision = false;
         size_t precision = 0;
@@ -166,6 +74,11 @@ check_fmt:
             fmt++;
             goto check_fmt;
         }
+        case '-': {
+            left_align = true;
+            fmt++;
+            goto check_fmt;
+        }
         // Width
         case '1':
         case '2':
@@ -176,14 +89,11 @@ check_fmt:
         case '7':
         case '8':
         case '9': {
-            has_width = true;
-            char number[10 + 1] = {0};
-            size_t idx = 0;
+            width = 0;
             while (*fmt >= '0' && *fmt <= '9') {
-                number[idx++] = *fmt;
+                width = width * 10 + (*fmt - '0');
                 fmt++;
             }
-            width = atolu(number, 10);
 
             goto check_fmt;
         }
@@ -191,16 +101,19 @@ check_fmt:
         case '.': {
             has_precision = true;
             fmt++;
+            precision = 0;
             if (*fmt == '*') {
-                precision = (char)va_arg(args, int32_t);
+                const int32_t arg = va_arg(args, int32_t);
+                if (arg < 0)
+                    has_precision = false;
+                else
+                    precision = arg;
+                fmt++;
             } else {
-                char number[10 + 1] = {0};
-                size_t idx = 0;
                 while (*fmt >= '0' && *fmt <= '9') {
-                    number[idx++] = *fmt;
+                    precision = precision * 10 + (*fmt - '0');
                     fmt++;
                 }
-                precision = atolu(number, 10);
             }
             goto check_fmt;
         }
@@ -231,10 +144,15 @@ check_fmt:
         // Character
         case 'c': {
             char c = (char)va_arg(args, int32_t);
+            const size_t pad = width > 1 ? width - 1 : 0;
 
+            if (!left_align)
+                print_pad(callback, ctx, ' ', pad);
             callback(ctx, &c, sizeof(c));
+            if (left_align)
+                print_pad(callback, ctx, ' ', pad);
 
-            written++;
+            written += 1 + pad;
             fmt++;
             break;
         }
@@ -248,10 +166,15 @@ check_fmt:
                 len = strnlen(str, precision);
             else
                 len = strlen(str);
+            const size_t pad = width > len ? width - len : 0;
 
+            if (!left_align)
+                print_pad(callback, ctx, ' ', pad);
             callback(ctx, str, len);
+            if (left_align)
+                print_pad(callback, ctx, ' ', pad);
 
-            written += len;
+            written += len + pad;
             fmt++;
             break;
         }
@@ -310,36 +233,38 @@ check_fmt:
                 break;
             }
 print_num:
-            size_t len = strlen(number);
-            if (has_precision) {
-                for (size_t i = 0; i < precision - len; i++)
-                    callback(ctx, " ", 1);
+            const char* digits = number;
+            char sign = 0;
+            if (digits[0] == '-') {
+                sign = '-';
+                digits++;
+            } else if (force_sign) {
+                sign = '+';
+            } else if (blank_sign) {
+                sign = ' ';
             }
-            if (has_width) {
-                char c = pad_zero ? '0' : ' ';
 
-                // We don't have to pad anything.
-                if (len < width) {
-                    for (size_t i = 0; i < width - len; i++)
-                        callback(ctx, &c, 1);
-                }
+            const size_t len = strlen(digits);
+            size_t zeros = (has_precision && precision > len) ? precision - len : 0;
+            const size_t total = (sign ? 1 : 0) + (write_prefix ? 2 : 0) + zeros + len;
+            size_t pad = width > total ? width - total : 0;
+            if (pad_zero && !left_align && !has_precision) {
+                zeros += pad;
+                pad = 0;
             }
-            if (write_prefix) {
+
+            if (!left_align)
+                print_pad(callback, ctx, ' ', pad);
+            if (sign)
+                callback(ctx, &sign, 1);
+            if (write_prefix)
                 callback(ctx, "0x", 2);
-            }
-            if (force_sign) {
-                char c = '+';
-                if (number[0] != '-')
-                    callback(ctx, &c, 1);
-            }
-            if (blank_sign) {
-                char c = ' ';
-                if (number[0] != '-')
-                    callback(ctx, &c, 1);
-            }
-            callback(ctx, number, len);
+            print_pad(callback, ctx, '0', zeros);
+            callback(ctx, digits, len);
+            if (left_align)
+                print_pad(callback, ctx, ' ', pad);
 
-            written += len;
+            written += width > total ? width : total;
             fmt++;
             break;
         }
@@ -370,13 +295,14 @@ print_num:
         }
         // No format, just normal text.
         default: {
-            fmt = format_begun_at;
-            size_t len = strlen(fmt);
+            size_t len = fmt - format_begun_at;
+            if (*fmt)
+                len++;
 
-            callback(ctx, fmt, len);
+            callback(ctx, format_begun_at, len);
 
             written += len;
-            fmt += len;
+            fmt = format_begun_at + len;
             break;
         }
         }
@@ -387,13 +313,23 @@ static void printf_console(void*, const char* msg, size_t len) {
     console_write(msg, len);
 }
 
+static struct spinlock print_lock = {};
+
 void kprintf(const char* message, ...) {
+    spin_lock(&print_lock);
+
     va_list args;
     va_start(args, message);
     do_print(printf_console, nullptr, message, args);
     va_end(args);
+
+    spin_unlock(&print_lock);
 }
 
 void kvprintf(const char* message, va_list args) {
+    spin_lock(&print_lock);
+
     do_print(printf_console, nullptr, message, args);
+
+    spin_unlock(&print_lock);
 }

core/sched/core.c

@@ -0,0 +1 @@
+#include <zinnia/sched.h>

core/sched/meson.build

@@ -0,0 +1,7 @@
+kernel_sources += files(
+    'core.c',
+)
+
+if kconfig.get('CONFIG_SCHED_DUMB', 'n') == 'y'
+    kernel_sources += files('sched_dumb.c')
+endif

core/sched/sched_dumb.c

@@ -0,0 +1,167 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/errno.h>
+#include <zinnia/irq.h>
+#include <zinnia/list.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/spin.h>
+#include <zinnia/stack.h>
+#include <zinnia/string.h>
+#include <zinnia/tailq.h>
+#include <zinnia/vma.h>
+#include <arch/sched.h>
+#include <stdatomic.h>
+#include <stdint.h>
+
+static void idle_func(uintptr_t) {
+    while (1) {
+        irq_set_state(true);
+        irq_wait();
+    }
+}
+
+void sched_to_user(uintptr_t info_ptr) {
+    struct task_start_info* info = (struct task_start_info*)info_ptr;
+    uintptr_t ip = info->ip;
+    uintptr_t sp = info->sp;
+    uintptr_t arg = info->arg;
+    mem_free(info);
+    arch_sched_jump_to_user(ip, sp, arg);
+}
+
+void sched_to_user_context(uintptr_t ctx) {
+    arch_sched_jump_to_context((struct arch_context*)ctx);
+}
+
+void sched_init(struct scheduler* sched) {
+    struct task* idle_task;
+    assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
+    sched->idle_task = idle_task;
+
+    struct task* boot_task = mem_alloc(sizeof(struct task), 0);
+    assert(boot_task != nullptr, "Unable to create boot task!\n");
+    strncpy(boot_task->name, "boot", sizeof(boot_task->name));
+    boot_task->space = &kernel_vma;
+    sched->current = boot_task;
+
+    tailq_init(&sched->run_queue);
+
+    log("sched: Scheduler initialized\n");
+}
+
+void sched_add_task(struct scheduler* sched, struct task* task) {
+    assert(task != nullptr, "Tried to add NULL to the run queue\n");
+    tailq_insert_tail(&sched->run_queue, task, next);
+}
+
+static struct task* next_task(struct scheduler* sched) {
+    struct task* t = tailq_first(&sched->run_queue);
+
+    if (t == nullptr)
+        return sched->idle_task;
+
+    tailq_remove(&sched->run_queue, t, next);
+
+    return t;
+}
+
+static void finish_switch(struct task* prev) {
+    if (prev->state == TASK_STATE_DEAD)
+        task_destroy(prev);
+}
+
+static void do_reschedule(struct scheduler* sched) {
+    struct task* from = sched->current;
+    struct task* to = next_task(sched);
+
+    // We'd end up on the same task, skip switching altogether.
+    if (from == to) {
+        irq_unlock();
+        return;
+    }
+
+    sched->current = to;
+    pmap_set(&to->space->pmap);
+
+    struct percpu* cpu = percpu_get();
+    from->user_stack = cpu->user_stack;
+    cpu->kernel_stack = kernel_stack_top(&to->kernel_stack);
+    cpu->user_stack = to->user_stack;
+
+    struct irq_percpu irq_state = *percpu_ref(irq_data);
+    struct task* prev = arch_sched_switch(from, to);
+    *percpu_ref(irq_data) = irq_state;
+    finish_switch(prev);
+    irq_unlock();
+}
+
+void sched_reschedule(struct scheduler* sched) {
+    irq_lock();
+    if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD)
+        sched_add_task(sched, sched->current);
+    do_reschedule(sched);
+}
+
+void sched_yield(struct scheduler* sched) {
+    irq_lock();
+    do_reschedule(sched);
+}
+
+errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
+    if (!out)
+        return EINVAL;
+
+    struct task* new_task = mem_alloc(sizeof(struct task), 0);
+    if (new_task == nullptr)
+        return ENOMEM;
+
+    static _Atomic(size_t) last_tid = 0;
+    new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire);
+    new_task->space = space;
+    strncpy(new_task->name, name, sizeof(new_task->name));
+
+    // Allocate a kernel stack.
+    errno_t s = stack_init(&new_task->kernel_stack, KERNEL_STACK_SIZE);
+    if (s != 0) {
+        mem_free(new_task);
+        return s;
+    }
+    new_task->user_stack = 0;
+
+    // Initialize the arch-dependent parts.
+    s = arch_task_init(&new_task->context, entry, arg0, kernel_stack_top(&new_task->kernel_stack), true);
+    if (s != 0) {
+        stack_destroy(&new_task->kernel_stack);
+        mem_free(new_task);
+        return s;
+    }
+
+    *out = new_task;
+    return 0;
+}
+
+void task_destroy(struct task* task) {
+    arch_task_fini(&task->context);
+    stack_destroy(&task->kernel_stack);
+    mem_free(task);
+}
+
+void task_entry(task_fn_t entry, uintptr_t arg0, struct task* prev) {
+    finish_switch(prev);
+
+    *percpu_ref(irq_data) = (struct irq_percpu){
+        .depth = 1,
+        .saved_state = true,
+    };
+    irq_unlock();
+
+    entry(arg0);
+
+    struct scheduler* sched = &percpu_get()->sched;
+    sched->current->state = TASK_STATE_DEAD;
+    sched_yield(sched);
+    panic("unreachable");
+}

core/spin.c

@@ -1,6 +1,11 @@
-#include <kernel/spin.h>
+#include <zinnia/spin.h>
 #include <stdatomic.h>
 
+void spin_init(struct spinlock* spin) {
+    spin->next = 0;
+    spin->owner = 0;
+}
+
 void spin_lock(struct spinlock* spin) {
     uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed);
 

core/wait.c

@@ -0,0 +1,17 @@
+#include <zinnia/waitq.h>
+
+void waitq_init(struct waitq* wq) {
+    tailq_init(&wq->waiters);
+}
+
+void waitq_wait(struct waitq* wq) {
+    // TODO
+}
+
+void waitq_wake_one(struct waitq* wq) {
+    // TODO
+}
+
+void waitq_wake_all(struct waitq* wq) {
+    // TODO
+}

dev/Kconfig

@@ -0,0 +1,18 @@
+menu "Device Drivers"
+
+config OF
+  bool "Support for OpenFirmware"
+  default y
+  help
+    Adds support for flattened device trees and OF platform drivers.
+
+config ACPI
+  bool "Support for ACPI firmware"
+  default y
+  help
+    Adds support for ACPI firmware, tables and devices.
+
+source "dev/block/Kconfig"
+source "dev/pci/Kconfig"
+
+endmenu

dev/acpi/include/zinnia/acpi.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <zinnia/init.h>
+#include <zinnia/types.h>
+
+void acpi_init(struct boot_info* info);
+
+extern phys_t acpi_rsdp_addr;

dev/acpi/init.c

@@ -0,0 +1,25 @@
+#include <zinnia/acpi.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/print.h>
+#include <zinnia/types.h>
+
+phys_t acpi_rsdp_addr = 0;
+
+static bool use_acpi = true;
+void use_acpi_option(bool use) {
+    use_acpi = use;
+}
+CMDLINE_OPTION("acpi", "Enable ACPI firmware subsystem", use_acpi_option);
+
+void acpi_init(struct boot_info* info) {
+    if (!use_acpi) {
+        log("acpi: ACPI disabled on the command line!\n");
+        return;
+    }
+
+    if (!info->rsdp) {
+        log("acpi: No RSDP was passed, unable to start ACPI subsystem!\n");
+        return;
+    }
+    acpi_rsdp_addr = info->rsdp;
+}

dev/acpi/meson.build

@@ -0,0 +1,16 @@
+uacpi = subproject('uacpi')
+
+kernel_sources += uacpi.get_variable('sources')
+kernel_sources += files(
+    'init.c',
+    'uacpi_glue.c',
+)
+
+kernel_includes += uacpi.get_variable('includes')
+kernel_includes += include_directories('include')
+
+kernel_c_args += [
+    '-DUACPI_FORMATTED_LOGGING',
+    '-DUACPI_BAREBONES_MODE', # TODO: Remove
+    '-DUACPI_OVERRIDE_TYPES',
+]

dev/acpi/uacpi_glue.c

@@ -0,0 +1,54 @@
+#include <zinnia/acpi.h>
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/mmio.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+#include <stdarg.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <uacpi/types.h>
+
+uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) {
+    *out_rsdp_address = (uacpi_phys_addr)acpi_rsdp_addr;
+    return UACPI_STATUS_OK;
+}
+
+void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
+    uintptr_t aligned_addr = ALIGN_DOWN(addr, arch_mem_page_size());
+    uintptr_t difference = addr - aligned_addr;
+    size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
+
+    uintptr_t vaddr = vm_alloc(aligned_len);
+    if (!vaddr)
+        return nullptr;
+
+    for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) {
+        pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PROT_READ | PROT_WRITE, CACHE_ANY);
+    }
+
+    return (void*)(vaddr + difference);
+}
+
+void uacpi_kernel_unmap(void* addr, uacpi_size len) {
+    // TODO
+}
+
+void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
+    log("acpi: ");
+
+    va_list args;
+    va_start(args, msg);
+
+    kvprintf(msg, args);
+
+    va_end(args);
+}
+
+void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
+    kvprintf(msg, args);
+}

dev/block/Kconfig

@@ -0,0 +1,5 @@
+menu "Block device drivers"
+
+source "dev/block/nvme/Kconfig"
+
+endmenu

dev/block/meson.build

@@ -0,0 +1,3 @@
+if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
+    subdir('nvme')
+endif

dev/block/nvme/Kconfig

@@ -0,0 +1,13 @@
+menu "NVMe Support"
+
+config NVME_CORE
+  bool
+  default y
+
+config NVME_PCI
+  bool "NVMe via PCI transport"
+  default y
+  depends on PCI
+  depends on NVME_CORE
+
+endmenu

dev/block/nvme/core.c

@@ -0,0 +1,23 @@
+#include <zinnia/cmdline.h>
+#include <zinnia/module.h>
+#include <zinnia/print.h>
+
+errno_t nvme_init(struct module* mod) {
+    log("Hello world from NVMe!\n");
+
+    return 0;
+}
+
+errno_t nvme_exit(struct module* mod) {
+    log("Good bye from NVMe!\n");
+
+    return 0;
+}
+
+MODULE_ENTRY(nvme_init);
+MODULE_EXIT(nvme_exit);
+
+MODULE_DESC("NVMe Block Devices");
+MODULE_AUTHOR("Marvin Friedrich");
+MODULE_LICENSE("ISC");
+MODULE_VERSION("1.0");

dev/block/nvme/meson.build

@@ -0,0 +1,12 @@
+nvme_sources = files('core.c')
+
+if kconfig.get('CONFIG_NVME_PCI', 'n') == 'y'
+    nvme_sources += files('pci.c')
+endif
+
+modules += {
+    'nvme': {
+        'modular': kconfig.get('CONFIG_NVME_CORE') == 'm',
+        'sources': nvme_sources,
+    },
+}

dev/block/nvme/spec.h

@@ -0,0 +1 @@
+#pragma once

dev/common/memfiles.c

@@ -0,0 +1 @@
+

dev/common/meson.build

@@ -0,0 +1,3 @@
+kernel_sources += files(
+    'memfiles.c',
+)

dev/meson.build

@@ -0,0 +1,9 @@
+subdir('common')
+subdir('block')
+subdir('dt')
+
+if kconfig.get('CONFIG_ACPI', 'n') == 'y'
+    subdir('acpi')
+endif
+
+module_lds = meson.current_source_dir() / 'module.ld'

dev/module.ld

@@ -0,0 +1,109 @@
+/* Linker script for kernel modules */
+
+PHDRS
+{
+    text        PT_LOAD     FLAGS(5);
+    rodata      PT_LOAD     FLAGS(4);
+    data        PT_LOAD     FLAGS(6);
+    dynamic     PT_DYNAMIC  FLAGS(6);
+    modversion  0x60000001;
+    modauthor   0x60000002;
+    moddesc     0x60000003;
+    modlicense  0x60000004;
+    modcmdline  0x60000005;
+    modentry    0x60000006;
+    modexit     0x60000007;
+}
+
+SECTIONS
+{
+    . = ALIGN(CONSTANT(MAXPAGESIZE));
+
+    /* Code */
+    .text : {
+        *(.init .init.*)
+        *(.text .text.*)
+        *(.fini .fini.*)
+        *(.plt .plt.*)
+    } :text
+
+    . = ALIGN(CONSTANT(MAXPAGESIZE));
+
+    /* Read-only data */
+    .rodata : {
+        *(.rodata .rodata.*)
+        *(.hash .hash.*)
+        *(.got .got.*)
+        *(.got.plt .got.plt.*)
+    } :rodata
+
+    .mod.version : {
+        KEEP(*(.mod.version))
+    } :rodata :modversion
+
+    .mod.author : {
+        KEEP(*(.mod.author))
+    } :rodata :modauthor
+
+    .mod.desc : {
+        KEEP(*(.mod.desc))
+    } :rodata :moddesc
+
+    .mod.license : {
+        KEEP(*(.mod.license))
+    } :rodata :modlicense
+
+    .cmdline : {
+        KEEP(*(.cmdline))
+    } :rodata :modcmdline
+
+    .mod.entry : {
+        KEEP(*(.mod.entry))
+    } :rodata :modentry
+
+    .mod.exit : {
+        KEEP(*(.mod.exit))
+    } :rodata :modexit
+
+    . = ALIGN(CONSTANT(MAXPAGESIZE));
+
+    /* Read-Write data */
+    .data : {
+        *(.data)
+        *(.data.*)
+    } :data
+
+    /* Zero-initialized memory */
+    .bss : {
+        *(.*bss .*bss.*)
+        *(COMMON)
+    } :data
+
+    .dynsym : {
+        *(.dynsym .dynsym.*)
+    } :data
+
+    .dynstr : {
+        *(.dynstr .dynstr.*)
+    } :data
+
+    .rela.dyn : {
+        *(.rela.dyn .rela.dyn.*)
+    } :data
+
+    /* Dynamic tags */
+    .dynamic : {
+        *(.dynamic)
+    } :data :dynamic
+
+    /DISCARD/ :
+    {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.rel.eh_frame*)
+        *(.relr.dyn .relr.dyn.*)
+        *(.gcc_except_table*)
+        *(.note*)
+        *(.interp)
+    }
+}

dev/pci/Kconfig

@@ -0,0 +1,7 @@
+menu "PCI bus drivers"
+
+config PCI
+  bool "PCI core drivers"
+  default y
+
+endmenu

fs/exec/elf.c

@@ -1,56 +1,93 @@
-#include <zinnia/mem.h>
-#include <zinnia/status.h>
-#include <common/elf.h>
-#include <common/utils.h>
-#include <kernel/alloc.h>
-#include <kernel/arch.h>
-#include <kernel/elf.h>
-#include <kernel/percpu.h>
-#include <kernel/pmap.h>
-#include <kernel/print.h>
-#include <kernel/sched.h>
-#include <kernel/vas.h>
-#include <kernel/vdso.h>
-#include <kernel/vmo.h>
-#include <assert.h>
-#include <string.h>
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/elf.h>
+#include <zinnia/errno.h>
+#include <zinnia/mman.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/string.h>
+#include <zinnia/utils.h>
+#include <zinnia/vfs.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+#include <bits/elf.h>
+#include <stdckdint.h>
 
-zn_status_t elf_load(struct exec_info* info, struct task** out) {
-    zn_status_t status;
+errno_t elf_load(struct exec_info* info, struct task** out) {
+    errno_t err;
 
     size_t read;
     struct elf_ehdr ehdr = {};
-    vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
+    err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
+    if (err)
+        return err;
+    if (read != sizeof(ehdr))
+        return ENOEXEC;
 
     // Check if the file is an ELF.
     if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
-        return ZN_ERR_INVALID;
+        return EINVAL;
     if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
-        return ZN_ERR_INVALID;
+        return EINVAL;
     if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS)
-        return ZN_ERR_INVALID;
+        return EINVAL;
     if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
-        return ZN_ERR_INVALID;
+        return EINVAL;
     if (ehdr.e_machine != ELF_ARCH_MACHINE)
-        return ZN_ERR_INVALID;
+        return EINVAL;
     if (ehdr.e_type != ET_EXEC)
-        return ZN_ERR_INVALID;
+        return EINVAL;
+    if (!arch_mem_addr_is_user(ehdr.e_entry))
+        return ENOEXEC;
+
+    const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
+    uintptr_t phdrs_end;
+    if (ehdr.e_phentsize < sizeof(struct elf_phdr) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
+        return ENOEXEC;
+    if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size))
+        return ENOEXEC;
 
     const size_t page_size = arch_mem_page_size();
+    uintptr_t phdr_addr = 0;
 
     for (size_t i = 0; i < ehdr.e_phnum; i++) {
         struct elf_phdr phdr = {};
-        vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
-        if (phdr.p_type == PT_LOAD) {
-            enum zn_vm_flags prot = 0;
-            if (phdr.p_flags & PF_R)
-                prot |= ZN_VM_MAP_READ;
-            if (phdr.p_flags & PF_W)
-                prot |= ZN_VM_MAP_WRITE;
-            if (phdr.p_flags & PF_X)
-                prot |= ZN_VM_MAP_EXEC;
+        err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
+        if (err)
+            return err;
+        if (read != sizeof(phdr))
+            return ENOEXEC;
 
-            assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
+        if (phdr.p_type == PT_PHDR)
+            phdr_addr = phdr.p_vaddr;
+
+        if (phdr.p_type == PT_LOAD) {
+            if (phdr.p_memsz == 0)
+                continue;
+            if (phdr.p_filesz > phdr.p_memsz)
+                return ENOEXEC;
+
+            uintptr_t seg_end;
+            if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz))
+                return ENOEXEC;
+            if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1))
+                return ENOEXEC;
+
+            if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
+                return ENOEXEC;
+
+            if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
+                phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset);
+
+            enum prot_flags prot = 0;
+            if (phdr.p_flags & PF_R)
+                prot |= PROT_READ;
+            if (phdr.p_flags & PF_W)
+                prot |= PROT_WRITE;
+            if (phdr.p_flags & PF_X)
+                prot |= PROT_EXEC;
 
             const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
             const uintptr_t map_address = phdr.p_vaddr - misalign;
@@ -59,18 +96,21 @@ zn_status_t elf_load(struct exec_info* info, struct task** out) {
 
             // Copy the file data into its own mapping.
             struct paged_vmo* phdr_obj;
-            status = vmo_new_phys(&phdr_obj);
-            if (status)
-                return status;
+            err = vmo_new_phys(&phdr_obj);
+            if (err)
+                return err;
 
-            status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
-            if (status)
-                return status;
+            err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
+            if (err) {
+                vmo_unref(&phdr_obj->object);
+                return err;
+            }
 
             // We map more than we copied so the rest is filled with zeroed pages.
-            status = vas_map_vmo(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
-            if (status)
-                return status;
+            err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
+            vmo_unref(&phdr_obj->object);
+            if (err)
+                return err;
         }
     }
 
@@ -80,9 +120,9 @@ zn_status_t elf_load(struct exec_info* info, struct task** out) {
 
     // Allocate stack.
     struct paged_vmo* stack;
-    status = vmo_new_phys(&stack);
-    if (status)
-        return status;
+    err = vmo_new_phys(&stack);
+    if (err)
+        return err;
 
     // Fill in stack info.
     uintptr_t stack_off = stack_size;
@@ -96,8 +136,21 @@ zn_status_t elf_load(struct exec_info* info, struct task** out) {
     for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
         num_argv++;
     }
+
+    size_t strings_size = 0;
+    for (size_t env = 0; env < num_envp; env++)
+        strings_size += strlen(info->envp[env]) + 1;
+    for (size_t arg = 0; arg < num_argv; arg++)
+        strings_size += strlen(info->argv[arg]) + 1;
+
     uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
     uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
+    if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
+        mem_free(envp_offsets);
+        mem_free(argv_offsets);
+        vmo_unref(&stack->object);
+        return ENOMEM;
+    }
 
     for (size_t env = 0; env < num_envp; env++) {
         const char nul = 0;
@@ -132,14 +185,6 @@ zn_status_t elf_load(struct exec_info* info, struct task** out) {
         vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
     }
 
-    // Add the init handle to the process namespace.
-    zn_handle_t init_handle = ZN_HANDLE_INVALID;
-    if (info->init_handle) {
-        status = namespace_add_desc(info->ns, *info->init_handle, &init_handle);
-        if (status)
-            return status;
-    }
-
 #define WRITE_AUXV(auxv, value) \
     do { \
         stack_off -= sizeof(uintptr_t); \
@@ -156,23 +201,24 @@ zn_status_t elf_load(struct exec_info* info, struct task** out) {
     WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
     WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
     WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
-    WRITE_AUXV(AT_INIT_HANDLE, init_handle);
+    if (phdr_addr)
+        WRITE_AUXV(AT_PHDR, phdr_addr);
 
     // envp pointers.
     stack_off -= sizeof(uintptr_t);
     const uintptr_t zero = 0;
     vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    for (size_t env = 0; env < num_envp; env++) {
+    for (size_t env = num_envp; env > 0; env--) {
         stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr);
+        vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr);
     }
 
     // argv pointers.
     stack_off -= sizeof(uintptr_t);
     vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    for (size_t arg = 0; arg < num_argv; arg++) {
+    for (size_t arg = num_argv; arg > 0; arg--) {
         stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr);
+        vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr);
     }
 
     mem_free(envp_offsets);
@@ -183,20 +229,25 @@ zn_status_t elf_load(struct exec_info* info, struct task** out) {
     uintptr_t argc = num_argv;
     vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
 
-    status = vas_map_vmo(info->space, &stack->object, stack_start, stack_size, ZN_VM_MAP_READ | ZN_VM_MAP_WRITE, 0);
-    if (status)
-        return status;
+    err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
+    vmo_unref(&stack->object);
+    if (err)
+        return err;
 
     struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
+    if (!start_info)
+        return ENOMEM;
     start_info->ip = ehdr.e_entry;
     start_info->sp = highest - stack_size + stack_off;
     start_info->arg = 0;
 
     struct task* result;
-    status = task_create(info->argv[0], info->space, info->ns, sched_to_user, (uintptr_t)start_info, &result);
-    if (status)
-        return status;
+    err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
+    if (err) {
+        mem_free(start_info);
+        return err;
+    }
 
     *out = result;
-    return ZN_OK;
+    return 0;
 }