initgraph, whole lot of refactor

arch/x86_64/gdt.c

@@ -1,6 +1,6 @@
 #include <zinnia/percpu.h>
-#include <zinnia/string.h>
 #include <stddef.h>
+#include <string.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>

arch/x86_64/hpet.c

@@ -1,7 +1,10 @@
+#include <zinnia/acpi.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/utils.h>
@@ -47,15 +50,15 @@ static struct hpet hpet_clock = {
     },
 };
 
-static char table_buf[0x1000];
-
-void hpet_init() {
+static void hpet_init(struct boot_info*) {
     if (!use_hpet)
         return;
 
-    uacpi_setup_early_table_access(table_buf, sizeof(table_buf));
     uacpi_table table = {0};
-    assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
+    if (uacpi_table_find_by_signature("HPET", &table) != UACPI_STATUS_OK) {
+        log("x86_64/hpet: No HPET table found\n");
+        return;
+    }
 
     struct acpi_hpet* hpet = table.ptr;
 
@@ -73,3 +76,6 @@ void hpet_init() {
 
     assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");
 }
+INIT_TASK(hpet_stage, "arch.x86_64.hpet", hpet_init);
+INIT_DEPENDS(hpet_stage, acpi_stage);
+INIT_ENTAILS(hpet_stage, percpu_stage);

arch/x86_64/include/bits/elf.h

@@ -11,19 +11,21 @@
 #define ELF_ST_BIND ELF64_ST_BIND
 #define ELF_ST_TYPE ELF64_ST_TYPE
 
-#define elf_ehdr elf64_ehdr
-#define elf_phdr elf64_phdr
-#define elf_dyn  elf64_dyn
-#define elf_addr elf64_addr
-#define elf_off  elf64_off
-#define elf_nhdr elf64_nhdr
-#define elf_auxv elf64_auxv
-#define elf_sym  elf64_sym
-#define elf_rela elf64_rela
-#define elf_rel  elf64_rel
+typedef elf64_ehdr_t elf_ehdr_t;
+typedef elf64_phdr_t elf_phdr_t;
+typedef elf64_dyn_t elf_dyn_t;
+typedef elf64_addr_t elf_addr_t;
+typedef elf64_off_t elf_off_t;
+typedef elf64_nhdr_t elf_nhdr_t;
+typedef elf64_auxv_t elf_auxv_t;
+typedef elf64_sym_t elf_sym_t;
+typedef elf64_rela_t elf_rela_t;
+typedef elf64_rel_t elf_rel_t;
+typedef elf64_relr_t elf_relr_t;
 
 enum {
     R_NONE = 0,
+    R_64 = 1,
     R_COPY = 5,
     R_GLOB_DAT = 6,
     R_JUMP_SLOT = 7,

arch/x86_64/include/x86_64/hpet.h

@@ -1,3 +0,0 @@
-#pragma once
-
-void hpet_init();

arch/x86_64/include/x86_64/smp.h

@@ -2,7 +2,7 @@
 
 #include <zinnia/compiler.h>
 #include <zinnia/utils.h>
-#include <stddef.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <x86_64/gdt.h>
 
@@ -17,7 +17,7 @@ struct [[__packed]] smp_info {
     uint64_t hhdm_offset;
     uint64_t percpu;
     uint32_t lapic_id;
-    _Atomic(uint8_t) booted;
+    atomic_bool booted;
 };
 
 // Discovers all APs and starts them.

arch/x86_64/meson.build

@@ -23,3 +23,7 @@ kernel_sources += files(
     'tsc.c',
     'usercopy.c',
 )
+
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('pci.c')
+endif

arch/x86_64/pci.c

@@ -0,0 +1,50 @@
+#include <zinnia/init.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <x86_64/asm.h>
+
+constexpr uint16_t PCI_CONFIG_ADDRESS = 0xcf8;
+constexpr uint16_t PCI_CONFIG_DATA = 0xcfc;
+
+static struct spinlock port_lock = {};
+
+static void port_select(const struct pci_address* addr, uint16_t offset) {
+    asm_outl(
+        PCI_CONFIG_ADDRESS,
+        1u << 31 | (uint32_t)addr->bus << 16 | (uint32_t)(addr->device & 0x1f) << 11 |
+            (uint32_t)(addr->function & 0x7) << 8 | (offset & 0xfc)
+    );
+}
+
+static uint32_t port_read32(struct pci_access*, const struct pci_address* addr, uint16_t offset) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    uint32_t value = asm_inl(PCI_CONFIG_DATA);
+    spin_unlock(&port_lock);
+    return value;
+}
+
+static void port_write32(struct pci_access*, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    asm_outl(PCI_CONFIG_DATA, value);
+    spin_unlock(&port_lock);
+}
+
+static struct pci_access port_access = {
+    .name = "port",
+    .domain = 0,
+    .start_bus = 0,
+    .end_bus = 255,
+    .priority = PCI_ACCESS_PRIO_LEGACY,
+    .read32 = port_read32,
+    .write32 = port_write32,
+};
+
+[[__init]]
+static void pci_x86_stage_init(struct boot_info*) {
+    pci_access_register(&port_access);
+}
+INIT_TASK(pci_x86_stage, "arch.x86_64.pci", pci_x86_stage_init);
+INIT_DEPENDS(pci_x86_stage, mem_stage);
+INIT_ENTAILS(pci_x86_stage, pci_stage);

arch/x86_64/percpu.c

@@ -1,3 +1,4 @@
+#include <zinnia/initgraph.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <config.h>
@@ -6,7 +7,6 @@
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
-#include <x86_64/hpet.h>
 #include <x86_64/idt.h>
 #include <x86_64/percpu.h>
 #include <x86_64/smp.h>
@@ -132,7 +132,6 @@ void x86_64_cpu_setup(struct percpu* cpu) {
 }
 
 void arch_percpu_init() {
-    hpet_init();
 #if CONFIG_TSC
     tsc_init();
 #endif

arch/x86_64/serial.c

@@ -1,4 +1,6 @@
 #include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/initgraph.h>
 #include <stdint.h>
 #include <x86_64/asm.h>
 
@@ -44,4 +46,10 @@ static struct console serial_con = {
     .write = serial_write,
     .init = serial_setup,
 };
-DEFINE_CONSOLE(&serial_con);
+
+[[__init]]
+static void serial_init(struct boot_info*) {
+    console_register(&serial_con);
+}
+INIT_TASK(serial_stage, "arch.x86_64.serial", serial_init);
+INIT_ENTAILS(serial_stage, console_stage);

arch/x86_64/smp.c

@@ -1,3 +1,5 @@
+#define pr_fmt(fmt) "smp: " fmt
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
@@ -7,13 +9,13 @@
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
-#include <zinnia/string.h>
 #include <zinnia/types.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
 #include <stdatomic.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 #include <uacpi/acpi.h>
 #include <uacpi/tables.h>
 #include <x86_64/apic.h>
@@ -37,11 +39,12 @@ static void smp_trampoline() {
         "    cld\n"
         "    jmp .Lmode16\n"
 
+        ".org smp_trampoline_start + %c[data_off]\n"
         ".global smp_trampoline_data\n"
         "smp_trampoline_data:\n"
         "    .skip %c[info_size]\n"
 
-        ".set .Ldata_offset, (smp_trampoline_data - smp_trampoline_start)\n"
+        ".set .Ldata_offset, %c[data_off]\n"
         ".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
         ".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
         ".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
@@ -130,7 +133,8 @@ static void smp_trampoline() {
         ".popsection\n"
         "    ud2\n"
         :
-        : [info_size] "i"(sizeof(struct smp_info)),
+        : [data_off] "i"(8),
+          [info_size] "i"(sizeof(struct smp_info)),
           [gdtr] "i"(offsetof(struct smp_info, gdtr)),
           [farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
           [temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
@@ -170,7 +174,7 @@ static void ap_entry(phys_t info_addr) {
     // `info` must not be touched after this point.
     atomic_store_explicit(&info->booted, 1, memory_order_release);
 
-    log("smp: CPU %zu online\n", cpu->id);
+    log("CPU %zu online\n", cpu->id);
 
     sched_yield(&cpu->sched);
     unreachable();
@@ -181,18 +185,18 @@ static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
 
     phys_t stack;
     if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
-        log("smp: Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
+        log_err("Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
         return false;
     }
 
     struct percpu* cpu = percpu_new();
     if (!cpu) {
-        log("smp: Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
+        log_err("Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
         mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
         return false;
     }
 
-    log("smp: Starting CPU %zu\n", cpu->id);
+    log("Starting CPU %zu\n", cpu->id);
 
     memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
 
@@ -230,7 +234,7 @@ static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
     for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) {
         if (waited >= AP_BOOT_TIMEOUT_NS) {
             // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
-            log("smp: CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
+            log_err("CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
             return false;
         }
         clock_spin_ns(1'000'000);
@@ -279,20 +283,20 @@ void x86_64_smp_init() {
 
     uacpi_table table;
     if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
-        log("smp: No MADT found, not starting any APs\n");
+        log_warn("No MADT found, not starting any APs\n");
         return;
     }
 
     phys_t trampoline;
     if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
-        log("smp: Failed to allocate 20-bit memory for the AP trampoline\n");
+        log_err("Failed to allocate 20-bit memory for the AP trampoline\n");
         uacpi_table_unref(&table);
         return;
     }
 
     phys_t temp_table;
     if (build_temp_table(&temp_table)) {
-        log("smp: Failed to allocate a temporary page table\n");
+        log_err("Failed to allocate a temporary page table\n");
         mem_phys_free(trampoline, 1);
         uacpi_table_unref(&table);
         return;
@@ -311,7 +315,7 @@ void x86_64_smp_init() {
 
         // Don't trust malformed firmware tables.
         if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
-            log("smp: MADT contains a malformed entry\n");
+            log_warn("MADT contains a malformed entry\n");
             break;
         }
 
@@ -348,5 +352,5 @@ void x86_64_smp_init() {
         mem_phys_free(trampoline, 1);
     }
 
-    log("smp: %zu CPU(s) online\n", num_cpus);
+    log("%zu CPU(s) online\n", num_cpus);
 }

arch/x86_64/tsc.c

@@ -1,3 +1,4 @@
+#define pr_fmt(fmt) "x86_64/tsc: " fmt
 #include <zinnia/clock.h>
 #include <zinnia/cmdline.h>
 #include <zinnia/print.h>
@@ -42,11 +43,13 @@ static struct clock tsc_clock = {
 };
 
 void tsc_init() {
-    if (!use_tsc)
+    if (!use_tsc) {
+        log_warn("TSC disabled on the command line");
         return;
+    }
 
     if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
-        log("x86_64/tsc: No invariant TSC detected!\n");
+        log_warn("No invariant TSC detected!\n");
         return;
     }
 
@@ -69,17 +72,17 @@ void tsc_init() {
         // On a normal system, this should usually never be called and is a last resort
         // since at this point we have at least the HPET timer.
         if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
-            log("x86_64/tsc: Calibrating TSC using CPUID 0x15");
+            log("Calibrating TSC using CPUID 0x15");
             tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
         } else {
-            log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information");
+            log_err("Unable to calibrate TSC using CPUID frequency information");
             return;
         }
     } else {
-        log("x86_64/tsc: Unable to calibrate the TSC without a clock or static frequency info!\n");
+        log_err("Unable to calibrate the TSC without a clock or static frequency info!\n");
         return;
     }
 
-    log("x86_64/tsc: Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
+    log("Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
     clock_switch(&tsc_clock);
 }

boot/cmdline.c

@@ -1,8 +1,8 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/init.h>
-#include <stddef.h>
 #include <zinnia/stdlib.h>
-#include <zinnia/string.h>
+#include <stddef.h>
+#include <string.h>
 
 // Defined in linker script.
 extern const struct cmdline_option* __ld_cmdline_start[];
@@ -10,17 +10,16 @@ extern const struct cmdline_option* __ld_cmdline_end[];
 
 static char cmdline_buf[0x1000];
 
-[[__init]]
-void cmdline_parse(const char* cmdline) {
-    if (cmdline == nullptr)
-        cmdline = "";
-
-    const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
+void cmdline_parse_buf(
+    char* cmdline_buf,
+    size_t len,
+    const struct cmdline_option* const* start,
+    const struct cmdline_option* const* end
+) {
     size_t idx = 0;
-    memcpy(cmdline_buf, cmdline, len);
 
     while (1) {
-        const struct cmdline_option** opt = __ld_cmdline_start;
+        const struct cmdline_option* const* opt = start;
         char* name = nullptr;
         char* value = nullptr;
 
@@ -62,7 +61,7 @@ void cmdline_parse(const char* cmdline) {
         }
 
         // Find the corresponding option.
-        while (opt < __ld_cmdline_end) {
+        while (opt < end) {
             const struct cmdline_option* current = *opt;
             if (!strcmp(current->name, name)) {
                 if (value == nullptr)
@@ -105,3 +104,13 @@ void cmdline_parse(const char* cmdline) {
             break;
     }
 }
+
+[[__init]]
+void cmdline_parse(const char* cmdline) {
+    if (cmdline == nullptr)
+        cmdline = "";
+
+    const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
+    memcpy(cmdline_buf, cmdline, len);
+    cmdline_parse_buf(cmdline_buf, len, __ld_cmdline_start, __ld_cmdline_end);
+}

boot/kernel.ld

@@ -6,6 +6,7 @@ PHDRS {
     data    PT_LOAD     FLAGS(6);
     dynamic PT_DYNAMIC  FLAGS(6);
     percpu  PT_LOAD     FLAGS(6);
+    cmdline 0x60000005;
 }
 
 SECTIONS
@@ -28,7 +29,7 @@ SECTIONS
         __ld_cmdline_start = .;
         KEEP(*(.cmdline))
         __ld_cmdline_end = .;
-    } :rodata
+    } :rodata :cmdline
 
     .rodata : {
         *(.rodata .rodata.*)
@@ -39,6 +40,15 @@ SECTIONS
         KEEP(*(.mod.entry .mod.entry.*))
         __ld_mod_entry_end = .;
 
+        . = ALIGN(8);
+        __ld_initgraph_nodes_start = .;
+        KEEP(*(.initgraph.nodes))
+        __ld_initgraph_nodes_end = .;
+
+        __ld_initgraph_edges_start = .;
+        KEEP(*(.initgraph.edges))
+        __ld_initgraph_edges_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -52,12 +62,6 @@ SECTIONS
         *(.data .data.*)
     } :data
 
-    .console : {
-        __ld_console_start = .;
-        KEEP(*(.console))
-        __ld_console_end = .;
-    } :data
-
     .dynsym : {
         __ld_dynsym_start = .;
         KEEP(*(.dynsym*))

boot/limine/entry.c

@@ -3,8 +3,8 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/init.h>
 #include <zinnia/print.h>
-#include <zinnia/string.h>
 #include <zinnia/utils.h>
+#include <string.h>
 #include "limine.h"
 
 [[__initdata_sorted("limine.0")]]
@@ -53,6 +53,8 @@ void kernel_entry() {
     assert(hhdm_res, "hhdm_request returned no response!");
 
     info.cmdline = exec_file_res->executable_file->string;
+    info.kernel_file.data = (phys_t)((uintptr_t)exec_file_res->executable_file->address - hhdm_res->offset);
+    info.kernel_file.length = exec_file_res->executable_file->size;
 
     info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
     for (size_t i = 0; i < info.num_mem_maps; i++) {

boot/main.c

@@ -6,8 +6,8 @@
 #include <zinnia/console.h>
 #include <zinnia/elf.h>
 #include <zinnia/init.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/irq.h>
-#include <zinnia/module.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
@@ -17,10 +17,6 @@
 #include <config.h>
 #include <stdint.h>
 
-#if CONFIG_ACPI
-#include <zinnia/acpi.h>
-#endif
-
 static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
                                     ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
 
@@ -30,20 +26,22 @@ static void kernel_main_task(uintptr_t arg0) {
 
     log("main: Hello from main task!\n");
 
-    // Initialize built-in modules.
-    module_init();
+    // Finish the initgraph.
+    initgraph_run(nullptr, info);
+
+    // TODO: Try to run init.
 
     assert(info->num_files >= 1, "No init executable provided");
 
     struct vma* init_space;
-    assert(vma_new(&init_space) == EOK, "");
+    assert(vma_new(&init_space) == 0, "");
 
     struct boot_file* file_data = &info->files[0];
     struct task* init_task = nullptr;
 
     struct paged_vmo* init_file;
     status = vmo_new_phys(&init_file);
-    assert(status == EOK, "Failed to allocate a memory object");
+    assert(status == 0, "Failed to allocate a memory object");
 
     vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
 
@@ -83,17 +81,8 @@ void kernel_main(struct boot_info* info) {
     kprintf("Command line: %s\n", info->cmdline);
 
     cmdline_parse(info->cmdline);
-    console_init();
-    kprintf("\ec");
 
-    mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
-
-#if CONFIG_ACPI
-    acpi_init(info);
-#endif
-
-    percpu_init();
-    sched_init(&percpu_get()->sched);
+    initgraph_run(sched_stage, info);
 
     struct task* main_task;
     task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);

core/clock.c

@@ -1,3 +1,5 @@
+#define pr_fmt(fmt) "clock: " fmt
+
 #include <zinnia/clock.h>
 #include <zinnia/compiler.h>
 #include <zinnia/print.h>
@@ -47,14 +49,14 @@ bool clock_switch(struct clock* clock) {
 
     spin_unlock(&lock);
 
-    log("clock: Switching to clock source \"%s\"\n", clock->name);
+    log("Switching to clock source \"%s\"\n", clock->name);
 
     return true;
 }
 
 void clock_spin_ns(uint64_t ns) {
     if (!clock_available()) {
-        log("clock: Unable to sleep for %lu ns, no clock available!\n", ns);
+        log_err("Unable to sleep for %lu ns, no clock available!\n", ns);
         return;
     }
 

core/console.c

@@ -1,45 +1,44 @@
-#include <zinnia/cmdline.h>
 #include <zinnia/console.h>
+#include <zinnia/init.h>
 #include <zinnia/print.h>
-#include <zinnia/string.h>
+#include <zinnia/spin.h>
 
-extern struct console* const __ld_console_start[];
-extern struct console* const __ld_console_end[];
-
-struct console* active = nullptr;
+static struct console* registered = nullptr;
+static struct spinlock console_lock;
 
 void console_write(const char* buf, size_t len) {
-    if (__likely(active))
-        active->write(active, buf, len);
+    spin_lock(&console_lock);
+    for (struct console* con = registered; con; con = con->next)
+        con->write(con, buf, len);
+    spin_unlock(&console_lock);
 }
 
-static void console_setup(const char* value) {
-    if (!value)
-        return;
+bool console_register(struct console* con) {
+    if (!con->init(con))
+        return false;
 
-    // Find a suitable boot console.
-    struct console* const* cur = __ld_console_start;
-    while (cur < __ld_console_end) {
-        if (!strcmp(value, (*cur)->name)) {
-            active = *cur;
+    spin_lock(&console_lock);
+    con->next = registered;
+    registered = con;
+    spin_unlock(&console_lock);
+
+    log("console: Registered console \"%s\"\n", con->name);
+    return true;
+}
+
+void console_unregister(struct console* con) {
+    spin_lock(&console_lock);
+    for (struct console** link = &registered; *link; link = &(*link)->next) {
+        if (*link == con) {
+            *link = con->next;
+            break;
         }
-        cur++;
     }
+    spin_unlock(&console_lock);
 }
 
-CMDLINE_OPTION("console", "Set a default console", console_setup);
-
-void console_init() {
-    // If we have no console set and at least one set, use that as a default.
-    if (!active && &__ld_console_start[0] != &__ld_console_end[0])
-        active = __ld_console_start[0];
-
-    if (active) {
-        if (active->init(active) == false) {
-            active = nullptr;
-            return;
-        }
-
-        log("console: Logging on console \"%s\"\n", active->name);
-    }
+[[__init]]
+static void console_stage_init(struct boot_info*) {
+    kprintf("\ec");
 }
+INIT_TASK(console_stage, "core.console", console_stage_init);

core/fbcon.c

@@ -0,0 +1,184 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/pmap.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stdint.h>
+#include <string.h>
+
+static const uint8_t fbcon_font[] = {
+#embed "fbcon_font.bin"
+};
+
+constexpr size_t FONT_WIDTH = 8;
+constexpr size_t FONT_HEIGHT = 12;
+constexpr uint32_t FG_COLOR = 0xaaaaaa;
+constexpr uint32_t BG_COLOR = 0x000000;
+
+enum esc_state {
+    ESC_NONE,
+    ESC_START,
+    ESC_CSI,
+};
+
+static struct {
+    struct boot_framebuffer fb;
+    uint32_t fg, bg;
+    size_t cols, rows;
+    size_t offset_x, offset_y;
+    size_t cx, cy;
+    enum esc_state esc;
+} fbcon;
+
+static inline uint8_t* fb_ptr() {
+    return HHDM_PTR(fbcon.fb.base);
+}
+
+static uint32_t pack_color(uint32_t rgb) {
+    const struct boot_framebuffer* fb = &fbcon.fb;
+    uint32_t r = (rgb >> 16) & 0xff;
+    uint32_t g = (rgb >> 8) & 0xff;
+    uint32_t b = rgb & 0xff;
+    return ((r >> (8 - fb->red.size)) << fb->red.shift) | ((g >> (8 - fb->green.size)) << fb->green.shift) |
+        ((b >> (8 - fb->blue.size)) << fb->blue.shift);
+}
+
+static void fill_rect(size_t x, size_t y, size_t w, size_t h, uint32_t color) {
+    for (size_t j = 0; j < h; j++) {
+        uint32_t* row = (uint32_t*)(fb_ptr() + (y + j) * fbcon.fb.pitch) + x;
+        for (size_t i = 0; i < w; i++)
+            row[i] = color;
+    }
+}
+
+static void clear_screen() {
+    fill_rect(0, 0, fbcon.fb.width, fbcon.fb.height, fbcon.bg);
+    fbcon.cx = fbcon.cy = 0;
+}
+
+static void draw_glyph(size_t col, size_t row, uint8_t ch) {
+    const uint8_t* glyph = &fbcon_font[ch * FONT_HEIGHT];
+    size_t x = fbcon.offset_x + col * FONT_WIDTH;
+    size_t y = fbcon.offset_y + row * FONT_HEIGHT;
+
+    for (size_t j = 0; j < FONT_HEIGHT; j++) {
+        uint32_t* line = (uint32_t*)(fb_ptr() + (y + j) * fbcon.fb.pitch) + x;
+        for (size_t i = 0; i < FONT_WIDTH; i++)
+            line[i] = (glyph[j] & (0x80 >> i)) ? fbcon.fg : fbcon.bg;
+    }
+}
+
+static void scroll_up() {
+    const size_t pitch = fbcon.fb.pitch;
+    const size_t text_y = fbcon.offset_y;
+    const size_t text_h = fbcon.rows * FONT_HEIGHT;
+
+    memmove(fb_ptr() + text_y * pitch, fb_ptr() + (text_y + FONT_HEIGHT) * pitch, (text_h - FONT_HEIGHT) * pitch);
+    fill_rect(0, text_y + text_h - FONT_HEIGHT, fbcon.fb.width, FONT_HEIGHT, fbcon.bg);
+}
+
+static void newline() {
+    fbcon.cx = 0;
+    if (++fbcon.cy >= fbcon.rows) {
+        fbcon.cy = fbcon.rows - 1;
+        scroll_up();
+    }
+}
+
+static void put_char(uint8_t ch) {
+    switch (fbcon.esc) {
+    case ESC_START:
+        fbcon.esc = ch == '[' ? ESC_CSI : ESC_NONE;
+        if (ch == 'c')
+            clear_screen();
+        return;
+    case ESC_CSI:
+        if (ch >= 0x40 && ch <= 0x7e)
+            fbcon.esc = ESC_NONE;
+        return;
+    case ESC_NONE:
+        break;
+    }
+
+    switch (ch) {
+    case 0x1b:
+        fbcon.esc = ESC_START;
+        break;
+    case '\n':
+        newline();
+        break;
+    case '\r':
+        fbcon.cx = 0;
+        break;
+    case '\b':
+        if (fbcon.cx > 0)
+            fbcon.cx--;
+        break;
+    case '\t':
+        fbcon.cx = (fbcon.cx + 8) & ~(size_t)7;
+        if (fbcon.cx >= fbcon.cols)
+            newline();
+        break;
+    default:
+        if (fbcon.cx >= fbcon.cols)
+            newline();
+        draw_glyph(fbcon.cx++, fbcon.cy, ch);
+        break;
+    }
+}
+
+static void fbcon_write(struct console* con, const char* buf, size_t count) {
+    for (size_t i = 0; i < count; i++)
+        put_char((uint8_t)buf[i]);
+}
+
+static bool fbcon_setup(struct console* con) {
+    const struct boot_framebuffer* fb = &fbcon.fb;
+    if (fb->bytes_per_pixel != 4 || fb->red.size > 8 || fb->green.size > 8 || fb->blue.size > 8)
+        return false;
+
+    fbcon.fg = pack_color(FG_COLOR);
+    fbcon.bg = pack_color(BG_COLOR);
+    fbcon.cols = fb->width / FONT_WIDTH;
+    fbcon.rows = fb->height / FONT_HEIGHT;
+    fbcon.offset_x = (fb->width % FONT_WIDTH) / 2;
+    fbcon.offset_y = (fb->height % FONT_HEIGHT) / 2;
+    if (!fbcon.cols || !fbcon.rows)
+        return false;
+
+    clear_screen();
+    return true;
+}
+
+static struct console fbcon_con = {
+    .name = "fbcon",
+    .init = fbcon_setup,
+    .write = fbcon_write,
+};
+
+[[__init]]
+void fbcon_init(struct boot_info* info) {
+    const struct boot_framebuffer* fb = &info->fb;
+    if (!fb->base)
+        return;
+
+    fbcon.fb = *fb;
+
+    const size_t pgsz = arch_mem_page_size();
+    const phys_t start = ALIGN_DOWN(fb->base, pgsz);
+    const phys_t end = ALIGN_UP(fb->base + fb->pitch * fb->height, pgsz);
+
+    for (phys_t p = start; p < end; p += pgsz) {
+        errno_t status =
+            pmap_map(&kernel_vma.pmap, (uintptr_t)HHDM_PTR(p), p, PTE_READ | PTE_WRITE, CACHE_WRITE_COMBINE);
+        if (status)
+            return;
+    }
+
+    console_register(&fbcon_con);
+}
+INIT_TASK(fbcon_stage, "core.fbcon", fbcon_init);
+INIT_DEPENDS(fbcon_stage, sched_stage);

core/initgraph.c

@@ -0,0 +1,168 @@
+#define pr_fmt(fmt) "init: " fmt
+
+#include <zinnia/assert.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+
+extern struct init_node* const __ld_initgraph_nodes_start[];
+extern struct init_node* const __ld_initgraph_nodes_end[];
+extern const struct init_edges* const __ld_initgraph_edges_start[];
+extern const struct init_edges* const __ld_initgraph_edges_end[];
+
+static bool dump_graph = false;
+static void initgraph_option(bool value) {
+    dump_graph = value;
+}
+CMDLINE_OPTION("initgraph", "Print the init graph in DOT format after running", initgraph_option);
+
+static bool edges_linked = false;
+
+struct init_queue {
+    struct init_node* head;
+    struct init_node* tail;
+};
+
+static void queue_push(struct init_queue* queue, struct init_node* node) {
+    node->queue_next = nullptr;
+    if (queue->tail)
+        queue->tail->queue_next = node;
+    else
+        queue->head = node;
+    queue->tail = node;
+}
+
+static struct init_node* queue_pop(struct init_queue* queue) {
+    struct init_node* node = queue->head;
+    if (node) {
+        queue->head = node->queue_next;
+        if (!queue->head)
+            queue->tail = nullptr;
+        node->queue_next = nullptr;
+    }
+    return node;
+}
+
+static void link_edges() {
+    for (auto cur = __ld_initgraph_edges_start; cur < __ld_initgraph_edges_end; cur++) {
+        const struct init_edges* group = *cur;
+        for (size_t i = 0; i < group->count; i++) {
+            struct init_edge* edge = &group->edges[i];
+            edge->source = group->depends ? group->nodes[i] : group->node;
+            edge->target = group->depends ? group->node : group->nodes[i];
+            edge->propagates_disable = group->depends;
+
+            edge->next_out = edge->source->out_edges;
+            edge->source->out_edges = edge;
+            edge->next_in = edge->target->in_edges;
+            edge->target->in_edges = edge;
+            edge->target->unsatisfied++;
+        }
+    }
+    edges_linked = true;
+}
+
+static void mark_wanted(struct init_node* goal) {
+    if (!goal) {
+        for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++)
+            (*cur)->wanted = true;
+        return;
+    }
+
+    struct init_queue queue = {};
+
+    if (!goal->wanted) {
+        goal->wanted = true;
+        queue_push(&queue, goal);
+    }
+
+    struct init_node* node;
+    while ((node = queue_pop(&queue))) {
+        for (struct init_edge* edge = node->in_edges; edge; edge = edge->next_in) {
+            if (!edge->source->wanted) {
+                edge->source->wanted = true;
+                queue_push(&queue, edge->source);
+            }
+        }
+    }
+}
+
+static bool is_ready(struct init_node* node) {
+    return node->wanted && !node->done && node->unsatisfied == 0;
+}
+
+static void reach_node(struct init_node* node, struct boot_info* info) {
+    if (node->disabled) {
+        log_status("Skipping stage \"%s\"\n", node->name);
+    } else {
+        log_status("Running stage \"%s\"\n", node->name);
+        switch (node->action) {
+        case INIT_EMPTY:
+            break;
+        case INIT_CALLBACK:
+            node->callback(info);
+            break;
+        case INIT_GATE:
+            if (!node->gate(info))
+                node->disabled = true;
+            break;
+        }
+    }
+    node->done = true;
+}
+
+static void dump() {
+    kprintf("digraph initgraph {\n");
+    for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
+        struct init_node* node = *cur;
+        kprintf("\tn%p [label=\"%s\"];\n", node, node->name);
+        for (struct init_edge* edge = node->in_edges; edge; edge = edge->next_in)
+            kprintf("\tn%p -> n%p;\n", edge->source, edge->target);
+    }
+    kprintf("}\n");
+}
+
+void initgraph_run(struct init_node* goal, struct boot_info* info) {
+    if (!edges_linked)
+        link_edges();
+
+    mark_wanted(goal);
+
+    struct init_queue pending = {};
+
+    for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
+        if (is_ready(*cur))
+            queue_push(&pending, *cur);
+    }
+
+    struct init_node* node;
+    while ((node = queue_pop(&pending))) {
+        reach_node(node, info);
+
+        for (struct init_edge* edge = node->out_edges; edge; edge = edge->next_out) {
+            struct init_node* successor = edge->target;
+            assert(successor->unsatisfied != 0, "Stage \"%s\" has no unsatisfied dependencies left", successor->name);
+            successor->unsatisfied--;
+
+            if (node->disabled && edge->propagates_disable)
+                successor->disabled = true;
+
+            if (is_ready(successor))
+                queue_push(&pending, successor);
+        }
+    }
+
+    for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
+        assert(
+            !(*cur)->wanted || (*cur)->done,
+            "The dependencies for stage \"%s\" could not be resolved",
+            (*cur)->name
+        );
+    }
+
+    if (!goal) {
+        log_status("All stages are complete!\n");
+        if (dump_graph)
+            dump();
+    }
+}

core/ksyms.c

@@ -0,0 +1,135 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/compiler.h>
+#include <zinnia/elf.h>
+#include <zinnia/hashmap.h>
+#include <zinnia/init.h>
+#include <zinnia/ksyms.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <bits/elf.h>
+#include <string.h>
+
+[[__visibility("hidden")]]
+extern const elf_sym_t __ld_dynsym_start[];
+[[__visibility("hidden")]]
+extern const elf_sym_t __ld_dynsym_end[];
+[[__visibility("hidden")]]
+extern const char __ld_dynstr_start[];
+[[__visibility("hidden")]]
+extern const char __ld_dynstr_end[];
+[[__visibility("hidden")]]
+extern const elf_dyn_t _DYNAMIC[];
+
+static uintptr_t slide;
+static bool slide_known;
+static const char* kernel_name;
+static hashmap(const char*, uintptr_t) symbols;
+
+static uint64_t sym_hash(const void* key, size_t) {
+    const char* name = *(const char* const*)key;
+    return hashmap_hash_default(name, strlen(name));
+}
+
+static bool sym_eq(const void* a, const void* b, size_t) {
+    return !strcmp(*(const char* const*)a, *(const char* const*)b);
+}
+
+static size_t num_syms() {
+    return __ld_dynsym_end - __ld_dynsym_start;
+}
+
+static const char* dynstr(uint64_t offset) {
+    if (offset >= (size_t)(__ld_dynstr_end - __ld_dynstr_start))
+        return nullptr;
+    return &__ld_dynstr_start[offset];
+}
+
+static uintptr_t kernel_slide() {
+    if (slide_known)
+        return slide;
+
+    for (size_t i = 0; i < num_syms(); i++) {
+        const char* name = dynstr(__ld_dynsym_start[i].st_name);
+        if (name != nullptr && !strcmp(name, "ksym_lookup")) {
+            slide = (uintptr_t)ksym_lookup - __ld_dynsym_start[i].st_value;
+            break;
+        }
+    }
+    slide_known = true;
+    return slide;
+}
+
+void ksym_init() {
+    const uintptr_t base = kernel_slide();
+
+    for (const elf_dyn_t* dyn = _DYNAMIC; dyn->d_tag != DT_NULL; dyn++) {
+        if (dyn->d_tag == DT_SONAME)
+            kernel_name = dynstr(dyn->d_un.d_val);
+    }
+
+    hashmap_init(&symbols);
+
+    size_t count = 0;
+    for (size_t i = 0; i < num_syms(); i++) {
+        const elf_sym_t* sym = &__ld_dynsym_start[i];
+        if (sym->st_shndx == SHN_UNDEF)
+            continue;
+
+        const char* name = dynstr(sym->st_name);
+        if (name == nullptr || *name == 0)
+            continue;
+
+        const errno_t status = hashmap_insert(&symbols, name, sym->st_value + base, sym_hash, sym_eq);
+        assert(status == 0, "Failed to register kernel symbol \"%s\" (%i)", name, status);
+        count++;
+    }
+
+    log("ksyms: Registered %zu kernel symbols\n", count);
+}
+
+bool ksym_find(const char* name, uintptr_t* addr) {
+    uintptr_t* value = hashmap_get(&symbols, name, sym_hash, sym_eq);
+    if (value == nullptr)
+        return false;
+    *addr = *value;
+    return true;
+}
+
+const char* ksym_kernel_name() {
+    return kernel_name;
+}
+
+bool ksym_lookup(uintptr_t addr, const char** name, size_t* offset) {
+    addr -= kernel_slide();
+    for (size_t i = 0; i < num_syms(); i++) {
+        const elf_sym_t* sym = &__ld_dynsym_start[i];
+        if (ELF_ST_TYPE(sym->st_info) != STT_FUNC || sym->st_shndx == SHN_UNDEF)
+            continue;
+        if (addr >= sym->st_value && addr - sym->st_value < sym->st_size) {
+            const char* sym_name = dynstr(sym->st_name);
+            if (sym_name == nullptr)
+                return false;
+            *name = sym_name;
+            *offset = addr - sym->st_value;
+            return true;
+        }
+    }
+    return false;
+}
+
+void ksym_format(uintptr_t addr, char* buf, size_t size) {
+    const char* name;
+    size_t offset;
+    if (ksym_lookup(addr, &name, &offset))
+        ksnprintf(buf, size, "%s+0x%zx", name, offset);
+    else
+        ksnprintf(buf, size, "<\?\?\?>");
+}
+
+[[__init]]
+static void ksyms_stage_init(struct boot_info*) {
+    ksym_init();
+}
+INIT_TASK(ksyms_stage, "core.ksyms", ksyms_stage_init);
+INIT_DEPENDS(ksyms_stage, mem_stage);

core/lib/stdlib.c

@@ -1,6 +1,6 @@
-#include <stdint.h>
 #include <zinnia/stdlib.h>
-#include <zinnia/string.h>
+#include <stdint.h>
+#include <string.h>
 
 static void reverse(char* s) {
     int i, j;

core/lib/string.c

@@ -1,5 +1,4 @@
 #include <zinnia/compiler.h>
-#include <zinnia/string.h>
 #include <zinnia/utils.h>
 #include <stddef.h>
 #include <stdint.h>

core/mem/init.c

@@ -1,9 +1,11 @@
+#define pr_fmt(fmt) "mem: " fmt
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
+#include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/panic.h>
 #include <zinnia/print.h>
-#include <zinnia/string.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
 
@@ -23,7 +25,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
 
     const size_t pgsz = arch_mem_page_size();
 
-    log("mem: Memory map:\n");
+    log("Memory map:\n");
     for (size_t i = 0; i < map_len; i++) {
         if (map[i].length == 0)
             continue;
@@ -44,7 +46,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
             break;
         }
 
-        log("mem: [%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
+        log("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
     }
 
     // Set up the bootstrap allocator.
@@ -55,7 +57,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
     }
     assert(largest && largest->length >= pgsz, "No usable memory for bootstrap allocator");
     mem_phys_bootstrap(largest);
-    log("mem: Using region [%p - %p] for bootstrap allocator\n",
+    log("Using region [%p - %p] for bootstrap allocator\n",
         (void*)largest->address,
         (void*)(largest->address + largest->length - 1));
 
@@ -68,7 +70,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
     tailq_init(&kernel_vma.mappings);
 
     // text
-    log("mem: Mapping text segment at %p\n", __ld_text_start);
+    log("Mapping text segment at %p\n", __ld_text_start);
     for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
         errno_t status = pmap_map(
             &kernel_vma.pmap,
@@ -81,7 +83,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
     }
 
     // rodata
-    log("mem: Mapping rodata segment at %p\n", __ld_rodata_start);
+    log("Mapping rodata segment at %p\n", __ld_rodata_start);
     for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
         errno_t status = pmap_map(
             &kernel_vma.pmap,
@@ -94,7 +96,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
     }
 
     // data
-    log("mem: Mapping data segment at %p\n", __ld_data_start);
+    log("Mapping data segment at %p\n", __ld_data_start);
     for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
         errno_t status = pmap_map(
             &kernel_vma.pmap,
@@ -165,5 +167,12 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
 
     slab_init();
 
-    log("mem: Memory initialization complete\n");
+    log("Memory initialization complete\n");
 }
+
+[[__init]]
+static void mem_stage_init(struct boot_info* info) {
+    mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
+}
+INIT_TASK(mem_stage, "core.mem", mem_stage_init);
+INIT_DEPENDS(mem_stage, console_stage);

core/mem/pmap.c

@@ -2,7 +2,6 @@
 #include <zinnia/errno.h>
 #include <zinnia/pmap.h>
 #include <zinnia/spin.h>
-#include <zinnia/string.h>
 #include <zinnia/vma.h>
 #include <config.h>
 

core/mem/pmm.c

@@ -3,7 +3,6 @@
 #include <zinnia/panic.h>
 #include <zinnia/print.h>
 #include <zinnia/spin.h>
-#include <zinnia/string.h>
 #include <zinnia/virt.h>
 #include <stdint.h>
 

core/mem/slab.c

@@ -2,7 +2,7 @@
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
 #include <zinnia/spin.h>
-#include <zinnia/string.h>
+#include <string.h>
 
 typedef struct {
     size_t num_pages; // Amount of pages connected to this slab.

core/mem/vmo.c

@@ -1,7 +1,6 @@
 #include <zinnia/alloc.h>
 #include <zinnia/errno.h>
 #include <zinnia/list.h>
-#include <zinnia/string.h>
 #include <zinnia/utils.h>
 #include <zinnia/vmo.h>
 

core/meson.build

@@ -1,13 +1,15 @@
 kernel_sources += files(
     'clock.c',
     'console.c',
+    'fbcon.c',
+    'initgraph.c',
     'irq.c',
+    'ksyms.c',
     'module.c',
     'panic.c',
     'percpu.c',
     'print.c',
     'spin.c',
-    'wait.c',
 )
 
 subdir('lib')

core/module.c

@@ -1,12 +1,613 @@
+#define pr_fmt(fmt) "module: " fmt
+
+#include <zinnia/alloc.h>
 #include <zinnia/assert.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/elf.h>
+#include <zinnia/ksyms.h>
 #include <zinnia/module.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <bits/elf.h>
+#include <stdckdint.h>
+#include <string.h>
 
 extern errno_t (*__ld_mod_entry_start[])(struct module*);
 extern errno_t (*__ld_mod_entry_end[])(struct module*);
 
-void module_init() {
+static struct spinlock module_lock;
+static tailq_head(struct module) module_list;
+
+static bool in_range(size_t total, uint64_t offset, uint64_t size) {
+    return offset <= total && size <= total - offset;
+}
+
+static void* image_ptr(struct module* mod, uint64_t vaddr, uint64_t size) {
+    uintptr_t addr, end;
+    if (ckd_add(&addr, mod->bias, vaddr) || ckd_add(&end, addr, size))
+        return nullptr;
+
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
+        if (addr >= seg->virt && end <= seg->virt + seg->size)
+            return (void*)addr;
+    }
+    return nullptr;
+}
+
+static const char* image_cstr(struct module* mod, uint64_t vaddr, uint64_t size) {
+    const char* str = image_ptr(mod, vaddr, size);
+    if (str == nullptr || size == 0 || strnlen(str, size) >= size)
+        return nullptr;
+    return str;
+}
+
+static const char* table_str(const char* table, size_t size, uint64_t offset) {
+    if (offset >= size)
+        return nullptr;
+    const char* str = table + offset;
+    if (strnlen(str, size - offset) >= size - offset)
+        return nullptr;
+    return str;
+}
+
+static const char* sym_name(struct module* mod, uint64_t offset) {
+    return table_str(mod->strtab, mod->strtab_size, offset);
+}
+
+static struct module* find_locked(const char* name) {
+    for (struct module* m = tailq_first(&module_list); m != nullptr; m = tailq_next(m, link)) {
+        if (!strcmp(m->name, name))
+            return m;
+    }
+    return nullptr;
+}
+
+void module_init(struct boot_info*) {
+    spin_init(&module_lock);
+    tailq_init(&module_list);
+
     for (auto start = __ld_mod_entry_start; start < __ld_mod_entry_end; start += 1) {
         errno_t mod_status = (*start)(nullptr);
-        assert(mod_status == EOK, "Built-in entry point returned %i", mod_status);
+        assert(mod_status == 0, "Built-in entry point returned %i", mod_status);
     }
 }
+INIT_TASK(module_stage, "core.module", module_init);
+INIT_DEPENDS(module_stage, sched_stage, ksyms_stage);
+
+static errno_t resolve_symbol(struct module* mod, uint32_t index, uintptr_t* out) {
+    if (index >= mod->num_syms)
+        return ENOEXEC;
+
+    const elf_sym_t* sym = &((const elf_sym_t*)mod->symtab)[index];
+    if (sym->st_shndx != SHN_UNDEF) {
+        *out = mod->bias + sym->st_value;
+        return 0;
+    }
+
+    const char* name = sym_name(mod, sym->st_name);
+    if (name == nullptr)
+        return ENOEXEC;
+
+    if (ksym_find(name, out))
+        return 0;
+
+    if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
+        *out = 0;
+        return 0;
+    }
+
+    log_err("%s: Unresolved symbol \"%s\"\n", mod->name, name);
+    return ENOENT;
+}
+
+static errno_t store_word(struct module* mod, uint64_t vaddr, uintptr_t value) {
+    uintptr_t* where = image_ptr(mod, vaddr, sizeof(uintptr_t));
+    if (where == nullptr)
+        return ENOEXEC;
+    *where = value;
+    return 0;
+}
+
+static errno_t relocate_relative(struct module* mod, uint64_t vaddr) {
+    uintptr_t* where = image_ptr(mod, vaddr, sizeof(uintptr_t));
+    if (where == nullptr)
+        return ENOEXEC;
+    *where += mod->bias;
+    return 0;
+}
+
+static errno_t apply_rela(struct module* mod, uint64_t addr, uint64_t size, uint64_t entsize) {
+    if (entsize == 0)
+        entsize = sizeof(elf_rela_t);
+    if (entsize < sizeof(elf_rela_t))
+        return ENOEXEC;
+
+    const uint8_t* table = image_ptr(mod, addr, size);
+    if (table == nullptr)
+        return ENOEXEC;
+
+    for (uint64_t off = 0; off + sizeof(elf_rela_t) <= size; off += entsize) {
+        elf_rela_t rela;
+        memcpy(&rela, table + off, sizeof(rela));
+
+        errno_t status = 0;
+        uintptr_t value;
+        switch (ELF_R_TYPE(rela.r_info)) {
+        case R_NONE:
+            break;
+        case R_64:
+        case R_GLOB_DAT:
+        case R_JUMP_SLOT:
+            status = resolve_symbol(mod, ELF_R_SYM(rela.r_info), &value);
+            if (status == 0)
+                status = store_word(mod, rela.r_offset, value + rela.r_addend);
+            break;
+        case R_RELATIVE:
+            status = store_word(mod, rela.r_offset, mod->bias + rela.r_addend);
+            break;
+        default:
+            log_err("%s: Unsupported relocation type %u\n", mod->name, ELF_R_TYPE(rela.r_info));
+            status = ENOEXEC;
+            break;
+        }
+
+        if (status)
+            return status;
+    }
+    return 0;
+}
+
+static errno_t apply_relr(struct module* mod, uint64_t addr, uint64_t size, uint64_t entsize) {
+    if (entsize != 0 && entsize != sizeof(elf_relr_t))
+        return ENOEXEC;
+
+    const elf_relr_t* table = image_ptr(mod, addr, size);
+    if (table == nullptr)
+        return ENOEXEC;
+
+    uint64_t where = 0;
+    for (size_t i = 0; i < size / sizeof(elf_relr_t); i++) {
+        elf_relr_t entry = table[i];
+        errno_t status;
+
+        if ((entry & 1) == 0) {
+            status = relocate_relative(mod, entry);
+            if (status)
+                return status;
+            where = entry + sizeof(elf_relr_t);
+        } else {
+            for (size_t bit = 0; (entry >>= 1) != 0; bit++) {
+                if ((entry & 1) == 0)
+                    continue;
+                status = relocate_relative(mod, where + bit * sizeof(elf_relr_t));
+                if (status)
+                    return status;
+            }
+            where += (sizeof(elf_relr_t) * 8 - 1) * sizeof(elf_relr_t);
+        }
+    }
+    return 0;
+}
+
+static void module_free(struct module* mod) {
+    const size_t page_size = arch_mem_page_size();
+
+    mem_free(mod->cmdline);
+
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
+        for (size_t off = 0; off < seg->size; off += page_size)
+            pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+        mem_phys_free(seg->phys, seg->size / page_size);
+    }
+    mem_free(mod->segments);
+
+    if (mod->base != 0)
+        vm_free(mod->base, mod->size);
+
+    mem_free(mod);
+}
+
+static enum pte_flags prot_from_phdr(const elf_phdr_t* phdr) {
+    enum pte_flags prot = 0;
+    if (phdr->p_flags & PF_R)
+        prot |= PTE_READ;
+    if (phdr->p_flags & PF_W)
+        prot |= PTE_WRITE;
+    if (phdr->p_flags & PF_X)
+        prot |= PTE_EXEC;
+    return prot;
+}
+
+static errno_t read_headers(const void* data, size_t length, elf_ehdr_t* ehdr, elf_phdr_t** out) {
+    const uint8_t* file = data;
+
+    if (data == nullptr || length < sizeof(elf_ehdr_t))
+        return ENOEXEC;
+
+    memcpy(ehdr, file, sizeof(*ehdr));
+
+    if (memcmp(ehdr->e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0 || ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
+        ehdr->e_ident[EI_CLASS] != ELF_ARCH_CLASS || ehdr->e_ident[EI_DATA] != ELF_ARCH_DATA ||
+        ehdr->e_machine != ELF_ARCH_MACHINE || ehdr->e_type != ET_DYN)
+        return ENOEXEC;
+
+    if (ehdr->e_phentsize < sizeof(elf_phdr_t) || ehdr->e_phnum == 0 ||
+        !in_range(length, ehdr->e_phoff, (uint64_t)ehdr->e_phnum * ehdr->e_phentsize))
+        return ENOEXEC;
+
+    elf_phdr_t* phdrs = mem_alloc(ehdr->e_phnum * sizeof(elf_phdr_t), 0);
+    if (phdrs == nullptr)
+        return ENOMEM;
+    for (size_t i = 0; i < ehdr->e_phnum; i++)
+        memcpy(&phdrs[i], file + ehdr->e_phoff + i * ehdr->e_phentsize, sizeof(elf_phdr_t));
+
+    *out = phdrs;
+    return 0;
+}
+
+errno_t module_load(const void* data, size_t length, const char* cmdline, struct module** out) {
+    const uint8_t* file = data;
+    const size_t page_size = arch_mem_page_size();
+
+    elf_ehdr_t ehdr;
+    elf_phdr_t* phdrs;
+    errno_t header_status = read_headers(data, length, &ehdr, &phdrs);
+    if (header_status)
+        return header_status;
+
+    struct module* mod = mem_alloc(sizeof(struct module), 0);
+    if (mod == nullptr) {
+        mem_free(phdrs);
+        return ENOMEM;
+    }
+    mod->version = "";
+    mod->author = "";
+    mod->desc = "";
+    mod->license = "";
+
+    errno_t status = ENOEXEC;
+    bool listed = false;
+
+    uintptr_t load_min = UINTPTR_MAX;
+    uintptr_t load_end = 0;
+    size_t num_loads = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
+            continue;
+        if (phdr->p_filesz > phdr->p_memsz || !in_range(length, phdr->p_offset, phdr->p_filesz))
+            goto fail;
+
+        uintptr_t end;
+        if (ckd_add(&end, phdr->p_vaddr, phdr->p_memsz) || end > UINTPTR_MAX - page_size)
+            goto fail;
+
+        const uintptr_t start = ALIGN_DOWN(phdr->p_vaddr, page_size);
+        if (start < load_end)
+            goto fail;
+        if (load_min == UINTPTR_MAX)
+            load_min = start;
+        load_end = ALIGN_UP(end, page_size);
+        num_loads++;
+    }
+
+    if (num_loads == 0)
+        goto fail;
+
+    status = ENOMEM;
+    mod->segments = mem_alloc(num_loads * sizeof(struct module_segment), 0);
+    if (mod->segments == nullptr)
+        goto fail;
+
+    mod->base = vm_alloc(load_end - load_min);
+    if (mod->base == 0)
+        goto fail;
+    mod->size = load_end - load_min;
+    mod->bias = mod->base - load_min;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
+            continue;
+
+        const uintptr_t start = ALIGN_DOWN(phdr->p_vaddr, page_size);
+        const size_t size = ALIGN_UP(phdr->p_vaddr + phdr->p_memsz, page_size) - start;
+
+        phys_t phys;
+        status = mem_phys_alloc(size / page_size, 0, &phys);
+        if (status)
+            goto fail;
+
+        struct module_segment* seg = &mod->segments[mod->num_segments++];
+        *seg = (struct module_segment){.phys = phys, .virt = mod->bias + start, .size = size};
+
+        for (size_t off = 0; off < size; off += page_size) {
+            status = pmap_map(&kernel_vma.pmap, seg->virt + off, phys + off, PTE_READ | PTE_WRITE, CACHE_WRITE_BACK);
+            if (status)
+                goto fail;
+        }
+
+        memcpy((void*)(mod->bias + phdr->p_vaddr), file + phdr->p_offset, phdr->p_filesz);
+    }
+
+    status = ENOEXEC;
+
+    const elf_dyn_t* dyn = nullptr;
+    size_t num_dyn = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        if (phdrs[i].p_type != PT_DYNAMIC)
+            continue;
+        dyn = image_ptr(mod, phdrs[i].p_vaddr, phdrs[i].p_memsz);
+        num_dyn = phdrs[i].p_memsz / sizeof(elf_dyn_t);
+        break;
+    }
+    if (dyn == nullptr)
+        goto fail;
+
+    uint64_t dt_strtab = 0, dt_strsz = 0, dt_symtab = 0, dt_hash = 0, dt_soname = 0;
+    uint64_t dt_rela = 0, dt_relasz = 0, dt_relaent = 0;
+    uint64_t dt_jmprel = 0, dt_pltrelsz = 0, dt_pltrel = DT_RELA;
+    uint64_t dt_relr = 0, dt_relrsz = 0, dt_relrent = 0;
+    bool has_strtab = false, has_symtab = false, has_hash = false, has_soname = false;
+    bool has_rela = false, has_jmprel = false, has_relr = false, has_needed = false;
+
+    for (size_t i = 0; i < num_dyn && dyn[i].d_tag != DT_NULL; i++) {
+        const uint64_t val = dyn[i].d_un.d_val;
+        switch (dyn[i].d_tag) {
+        case DT_STRTAB:
+            dt_strtab = val;
+            has_strtab = true;
+            break;
+        case DT_STRSZ:
+            dt_strsz = val;
+            break;
+        case DT_SYMTAB:
+            dt_symtab = val;
+            has_symtab = true;
+            break;
+        case DT_HASH:
+            dt_hash = val;
+            has_hash = true;
+            break;
+        case DT_SONAME:
+            dt_soname = val;
+            has_soname = true;
+            break;
+        case DT_NEEDED:
+            has_needed = true;
+            break;
+        case DT_RELA:
+            dt_rela = val;
+            has_rela = true;
+            break;
+        case DT_RELASZ:
+            dt_relasz = val;
+            break;
+        case DT_RELAENT:
+            dt_relaent = val;
+            break;
+        case DT_JMPREL:
+            dt_jmprel = val;
+            has_jmprel = true;
+            break;
+        case DT_PLTRELSZ:
+            dt_pltrelsz = val;
+            break;
+        case DT_PLTREL:
+            dt_pltrel = val;
+            break;
+        case DT_RELR:
+            dt_relr = val;
+            has_relr = true;
+            break;
+        case DT_RELRSZ:
+            dt_relrsz = val;
+            break;
+        case DT_RELRENT:
+            dt_relrent = val;
+            break;
+        default:
+            break;
+        }
+    }
+
+    if (!has_strtab || !has_symtab || !has_hash || !has_soname || dt_strsz == 0 || dt_pltrel != DT_RELA)
+        goto fail;
+
+    const elf_hash_t* hash = image_ptr(mod, dt_hash, sizeof(elf_hash_t));
+    if (hash == nullptr)
+        goto fail;
+
+    mod->num_syms = hash->n_chain;
+    mod->symtab = image_ptr(mod, dt_symtab, (uint64_t)mod->num_syms * sizeof(elf_sym_t));
+    mod->strtab = image_ptr(mod, dt_strtab, dt_strsz);
+    mod->strtab_size = dt_strsz;
+    if (mod->symtab == nullptr || mod->strtab == nullptr) {
+        mod->num_syms = 0;
+        goto fail;
+    }
+
+    mod->name = sym_name(mod, dt_soname);
+    if (mod->name == nullptr || *mod->name == 0)
+        goto fail;
+
+    for (size_t i = 0; has_needed && i < num_dyn && dyn[i].d_tag != DT_NULL; i++) {
+        if (dyn[i].d_tag != DT_NEEDED)
+            continue;
+
+        // TODO: Modules can technically claim they have dependencies on other modules, but we don't support that yet.
+        const char* needed = sym_name(mod, dyn[i].d_un.d_val);
+        const char* kernel_name = ksym_kernel_name();
+        if (needed == nullptr || kernel_name == nullptr || strcmp(needed, kernel_name) != 0) {
+            log_err("%s: Modules can only depend on the kernel, not \"%s\"\n", mod->name, needed ? needed : "?");
+            goto fail;
+        }
+    }
+
+    if (has_relr) {
+        status = apply_relr(mod, dt_relr, dt_relrsz, dt_relrent);
+        if (status)
+            goto fail;
+    }
+    if (has_rela) {
+        status = apply_rela(mod, dt_rela, dt_relasz, dt_relaent);
+        if (status)
+            goto fail;
+    }
+    if (has_jmprel) {
+        status = apply_rela(mod, dt_jmprel, dt_pltrelsz, dt_relaent);
+        if (status)
+            goto fail;
+    }
+
+    spin_lock(&module_lock);
+    if (find_locked(mod->name) != nullptr) {
+        spin_unlock(&module_lock);
+        status = EEXIST;
+        goto fail;
+    }
+    tailq_insert_tail(&module_list, mod, link);
+    listed = true;
+    spin_unlock(&module_lock);
+
+    const struct cmdline_option* const* opts_start = nullptr;
+    const struct cmdline_option* const* opts_end = nullptr;
+
+    status = ENOEXEC;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        const char* str;
+        switch (phdr->p_type) {
+        case PT_MOD_VERSION:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->version = str;
+            break;
+        case PT_MOD_AUTHOR:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->author = str;
+            break;
+        case PT_MOD_DESC:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->desc = str;
+            break;
+        case PT_MOD_LICENSE:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->license = str;
+            break;
+        case PT_MOD_CMDLINE:
+            if (phdr->p_memsz < sizeof(void*))
+                break;
+            opts_start = image_ptr(mod, phdr->p_vaddr, phdr->p_memsz);
+            if (opts_start == nullptr)
+                goto fail;
+            opts_end = opts_start + (phdr->p_memsz / sizeof(void*));
+            break;
+        case PT_MOD_EXIT: {
+            if (phdr->p_memsz < sizeof(void*))
+                break;
+            const void* slot = image_ptr(mod, phdr->p_vaddr, sizeof(void*));
+            if (slot == nullptr)
+                goto fail;
+            memcpy(&mod->exit, slot, sizeof(void*));
+            break;
+        }
+        default:
+            break;
+        }
+    }
+
+    if (image_ptr(mod, ehdr.e_entry, 1) == nullptr)
+        goto fail;
+
+    size_t seg_idx = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
+            continue;
+
+        const struct module_segment* seg = &mod->segments[seg_idx++];
+        const enum pte_flags prot = prot_from_phdr(phdr);
+        for (size_t off = 0; off < seg->size; off += page_size) {
+            pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+            status = pmap_map(&kernel_vma.pmap, seg->virt + off, seg->phys + off, prot, CACHE_WRITE_BACK);
+            if (status)
+                goto fail;
+        }
+    }
+
+    if (cmdline == nullptr)
+        cmdline = "";
+    const size_t cmdline_len = strlen(cmdline);
+    mod->cmdline = mem_alloc(cmdline_len + 1, 0);
+    if (mod->cmdline == nullptr) {
+        status = ENOMEM;
+        goto fail;
+    }
+    memcpy(mod->cmdline, cmdline, cmdline_len);
+    if (opts_start != nullptr)
+        cmdline_parse_buf(mod->cmdline, cmdline_len, opts_start, opts_end);
+
+    log("Loaded module \"%s\" (%s, %s) at %p\n", mod->name, mod->desc, mod->version, (void*)mod->base);
+
+    errno_t (*entry)(struct module*) = (errno_t (*)(struct module*))(mod->bias + ehdr.e_entry);
+    status = entry(mod);
+    if (status) {
+        log_err("%s: Entry point returned %i\n", mod->name, status);
+        goto fail;
+    }
+
+    mem_free(phdrs);
+
+    if (out != nullptr)
+        *out = mod;
+    return 0;
+
+fail:
+    if (listed) {
+        spin_lock(&module_lock);
+        tailq_remove(&module_list, mod, link);
+        spin_unlock(&module_lock);
+    }
+    module_free(mod);
+    mem_free(phdrs);
+    return status;
+}
+
+errno_t module_unload(struct module* mod) {
+    if (mod == nullptr)
+        return EINVAL;
+
+    if (mod->exit == nullptr)
+        return ENOTSUP;
+
+    spin_lock(&module_lock);
+    tailq_remove(&module_list, mod, link);
+    spin_unlock(&module_lock);
+
+    errno_t status = mod->exit(mod);
+    if (status) {
+        log_err("%s: Exit function returned %i\n", mod->name, status);
+        spin_lock(&module_lock);
+        tailq_insert_tail(&module_list, mod, link);
+        spin_unlock(&module_lock);
+        return status;
+    }
+
+    log("Unloaded module \"%s\"\n", mod->name);
+    module_free(mod);
+    return 0;
+}
+
+struct module* module_find(const char* name) {
+    spin_lock(&module_lock);
+    struct module* mod = find_locked(name);
+    spin_unlock(&module_lock);
+    return mod;
+}

core/panic.c

@@ -1,11 +1,42 @@
+#include <zinnia/alloc.h>
 #include <zinnia/irq.h>
+#include <zinnia/ksyms.h>
 #include <zinnia/panic.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <stdarg.h>
+#include <stdint.h>
 
 #define panic_log(msg, ...) kprintf("[    !!!!    ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
 
+static void print_backtrace() {
+    constexpr size_t max_depth = 32;
+    void** fp = __builtin_frame_address(0);
+
+    panic_log("Backtrace:\n");
+    for (size_t i = 0; i < max_depth; i++) {
+        if (!fp || (uintptr_t)fp < mem_hhdm_base || (uintptr_t)fp % alignof(void*))
+            break;
+
+        void* ret = fp[1];
+        if (!ret)
+            break;
+
+        const char* name;
+        size_t offset;
+        // The return address may already point past the end of a noreturn call.
+        if (ksym_lookup((uintptr_t)ret - 1, &name, &offset))
+            panic_log("  #%zu [%p] %s+0x%zx\n", i, ret, name, offset + 1);
+        else
+            panic_log("  #%zu [%p] <\?\?\?>\n", i, ret);
+
+        void** next = fp[0];
+        if (next <= fp)
+            break;
+        fp = next;
+    }
+}
+
 [[noreturn]]
 void panic(const char* msg, ...) {
     // TODO: Stop all other CPUs with IPI.
@@ -15,12 +46,14 @@ void panic(const char* msg, ...) {
     struct task* current = cpu->sched.current;
     kprintf(
         "[    !!!!    ] \e[31m"
-        "Kernel panic on CPU %zu (T%zu, \"%s\"): ",
+        "Kernel panic on CPU %zu (T%zu, \"%s\"):\e[0m\n",
         cpu->id,
         current ? current->id : 0,
         current ? current->name : nullptr
     );
 
+    kprintf("[    !!!!    ] \e[31m");
+
     va_list args;
     va_start(args, msg);
     kvprintf(msg, args);
@@ -28,9 +61,10 @@ void panic(const char* msg, ...) {
 
     kprintf("\e[0m");
 
+    print_backtrace();
     panic_log("End of panic message\n");
 
-    while (1) {
+    while (true) {
         irq_wait();
     }
 }

core/percpu.c

@@ -2,7 +2,6 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/compiler.h>
 #include <zinnia/percpu.h>
-#include <zinnia/string.h>
 #include <zinnia/utils.h>
 #include <arch/percpu.h>
 #include <stdatomic.h>
@@ -55,6 +54,8 @@ void percpu_bsp_init() {
     arch_percpu_bsp_init();
 }
 
-void percpu_init() {
+void percpu_init(struct boot_info*) {
     arch_percpu_init();
 }
+INIT_TASK(percpu_stage, "core.percpu", percpu_init);
+INIT_DEPENDS(percpu_stage, mem_stage);

core/print.c

@@ -1,22 +1,22 @@
 #include <zinnia/clock.h>
 #include <zinnia/console.h>
 #include <zinnia/panic.h>
+#include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/stdlib.h>
-#include <zinnia/string.h>
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdint.h>
-
-typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
+#include <string.h>
 
 static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) {
     for (size_t i = 0; i < count; i++)
         callback(ctx, &c, 1);
 }
 
-static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
+void print_fmt(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
     // Amount of bytes written.
     int32_t written = 0;
 
@@ -320,16 +320,159 @@ void kprintf(const char* message, ...) {
 
     va_list args;
     va_start(args, message);
-    do_print(printf_console, nullptr, message, args);
+    print_fmt(printf_console, nullptr, message, args);
     va_end(args);
 
     spin_unlock(&print_lock);
 }
 
+struct snprintf_ctx {
+    char* buf;
+    size_t size;
+    size_t len;
+};
+
+static void snprintf_write(void* ctx, const char* str, size_t len) {
+    struct snprintf_ctx* c = ctx;
+    // Always keep room for the terminator.
+    if (c->len < c->size) {
+        size_t room = c->size - c->len - 1;
+        memcpy(c->buf + c->len, str, len < room ? len : room);
+    }
+    c->len += len;
+}
+
+size_t kvsnprintf(char* buf, size_t size, const char* format, va_list args) {
+    struct snprintf_ctx ctx = {.buf = buf, .size = size};
+    print_fmt(snprintf_write, &ctx, format, args);
+    if (size > 0)
+        buf[ctx.len < size ? ctx.len : size - 1] = '\0';
+    return ctx.len;
+}
+
+size_t ksnprintf(char* buf, size_t size, const char* format, ...) {
+    va_list args;
+    va_start(args, format);
+    size_t len = kvsnprintf(buf, size, format, args);
+    va_end(args);
+    return len;
+}
+
 void kvprintf(const char* message, va_list args) {
     spin_lock(&print_lock);
 
-    do_print(printf_console, nullptr, message, args);
+    print_fmt(printf_console, nullptr, message, args);
 
     spin_unlock(&print_lock);
 }
+
+struct log_record {
+    uint64_t time_ns;
+    uint32_t tid;
+    uint16_t len;
+    uint8_t level;
+};
+
+#define LOG_BUFFER_SIZE (64 * 1024)
+#define LOG_MESSAGE_MAX 255
+
+static uint8_t log_buffer[LOG_BUFFER_SIZE];
+
+// Protected by print_lock.
+static size_t log_head = 0;
+static size_t log_tail = 0;
+
+static void log_ring_copy_in(const void* src, size_t len) {
+    for (size_t i = 0; i < len; i++)
+        log_buffer[(log_head + i) % LOG_BUFFER_SIZE] = ((const uint8_t*)src)[i];
+    log_head += len;
+}
+
+static void log_ring_peek(size_t offset, void* dst, size_t len) {
+    for (size_t i = 0; i < len; i++)
+        ((uint8_t*)dst)[i] = log_buffer[(offset + i) % LOG_BUFFER_SIZE];
+}
+
+static void log_store(const struct log_record* rec, const char* msg) {
+    const size_t total = sizeof(*rec) + rec->len;
+
+    // Drop the oldest records until the new one fits.
+    while (LOG_BUFFER_SIZE - (log_head - log_tail) < total) {
+        struct log_record old;
+        log_ring_peek(log_tail, &old, sizeof(old));
+        log_tail += sizeof(old) + old.len;
+    }
+
+    log_ring_copy_in(rec, sizeof(*rec));
+    log_ring_copy_in(msg, rec->len);
+}
+
+struct log_line {
+    char buf[LOG_MESSAGE_MAX];
+    size_t len;
+};
+
+static void log_line_append(void* ctx, const char* str, size_t len) {
+    struct log_line* line = ctx;
+    if (len > sizeof(line->buf) - line->len)
+        len = sizeof(line->buf) - line->len;
+    memcpy(line->buf + line->len, str, len);
+    line->len += len;
+}
+
+// ANSI colors for each level.
+static const char* const log_level_colors[] = {
+    [LOG_DEBUG] = "\e[90m",
+    [LOG_STATUS] = "\e[32m",
+    [LOG_INFO] = "",
+    [LOG_WARN] = "\e[33m",
+    [LOG_ERR] = "\e[31m",
+    [LOG_PANIC] = "\e[1;31m",
+};
+
+static void log_console_printf(const char* fmt, ...) {
+    va_list args;
+    va_start(args, fmt);
+    print_fmt(printf_console, nullptr, fmt, args);
+    va_end(args);
+}
+
+void log_vwrite(enum log_level level, const char* message, va_list args) {
+    log_vwrite_prefixed(level, "", message, args);
+}
+
+void log_vwrite_prefixed(enum log_level level, const char* prefix, const char* message, va_list args) {
+    struct log_line line = {};
+    log_line_append(&line, prefix, strlen(prefix));
+    print_fmt(log_line_append, &line, message, args);
+    if (line.len > 0 && line.buf[line.len - 1] == '\n')
+        line.len--;
+
+    auto cur = percpu_get()->sched.current;
+    const struct log_record rec = {
+        .time_ns = clock_get_elapsed_ns(),
+        .tid = cur ? cur->id : 0,
+        .len = line.len,
+        .level = level,
+    };
+
+    spin_lock(&print_lock);
+    log_store(&rec, line.buf);
+    log_console_printf(
+        "[%5lu.%06lu] [%5lu] %s",
+        rec.time_ns / 1'000'000'000,
+        rec.time_ns / 1'000 % 1'000'000,
+        (uint64_t)rec.tid,
+        log_level_colors[level]
+    );
+    console_write(line.buf, line.len);
+    console_write("\e[0m\n", 5);
+    spin_unlock(&print_lock);
+}
+
+void log_write(enum log_level level, const char* message, ...) {
+    va_list args;
+    va_start(args, message);
+    log_vwrite(level, message, args);
+    va_end(args);
+}

core/sched/sched_dumb.c

@@ -9,7 +9,6 @@
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/stack.h>
-#include <zinnia/string.h>
 #include <zinnia/tailq.h>
 #include <zinnia/vma.h>
 #include <arch/sched.h>
@@ -54,6 +53,7 @@ void sched_init(struct scheduler* sched) {
 
 void sched_add_task(struct scheduler* sched, struct task* task) {
     assert(task != nullptr, "Tried to add NULL to the run queue\n");
+    task->state = TASK_STATE_READY;
     tailq_insert_tail(&sched->run_queue, task, next);
 }
 
@@ -64,6 +64,7 @@ static struct task* next_task(struct scheduler* sched) {
         return sched->idle_task;
 
     tailq_remove(&sched->run_queue, t, next);
+    t->state = TASK_STATE_RUNNING;
 
     return t;
 }
@@ -110,6 +111,25 @@ void sched_yield(struct scheduler* sched) {
     do_reschedule(sched);
 }
 
+void sched_block(struct scheduler* sched) {
+    irq_lock();
+    if (atomic_exchange_explicit(&sched->current->wake_pending, false, memory_order_acq_rel)) {
+        irq_unlock();
+        return;
+    }
+    sched->current->state = TASK_STATE_BLOCKED;
+    do_reschedule(sched);
+}
+
+void sched_wake(struct task* task) {
+    irq_lock();
+    if (task->state == TASK_STATE_BLOCKED)
+        sched_add_task(&percpu_get()->sched, task);
+    else
+        atomic_store_explicit(&task->wake_pending, true, memory_order_release);
+    irq_unlock();
+}
+
 errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
     if (!out)
         return EINVAL;
@@ -118,7 +138,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
     if (new_task == nullptr)
         return ENOMEM;
 
-    static _Atomic(size_t) last_tid = 0;
+    static atomic_size_t last_tid = 0;
     new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire);
     new_task->space = space;
     strncpy(new_task->name, name, sizeof(new_task->name));
@@ -165,3 +185,10 @@ void task_entry(task_fn_t entry, uintptr_t arg0, struct task* prev) {
     sched_yield(sched);
     panic("unreachable");
 }
+
+[[__init]]
+static void sched_stage_init(struct boot_info*) {
+    sched_init(&percpu_get()->sched);
+}
+INIT_TASK(sched_stage, "core.sched", sched_stage_init);
+INIT_DEPENDS(sched_stage, percpu_stage);

dev/acpi/include/zinnia/acpi.h

@@ -1,8 +1,17 @@
 #pragma once
 
 #include <zinnia/init.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/types.h>
+#include <stdint.h>
 
-void acpi_init(struct boot_info* info);
+INIT_EXTERN(acpi_stage);
 
 extern phys_t acpi_rsdp_addr;
+
+struct acpi_match {
+    const char* hid;
+    uintptr_t data;
+};
+
+#define ACPI_MATCH_HID(id) .hid = (id)

dev/acpi/init.c

@@ -1,25 +1,45 @@
+#define pr_fmt(fmt) "acpi: " fmt
+
 #include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <zinnia/types.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <uacpi/uacpi.h>
 
 phys_t acpi_rsdp_addr = 0;
 
+static char table_buf[0x1000];
+
 static bool use_acpi = true;
 void use_acpi_option(bool use) {
     use_acpi = use;
 }
 CMDLINE_OPTION("acpi", "Enable ACPI firmware subsystem", use_acpi_option);
 
-void acpi_init(struct boot_info* info) {
+[[__init]]
+static bool acpi_stage_init(struct boot_info* info) {
     if (!use_acpi) {
-        log("acpi: ACPI disabled on the command line!\n");
-        return;
+        log_warn("ACPI disabled on the command line!\n");
+        return false;
     }
 
     if (!info->rsdp) {
-        log("acpi: No RSDP was passed, unable to start ACPI subsystem!\n");
-        return;
+        log_warn("No RSDP was passed, unable to start ACPI subsystem!\n");
+        return false;
     }
     acpi_rsdp_addr = info->rsdp;
+
+    if (uacpi_setup_early_table_access(table_buf, sizeof(table_buf)) != UACPI_STATUS_OK) {
+        log_err("Failed to set up early table access!\n");
+        return false;
+    }
+
+    return true;
 }
+INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
+INIT_DEPENDS(acpi_stage, mem_stage);
+INIT_ENTAILS(acpi_stage, percpu_stage);

dev/acpi/mcfg.c

@@ -0,0 +1,32 @@
+#define pr_fmt(fmt) "acpi: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/pci.h>
+#include <zinnia/print.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+
+[[__init]]
+static void acpi_mcfg_stage_init(struct boot_info*) {
+    uacpi_table table = {};
+    if (uacpi_table_find_by_signature(ACPI_MCFG_SIGNATURE, &table) != UACPI_STATUS_OK)
+        return;
+
+    struct acpi_mcfg* mcfg = table.ptr;
+    size_t count = 0;
+    if (mcfg->hdr.length > sizeof(*mcfg))
+        count = (mcfg->hdr.length - sizeof(*mcfg)) / sizeof(struct acpi_mcfg_allocation);
+
+    for (size_t i = 0; i < count; i++) {
+        struct acpi_mcfg_allocation* entry = &mcfg->entries[i];
+        errno_t status = pci_ecam_register(entry->address, entry->segment, entry->start_bus, entry->end_bus);
+        if (status != 0)
+            log_warn("Failed to register ECAM for segment %hu: %i\n", entry->segment, status);
+    }
+
+    uacpi_table_unref(&table);
+}
+INIT_TASK(acpi_mcfg_stage, "dev.acpi.mcfg", acpi_mcfg_stage_init);
+INIT_DEPENDS(acpi_mcfg_stage, acpi_stage);
+INIT_ENTAILS(acpi_mcfg_stage, pci_stage);

dev/acpi/meson.build

@@ -6,6 +6,10 @@ kernel_sources += files(
     'uacpi_glue.c',
 )
 
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('mcfg.c')
+endif
+
 kernel_includes += uacpi.get_variable('includes')
 kernel_includes += include_directories('include')
 

dev/acpi/uacpi_glue.c

@@ -38,17 +38,29 @@ void uacpi_kernel_unmap(void* addr, uacpi_size len) {
     // TODO
 }
 
-void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
-    log("acpi: ");
+static enum log_level acpi_log_level(uacpi_log_level lvl) {
+    switch (lvl) {
+    case UACPI_LOG_ERROR:
+        return LOG_ERR;
 
+    case UACPI_LOG_WARN:
+        return LOG_WARN;
+
+    case UACPI_LOG_INFO:
+        return LOG_INFO;
+
+    default:
+        return LOG_DEBUG;
+    }
+}
+
+void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
     va_list args;
     va_start(args, msg);
-
-    kvprintf(msg, args);
-
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
     va_end(args);
 }
 
 void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
-    kvprintf(msg, args);
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
 }

dev/block/nvme/core.c

@@ -1,16 +1,34 @@
-#include <zinnia/cmdline.h>
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/fdt.h>
 #include <zinnia/module.h>
+#include <zinnia/pci.h>
 #include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <config.h>
+
+#if CONFIG_NVME_PCI
+#include "nvme_pci.h"
+#endif
+
+struct nvme_dev {
+#if CONFIG_NVME_PCI
+    struct pci_device pci_dev;
+#endif
+};
 
 errno_t nvme_init(struct module* mod) {
-    log("Hello world from NVMe!\n");
+    errno_t status = 0;
 
-    return 0;
+#if CONFIG_NVME_PCI
+    status = pci_driver_register(&nvme_driver);
+#endif
+
+    return status;
 }
 
 errno_t nvme_exit(struct module* mod) {
-    log("Good bye from NVMe!\n");
-
+    // TODO: Unregister
     return 0;
 }
 

dev/block/nvme/nvme_pci.h

@@ -0,0 +1,3 @@
+#pragma once
+
+extern struct pci_driver nvme_driver;

dev/block/nvme/pci.c

@@ -0,0 +1,23 @@
+#include <zinnia/errno.h>
+#include <zinnia/module.h>
+#include <zinnia/pci.h>
+
+static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
+    pci_log(dev, "Detected controller \n");
+
+    return 0;
+}
+
+static const struct pci_match nvme_match[] = {
+    // Generic NVMe device. Assume no quirks or any funny business.
+    {PCI_MATCH_CLASS(1, 8, 2), .data = 0},
+};
+
+MATCH_TABLE("pci", nvme_match);
+
+struct pci_driver nvme_driver = {
+    .name = "nvme_pci",
+    .match = nvme_match,
+    .num_matches = ARRAY_SIZE(nvme_match),
+    .probe = nvme_pci_probe,
+};

dev/dt/fdt.c

@@ -0,0 +1,235 @@
+#include <zinnia/fdt.h>
+#include <string.h>
+
+uint32_t fdt_read_u32(const void* data) {
+    const uint8_t* p = data;
+    return (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | p[3];
+}
+
+uint64_t fdt_read_u64(const void* data) {
+    const uint8_t* p = data;
+    return (uint64_t)fdt_read_u32(p) << 32 | fdt_read_u32(p + 4);
+}
+
+static bool range_valid(size_t offset, size_t length, size_t total) {
+    return offset <= total && length <= total - offset;
+}
+
+struct token {
+    uint32_t kind;
+    size_t next;
+    const char* name;
+    const void* data;
+    size_t length;
+};
+
+static bool read_token(struct fdt* tree, size_t offset, struct token* token) {
+    if (!range_valid(offset, 4, tree->structs_len) || offset % 4)
+        return false;
+    *token = (struct token){.kind = fdt_read_u32(tree->structs + offset), .next = offset + 4};
+    if (token->kind == FDT_BEGIN_NODE) {
+        token->name = (const char*)tree->structs + token->next;
+        size_t left = tree->structs_len - token->next;
+        size_t length = strnlen(token->name, left);
+        if (length == left)
+            return false;
+        token->next += (length + 1 + 3) & ~(size_t)3;
+    } else if (token->kind == FDT_PROP) {
+        if (!range_valid(token->next, 8, tree->structs_len))
+            return false;
+        token->length = fdt_read_u32(tree->structs + token->next);
+        size_t name_offset = fdt_read_u32(tree->structs + token->next + 4);
+        token->next += 8;
+        if (name_offset >= tree->strings_len || !range_valid(token->next, token->length, tree->structs_len))
+            return false;
+        token->name = (const char*)tree->strings + name_offset;
+        size_t left = tree->strings_len - name_offset;
+        if (strnlen(token->name, left) == left || !*token->name)
+            return false;
+        token->data = tree->structs + token->next;
+        token->next += (token->length + 3) & ~(size_t)3;
+    } else if (token->kind != FDT_END_NODE && token->kind != FDT_NOP && token->kind != FDT_END) {
+        return false;
+    }
+    return token->next <= tree->structs_len;
+}
+
+bool fdt_init(struct fdt* tree, const void* data, size_t length) {
+    if (!tree)
+        return false;
+    *tree = (struct fdt){};
+    if (!data || length < sizeof(struct fdt_header))
+        return false;
+    const uint8_t* blob = data;
+    size_t total = fdt_read_u32(blob + 4);
+    size_t structs = fdt_read_u32(blob + 8);
+    size_t strings = fdt_read_u32(blob + 12);
+    size_t reserves = fdt_read_u32(blob + 16);
+    size_t strings_len = fdt_read_u32(blob + 32);
+    size_t structs_len = fdt_read_u32(blob + 36);
+    uint32_t version = fdt_read_u32(blob + 20);
+    uint32_t compat = fdt_read_u32(blob + 24);
+    if (fdt_read_u32(blob) != FDT_MAGIC || total > length || total < sizeof(struct fdt_header) || version < 17 ||
+        compat > 17 || compat > version || structs % 4 || reserves % 8 || structs < sizeof(struct fdt_header) ||
+        strings < sizeof(struct fdt_header) || reserves < sizeof(struct fdt_header) ||
+        !range_valid(structs, structs_len, total) || !range_valid(strings, strings_len, total) ||
+        (structs < strings + strings_len && strings < structs + structs_len))
+        return false;
+    size_t reserved_end = reserves;
+    for (;;) {
+        if (!range_valid(reserved_end, 16, total))
+            return false;
+        bool end = !fdt_read_u64(blob + reserved_end) && !fdt_read_u64(blob + reserved_end + 8);
+        reserved_end += 16;
+        if (end)
+            break;
+    }
+    if ((reserves < structs + structs_len && structs < reserved_end) ||
+        (reserves < strings + strings_len && strings < reserved_end))
+        return false;
+    struct fdt candidate = {
+        .blob = blob,
+        .total_size = total,
+        .version = version,
+        .compat_version = compat,
+        .boot_cpuid = fdt_read_u32(blob + 28),
+        .structs = blob + structs,
+        .structs_len = structs_len,
+        .strings = blob + strings,
+        .strings_len = strings_len,
+    };
+    size_t depth = 0;
+    bool started = false;
+    bool children[64] = {};
+    for (size_t offset = 0; offset < structs_len;) {
+        struct token token;
+        if (!read_token(&candidate, offset, &token))
+            return false;
+        if (token.kind == FDT_BEGIN_NODE) {
+            if (depth == 64 || (!depth && (started || *token.name)) || (depth && !*token.name))
+                return false;
+            if (depth)
+                children[depth - 1] = true;
+            children[depth++] = false;
+            started = true;
+        } else if (token.kind == FDT_END_NODE) {
+            if (!depth)
+                return false;
+            depth--;
+        } else if (token.kind == FDT_PROP) {
+            if (!depth || children[depth - 1])
+                return false;
+        } else if (token.kind == FDT_END) {
+            if (!started || depth || token.next != structs_len)
+                return false;
+            *tree = candidate;
+            return true;
+        }
+        offset = token.next;
+    }
+    return false;
+}
+
+struct fdt_node fdt_root(struct fdt* tree) {
+    struct token token;
+    size_t offset = 0;
+    if (tree && tree->blob) {
+        while (read_token(tree, offset, &token)) {
+            if (token.kind == FDT_BEGIN_NODE)
+                return (struct fdt_node){tree, offset};
+            if (token.kind != FDT_NOP)
+                break;
+            offset = token.next;
+        }
+    }
+    return (struct fdt_node){};
+}
+
+static struct fdt_node find_child(struct fdt_node node, const char* name, size_t length) {
+    struct token token;
+    if (!node.tree || !read_token(node.tree, node.offset, &token) || token.kind != FDT_BEGIN_NODE)
+        return (struct fdt_node){};
+    size_t depth = 0;
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
+        if (token.kind == FDT_BEGIN_NODE) {
+            if (!depth && strlen(token.name) == length && !memcmp(token.name, name, length))
+                return (struct fdt_node){node.tree, offset};
+            depth++;
+        } else if (token.kind == FDT_END_NODE) {
+            if (!depth)
+                break;
+            depth--;
+        }
+    }
+    return (struct fdt_node){};
+}
+
+struct fdt_node fdt_find(struct fdt* tree, const char* path) {
+    if (!path || *path != '/')
+        return (struct fdt_node){};
+    struct fdt_node node = fdt_root(tree);
+    path++;
+    while (*path && node.tree) {
+        size_t length = 0;
+        while (path[length] && path[length] != '/')
+            length++;
+        if (!length)
+            return (struct fdt_node){};
+        node = find_child(node, path, length);
+        path += length;
+        if (*path == '/')
+            path++;
+    }
+    return node;
+}
+
+struct fdt_prop fdt_property(struct fdt_node node, const char* name) {
+    struct token token;
+    if (!node.tree || !name || !read_token(node.tree, node.offset, &token) || token.kind != FDT_BEGIN_NODE)
+        return (struct fdt_prop){};
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
+        if (token.kind == FDT_PROP && !strcmp(token.name, name))
+            return (struct fdt_prop){node.tree, token.data, token.length};
+        if (token.kind == FDT_BEGIN_NODE || token.kind == FDT_END_NODE)
+            break;
+    }
+    return (struct fdt_prop){};
+}
+
+bool fdt_property_u32(struct fdt_node node, const char* name, uint32_t* value) {
+    struct fdt_prop property = fdt_property(node, name);
+    if (!property.tree || property.length != 4 || !value)
+        return false;
+    *value = fdt_read_u32(property.data);
+    return true;
+}
+
+bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value) {
+    struct fdt_prop property = fdt_property(node, name);
+    if (!property.tree || property.length != 8 || !value)
+        return false;
+    *value = fdt_read_u64(property.data);
+    return true;
+}
+
+struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
+    if (!tree || !tree->blob || !compatible)
+        return (struct fdt_node){};
+    struct token token;
+    for (size_t offset = 0; read_token(tree, offset, &token); offset = token.next) {
+        if (token.kind != FDT_BEGIN_NODE)
+            continue;
+        struct fdt_node node = {tree, offset};
+        struct fdt_prop property = fdt_property(node, "compatible");
+        const char* data = property.data;
+        for (size_t i = 0; i < property.length;) {
+            size_t length = strnlen(data + i, property.length - i);
+            if (length == property.length - i)
+                break;
+            if (!strcmp(data + i, compatible))
+                return node;
+            i += length + 1;
+        }
+    }
+    return (struct fdt_node){};
+}

dev/dt/meson.build

@@ -0,0 +1 @@
+kernel_sources += files('fdt.c')

dev/meson.build

@@ -2,6 +2,10 @@ subdir('common')
 subdir('block')
 subdir('dt')
 
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    subdir('pci')
+endif
+
 if kconfig.get('CONFIG_ACPI', 'n') == 'y'
     subdir('acpi')
 endif

dev/module.ld

@@ -1,5 +1,7 @@
 /* Linker script for kernel modules */
 
+ENTRY(__module_entry)
+
 PHDRS
 {
     text        PT_LOAD     FLAGS(5);
@@ -11,8 +13,8 @@ PHDRS
     moddesc     0x60000003;
     modlicense  0x60000004;
     modcmdline  0x60000005;
-    modentry    0x60000006;
-    modexit     0x60000007;
+    modexit     0x60000006;
+    modmatch    0x60000007;
 }
 
 SECTIONS
@@ -57,14 +59,14 @@ SECTIONS
         KEEP(*(.cmdline))
     } :rodata :modcmdline
 
-    .mod.entry : {
-        KEEP(*(.mod.entry))
-    } :rodata :modentry
-
     .mod.exit : {
         KEEP(*(.mod.exit))
     } :rodata :modexit
 
+    .mod.match : {
+        KEEP(*(.mod.match))
+    } :rodata :modmatch
+
     . = ALIGN(CONSTANT(MAXPAGESIZE));
 
     /* Read-Write data */
@@ -91,6 +93,10 @@ SECTIONS
         *(.rela.dyn .rela.dyn.*)
     } :data
 
+    .relr.dyn : {
+        *(.relr.dyn .relr.dyn.*)
+    } :data
+
     /* Dynamic tags */
     .dynamic : {
         *(.dynamic)
@@ -101,7 +107,6 @@ SECTIONS
         *(.comment*)
         *(.eh_frame*)
         *(.rel.eh_frame*)
-        *(.relr.dyn .relr.dyn.*)
         *(.gcc_except_table*)
         *(.note*)
         *(.interp)

dev/pci/core.c

@@ -0,0 +1,376 @@
+#define pr_fmt(fmt) "pci: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/init.h>
+#include <zinnia/module.h>
+#include <zinnia/pci.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <string.h>
+
+static struct spinlock access_lock = {};
+static vec(struct pci_access*) accesses = {};
+
+static struct spinlock device_lock = {};
+static vec(struct pci_device*) devices = {};
+static vec(struct pci_driver*) drivers = {};
+
+errno_t pci_access_register(struct pci_access* access) {
+    spin_lock(&access_lock);
+    errno_t status = vec_push(&accesses, access);
+    spin_unlock(&access_lock);
+    return status;
+}
+
+static bool pci_access_decodes(const struct pci_access* access, const struct pci_address* addr) {
+    return access->domain == addr->domain && addr->bus >= access->start_bus && addr->bus <= access->end_bus;
+}
+
+struct pci_access* pci_access_find(const struct pci_address* addr) {
+    struct pci_access* found = nullptr;
+
+    spin_lock(&access_lock);
+    vec_foreach(&accesses, i) {
+        struct pci_access* access = vec_elem_get(&accesses, i);
+        if (!pci_access_decodes(access, addr))
+            continue;
+        if (found == nullptr || access->priority > found->priority)
+            found = access;
+    }
+    spin_unlock(&access_lock);
+
+    return found;
+}
+
+static uint32_t cfg_read32(struct pci_access* access, const struct pci_address* addr, uint16_t offset) {
+    return access->read32(access, addr, offset & ~3);
+}
+
+static void cfg_write32(struct pci_access* access, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    access->write32(access, addr, offset & ~3, value);
+}
+
+uint32_t pci_read32(struct pci_device* dev, uint16_t offset) {
+    return cfg_read32(dev->access, &dev->address, offset);
+}
+
+uint16_t pci_read16(struct pci_device* dev, uint16_t offset) {
+    return pci_read32(dev, offset) >> ((offset & 2) * 8);
+}
+
+uint8_t pci_read8(struct pci_device* dev, uint16_t offset) {
+    return pci_read32(dev, offset) >> ((offset & 3) * 8);
+}
+
+void pci_write32(struct pci_device* dev, uint16_t offset, uint32_t value) {
+    cfg_write32(dev->access, &dev->address, offset, value);
+}
+
+void pci_write16(struct pci_device* dev, uint16_t offset, uint16_t value) {
+    uint32_t shift = (offset & 2) * 8;
+    uint32_t reg = pci_read32(dev, offset);
+    reg &= ~(0xffffu << shift);
+    reg |= (uint32_t)value << shift;
+    pci_write32(dev, offset, reg);
+}
+
+void pci_write8(struct pci_device* dev, uint16_t offset, uint8_t value) {
+    uint32_t shift = (offset & 3) * 8;
+    uint32_t reg = pci_read32(dev, offset);
+    reg &= ~(0xffu << shift);
+    reg |= (uint32_t)value << shift;
+    pci_write32(dev, offset, reg);
+}
+
+errno_t pci_bar_get(struct pci_device* dev, size_t index, struct pci_bar* out) {
+    if (index >= PCI_NUM_BARS)
+        return EINVAL;
+
+    uint16_t bar_offset = PCI_BAR0 + index * sizeof(uint32_t);
+    uint16_t high_offset = bar_offset + sizeof(uint32_t);
+    uint32_t bar = pci_read32(dev, bar_offset);
+
+    bool is_mmio = (bar & 0x1) == 0;
+    bool is_mmio64 = is_mmio && ((bar >> 1) & 0x3) == 0x2;
+    bool is_prefetchable = is_mmio && (bar & (1 << 3));
+    bool is_64bit_pair = is_mmio64 && index + 1 < PCI_NUM_BARS;
+
+    uint16_t command = pci_read16(dev, PCI_COMMAND);
+    pci_write16(dev, PCI_COMMAND, command & ~(PCI_COMMAND_IO | PCI_COMMAND_MEM));
+
+    uint32_t high = is_64bit_pair ? pci_read32(dev, high_offset) : 0;
+
+    pci_write32(dev, bar_offset, 0xffff'ffff);
+    if (is_64bit_pair)
+        pci_write32(dev, high_offset, 0xffff'ffff);
+
+    uint32_t mask_low = pci_read32(dev, bar_offset);
+    uint32_t mask_high = is_64bit_pair ? pci_read32(dev, high_offset) : 0;
+
+    pci_write32(dev, bar_offset, bar);
+    if (is_64bit_pair)
+        pci_write32(dev, high_offset, high);
+
+    pci_write16(dev, PCI_COMMAND, command);
+
+    uint64_t addr_mask = is_mmio ? ~0xfull : ~0x3ull;
+    uint64_t raw = (uint64_t)mask_high << 32 | mask_low;
+    uint64_t masked = raw & addr_mask & (is_64bit_pair ? UINT64_MAX : UINT32_MAX);
+
+    struct pci_bar result = {
+        .size = masked == 0 ? 0 : (size_t)1 << __builtin_ctzll(masked),
+        .prefetchable = is_prefetchable,
+    };
+
+    if (is_mmio && is_64bit_pair) {
+        result.type = PCI_BAR_MMIO64;
+        result.address = (phys_t)high << 32 | (bar & 0xffff'fff0);
+    } else if (is_mmio) {
+        result.type = PCI_BAR_MMIO32;
+        result.address = bar & 0xffff'fff0;
+    } else {
+        result.type = PCI_BAR_IO;
+        result.address = (uint16_t)(bar & 0xffff'fffc);
+    }
+
+    if (result.address == 0)
+        return ENOENT;
+
+    *out = result;
+    return 0;
+}
+
+uint8_t pci_cap_next(struct pci_device* dev, uint8_t cur) {
+    uint8_t ptr;
+    if (cur == 0) {
+        if (!(pci_read16(dev, PCI_STATUS) & PCI_STATUS_CAP_LIST))
+            return 0;
+        ptr = pci_read8(dev, PCI_CAP_PTR) & ~3;
+    } else {
+        ptr = (pci_read32(dev, cur) >> 8) & ~3;
+    }
+
+    if (ptr == 0)
+        return 0;
+
+    if ((pci_read32(dev, ptr) & 0xff) == 0xff)
+        return 0;
+
+    return ptr;
+}
+
+uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) {
+    for (uint8_t cap = pci_cap_next(dev, 0); cap != 0; cap = pci_cap_next(dev, cap)) {
+        if (pci_read8(dev, cap) == id)
+            return cap;
+    }
+    return 0;
+}
+
+static void pci_device_name(const struct pci_device* dev, char buf[static 18]) {
+    ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
+}
+
+static bool scan_function(struct pci_access* access, struct pci_address addr) {
+    uint32_t reg0 = cfg_read32(access, &addr, PCI_VENDOR_ID);
+    uint16_t vendor_id = reg0;
+    uint16_t device_id = reg0 >> 16;
+
+    if (vendor_id == 0xffff || vendor_id == 0)
+        return false;
+    if (device_id == 0xffff || device_id == 0)
+        return false;
+
+    struct pci_device* dev = mem_alloc(sizeof(*dev), 0);
+    if (dev == nullptr) {
+        log_err("Out of memory while scanning " PCI_ADDR_FMT "\n", PCI_ADDR_ARGS(&addr));
+        return false;
+    }
+
+    dev->address = addr;
+    dev->access = access;
+    dev->id.vendor_id = vendor_id;
+    dev->id.device_id = device_id;
+    dev->class.prog_if = pci_read8(dev, PCI_PROG_IF);
+    dev->class.subclass = pci_read8(dev, PCI_SUBCLASS);
+    dev->class.class = pci_read8(dev, PCI_CLASS);
+    dev->header_type = pci_read8(dev, PCI_HEADER_TYPE);
+
+    if ((dev->header_type & PCI_HEADER_TYPE_MASK) == 0) {
+        dev->id.subvendor_id = pci_read16(dev, PCI_SUBSYS_VENDOR_ID);
+        dev->id.subdevice_id = pci_read16(dev, PCI_SUBSYS_ID);
+    }
+
+    pci_log(
+        dev,
+        "<id = %.4hx:%.4hx, class = %.2hhx:%.2hhx:%.2hhx>\n",
+        dev->id.vendor_id,
+        dev->id.device_id,
+        dev->class.class,
+        dev->class.subclass,
+        dev->class.prog_if
+    );
+
+    if (vec_push(&devices, dev) != 0) {
+        mem_free(dev);
+        return false;
+    }
+
+    return true;
+}
+
+static void scan_slot(struct pci_access* access, struct pci_address addr) {
+    if (!scan_function(access, addr))
+        return;
+
+    if (!(vec_elem_get(&devices, vec_length(&devices) - 1)->header_type & PCI_HEADER_TYPE_MULTIFUNCTION))
+        return;
+
+    for (addr.function = 1; addr.function < PCI_NUM_FUNCTIONS; addr.function++)
+        scan_function(access, addr);
+}
+
+static void format_match_id(char buf[static 5], uint16_t value) {
+    if (value == PCI_ANY_ID)
+        ksnprintf(buf, 5, "****");
+    else
+        ksnprintf(buf, 5, "%.4hx", value);
+}
+
+static void format_match_class(char buf[static 3], uint8_t value) {
+    if (value == PCI_ANY_CLASS)
+        ksnprintf(buf, 3, "**");
+    else
+        ksnprintf(buf, 3, "%.2hhx", value);
+}
+
+static void format_match(const struct pci_match* match, char* buf, size_t size) {
+    char vendor[5], device[5], class[3], subclass[3], prog_if[3];
+    format_match_id(vendor, match->vendor_id);
+    format_match_id(device, match->device_id);
+    format_match_class(class, match->class);
+    format_match_class(subclass, match->subclass);
+    format_match_class(prog_if, match->prog_if);
+    ksnprintf(buf, size, "<id = %s:%s, class = %s:%s:%s>", vendor, device, class, subclass, prog_if);
+}
+
+static bool pci_match_device(const struct pci_match* m, const struct pci_device* dev) {
+    return (m->vendor_id == PCI_ANY_ID || m->vendor_id == dev->id.vendor_id) &&
+        (m->device_id == PCI_ANY_ID || m->device_id == dev->id.device_id) &&
+        (m->class == PCI_ANY_CLASS || m->class == dev->class.class) &&
+        (m->subclass == PCI_ANY_CLASS || m->subclass == dev->class.subclass) &&
+        (m->prog_if == PCI_ANY_CLASS || m->prog_if == dev->class.prog_if);
+}
+
+// Returns the first entry that matches, or nullptr.
+static const struct pci_match* pci_match_table(
+    const struct pci_match* table,
+    size_t num_matches,
+    const struct pci_device* dev
+) {
+    for (size_t i = 0; i < num_matches; i++) {
+        if (pci_match_device(&table[i], dev))
+            return &table[i];
+    }
+    return nullptr;
+}
+
+errno_t pci_driver_register(struct pci_driver* driver) {
+    if (driver->match == nullptr || driver->num_matches == 0) {
+        log_warn("Driver \"%s\" does not define any matches, ignoring\n", driver->name);
+        return 0;
+    }
+
+    spin_lock(&device_lock);
+
+    vec_foreach(&drivers, i) {
+        if (strcmp(vec_elem_get(&drivers, i)->name, driver->name) == 0) {
+            spin_unlock(&device_lock);
+            log_warn("Driver \"%s\" is already registered\n", driver->name);
+            return EEXIST;
+        }
+    }
+
+    errno_t status = vec_push(&drivers, driver);
+    spin_unlock(&device_lock);
+    if (status != 0)
+        return status;
+
+    log("Registered PCI driver \"%s\" with %zu match%s\n",
+        driver->name,
+        driver->num_matches,
+        driver->num_matches == 1 ? "" : "es");
+    for (size_t i = 0; i < driver->num_matches; i++) {
+        char rule[48];
+        format_match(&driver->match[i], rule, sizeof(rule));
+        log("  [%zu] = %s\n", i, rule);
+    }
+
+    for (size_t i = 0;; i++) {
+        spin_lock(&device_lock);
+        if (i >= vec_length(&devices)) {
+            spin_unlock(&device_lock);
+            break;
+        }
+
+        struct pci_device* dev = vec_elem_get(&devices, i);
+        const struct pci_match* match = nullptr;
+        if (dev->driver == nullptr) {
+            match = pci_match_table(driver->match, driver->num_matches, dev);
+            if (match != nullptr)
+                dev->driver = driver;
+        }
+        spin_unlock(&device_lock);
+
+        if (match == nullptr)
+            continue;
+
+        pci_log(dev, "Matched driver \"%s\" on rule [%zu]\n", driver->name, (size_t)(match - driver->match));
+
+        status = driver->probe(dev, match);
+        if (status != 0) {
+            pci_log_warn(dev, "%s failed to probe: %i\n", driver->name, status);
+            spin_lock(&device_lock);
+            dev->driver = nullptr;
+            spin_unlock(&device_lock);
+        }
+    }
+
+    return 0;
+}
+
+[[__init]]
+static void pci_stage_init(struct boot_info*) {
+    if (vec_is_empty(&accesses)) {
+        log_warn("No configuration space access method available\n");
+        return;
+    }
+
+    log("Scanning PCI devices\n");
+
+    spin_lock(&device_lock);
+    vec_foreach(&accesses, i) {
+        struct pci_access* access = vec_elem_get(&accesses, i);
+        for (uint32_t bus = access->start_bus; bus <= access->end_bus; bus++) {
+            for (uint8_t slot = 0; slot < PCI_NUM_SLOTS; slot++) {
+                struct pci_address addr = {
+                    .domain = access->domain,
+                    .bus = bus,
+                    .device = slot,
+                };
+
+                if (pci_access_find(&addr) != access)
+                    continue;
+
+                scan_slot(access, addr);
+            }
+        }
+    }
+    spin_unlock(&device_lock);
+
+    log("Found %zu device(s)\n", vec_length(&devices));
+}
+INIT_TASK(pci_stage, "dev.pci", pci_stage_init);
+INIT_DEPENDS(pci_stage, mem_stage);
+INIT_ENTAILS(pci_stage, module_stage);

dev/pci/ecam.c

@@ -0,0 +1,76 @@
+#define pr_fmt(fmt) "pci: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/mmio.h>
+#include <zinnia/pci.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+
+struct pci_ecam {
+    struct pci_access access;
+    void* base;
+};
+
+static void* ecam_ptr(struct pci_access* self, const struct pci_address* addr, uint16_t offset) {
+    struct pci_ecam* ecam = containerof(self, struct pci_ecam, access);
+    size_t index = (size_t)(addr->bus - self->start_bus) << 20 | (size_t)(addr->device & 0x1f) << 15 |
+        (size_t)(addr->function & 0x7) << 12 | (offset & 0xffc);
+    return (uint8_t*)ecam->base + index;
+}
+
+static uint32_t ecam_read32(struct pci_access* self, const struct pci_address* addr, uint16_t offset) {
+    return mmio_read32(ecam_ptr(self, addr, offset));
+}
+
+static void ecam_write32(struct pci_access* self, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    mmio_write32(ecam_ptr(self, addr, offset), value);
+}
+
+errno_t pci_ecam_register(phys_t base, uint16_t domain, uint8_t start_bus, uint8_t end_bus) {
+    if (end_bus < start_bus)
+        return EINVAL;
+
+    size_t length = ((size_t)(end_bus - start_bus) + 1) << 20;
+    phys_t phys = base + ((phys_t)start_bus << 20);
+
+    uintptr_t vaddr = vm_alloc(length);
+    if (!vaddr)
+        return ENOMEM;
+
+    for (size_t i = 0; i < length; i += arch_mem_page_size()) {
+        errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, phys + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status != 0)
+            return status;
+    }
+
+    struct pci_ecam* ecam = mem_alloc(sizeof(*ecam), 0);
+    if (ecam == nullptr)
+        return ENOMEM;
+
+    *ecam = (struct pci_ecam){
+        .access =
+            {
+                .name = "ecam",
+                .domain = domain,
+                .start_bus = start_bus,
+                .end_bus = end_bus,
+                .priority = PCI_ACCESS_PRIO_ECAM,
+                .read32 = ecam_read32,
+                .write32 = ecam_write32,
+            },
+        .base = (void*)vaddr,
+    };
+
+    errno_t status = pci_access_register(&ecam->access);
+    if (status != 0) {
+        mem_free(ecam);
+        return status;
+    }
+
+    log("ECAM for %.4hx:%.2hhx - %.4hx:%.2hhx at %p\n", domain, start_bus, domain, end_bus, (void*)phys);
+    return 0;
+}

dev/pci/meson.build

@@ -0,0 +1,4 @@
+kernel_sources += files(
+    'core.c',
+    'ecam.c',
+)

fs/exec/elf.c

@@ -7,7 +7,6 @@
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
-#include <zinnia/string.h>
 #include <zinnia/utils.h>
 #include <zinnia/vfs.h>
 #include <zinnia/vma.h>
@@ -19,7 +18,7 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     errno_t err;
 
     size_t read;
-    struct elf_ehdr ehdr = {};
+    elf_ehdr_t ehdr = {};
     err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
     if (err)
         return err;
@@ -44,7 +43,7 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
 
     const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
     uintptr_t phdrs_end;
-    if (ehdr.e_phentsize < sizeof(struct elf_phdr) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
+    if (ehdr.e_phentsize < sizeof(elf_phdr_t) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
         return ENOEXEC;
     if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size))
         return ENOEXEC;
@@ -53,7 +52,7 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     uintptr_t phdr_addr = 0;
 
     for (size_t i = 0; i < ehdr.e_phnum; i++) {
-        struct elf_phdr phdr = {};
+        elf_phdr_t phdr = {};
         err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
         if (err)
             return err;

include/stdarg.h

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: ISC
+#pragma once
+
+typedef __builtin_va_list va_list;
+
+#undef va_start
+#define va_start(v, ...) __builtin_va_start(v, 0)
+
+#undef va_end
+#define va_end(v) __builtin_va_end(v)
+
+#undef va_arg
+#define va_arg(v, l) __builtin_va_arg(v, l)
+
+#undef va_copy
+#define va_copy(d, s) __builtin_va_copy(d, s)

include/stdatomic.h

@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: ISC
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef enum memory_order {
+    memory_order_relaxed = __ATOMIC_RELAXED,
+    memory_order_consume = __ATOMIC_CONSUME,
+    memory_order_acquire = __ATOMIC_ACQUIRE,
+    memory_order_release = __ATOMIC_RELEASE,
+    memory_order_acq_rel = __ATOMIC_ACQ_REL,
+    memory_order_seq_cst = __ATOMIC_SEQ_CST,
+} memory_order;
+
+#define atomic_thread_fence(order) __atomic_thread_fence(order)
+#define atomic_signal_fence(order) __atomic_signal_fence(order)
+
+#ifdef __clang__
+
+#define atomic_init(obj, value) __c11_atomic_init(obj, value)
+
+#define atomic_store_explicit(obj, val, order) __c11_atomic_store(obj, val, order)
+#define atomic_load_explicit(obj, order)       __c11_atomic_load(obj, order)
+
+#define atomic_exchange_explicit(obj, val, order) __c11_atomic_exchange(obj, val, order)
+
+#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \
+    __c11_atomic_compare_exchange_strong(obj, expected, desired, succ, fail)
+#define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \
+    __c11_atomic_compare_exchange_weak(obj, expected, desired, succ, fail)
+
+#define atomic_fetch_add_explicit(obj, arg, order) __c11_atomic_fetch_add(obj, arg, order)
+#define atomic_fetch_sub_explicit(obj, arg, order) __c11_atomic_fetch_sub(obj, arg, order)
+#define atomic_fetch_or_explicit(obj, arg, order)  __c11_atomic_fetch_or(obj, arg, order)
+#define atomic_fetch_xor_explicit(obj, arg, order) __c11_atomic_fetch_xor(obj, arg, order)
+#define atomic_fetch_and_explicit(obj, arg, order) __c11_atomic_fetch_and(obj, arg, order)
+
+#define atomic_is_lock_free(obj) __c11_atomic_is_lock_free(sizeof(*(obj)))
+
+#else /* GCC */
+
+#define atomic_init(obj, value) \
+    do { \
+        *(obj) = (value); \
+    } while (0)
+
+#define atomic_store_explicit(obj, val, order) \
+    ({ \
+        auto __ao = (obj); \
+        typeof(*__ao) __av = (val); \
+        __atomic_store(__ao, &__av, order); \
+    })
+
+#define atomic_load_explicit(obj, order) \
+    ({ \
+        auto __ao = (obj); \
+        typeof(*__ao) __ar; \
+        __atomic_load(__ao, &__ar, order); \
+        __ar; \
+    })
+
+#define atomic_exchange_explicit(obj, val, order) \
+    ({ \
+        auto __ao = (obj); \
+        typeof(*__ao) __av = (val); \
+        typeof(*__ao) __ar; \
+        __atomic_exchange(__ao, &__av, &__ar, order); \
+        __ar; \
+    })
+
+#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \
+    ({ \
+        auto __ao = (obj); \
+        auto __ae = (expected); \
+        typeof(*__ao) __ad = (desired); \
+        __atomic_compare_exchange(__ao, __ae, &__ad, 0, succ, fail); \
+    })
+
+#define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \
+    ({ \
+        auto __ao = (obj); \
+        auto __ae = (expected); \
+        typeof(*__ao) __ad = (desired); \
+        __atomic_compare_exchange(__ao, __ae, &__ad, 1, succ, fail); \
+    })
+
+#define atomic_fetch_add_explicit(obj, arg, order) __atomic_fetch_add(obj, arg, order)
+#define atomic_fetch_sub_explicit(obj, arg, order) __atomic_fetch_sub(obj, arg, order)
+#define atomic_fetch_or_explicit(obj, arg, order)  __atomic_fetch_or(obj, arg, order)
+#define atomic_fetch_xor_explicit(obj, arg, order) __atomic_fetch_xor(obj, arg, order)
+#define atomic_fetch_and_explicit(obj, arg, order) __atomic_fetch_and(obj, arg, order)
+
+#define atomic_is_lock_free(obj) __atomic_is_lock_free(sizeof(*(obj)), (obj))
+
+#endif
+
+#define atomic_store(obj, val) atomic_store_explicit(obj, val, memory_order_seq_cst)
+#define atomic_load(obj)       atomic_load_explicit(obj, memory_order_seq_cst)
+
+#define atomic_exchange(obj, val) atomic_exchange_explicit(obj, val, memory_order_seq_cst)
+
+#define atomic_compare_exchange_strong(obj, expected, desired) \
+    atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst)
+#define atomic_compare_exchange_weak(obj, expected, desired) \
+    atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst)
+
+#define atomic_fetch_add(obj, arg) atomic_fetch_add_explicit(obj, arg, memory_order_seq_cst)
+#define atomic_fetch_sub(obj, arg) atomic_fetch_sub_explicit(obj, arg, memory_order_seq_cst)
+#define atomic_fetch_or(obj, arg)  atomic_fetch_or_explicit(obj, arg, memory_order_seq_cst)
+#define atomic_fetch_xor(obj, arg) atomic_fetch_xor_explicit(obj, arg, memory_order_seq_cst)
+#define atomic_fetch_and(obj, arg) atomic_fetch_and_explicit(obj, arg, memory_order_seq_cst)
+
+// atomic_flag
+typedef struct atomic_flag {
+    _Atomic(bool) __val;
+} atomic_flag;
+
+#define ATOMIC_FLAG_INIT {0}
+
+#define atomic_flag_test_and_set_explicit(obj, order) atomic_exchange_explicit(&(obj)->__val, 1, order)
+#define atomic_flag_test_and_set(obj)                 atomic_flag_test_and_set_explicit(obj, memory_order_seq_cst)
+#define atomic_flag_clear_explicit(obj, order)        atomic_store_explicit(&(obj)->__val, 0, order)
+#define atomic_flag_clear(obj)                        atomic_flag_clear_explicit(obj, memory_order_seq_cst)
+
+typedef _Atomic(bool) atomic_bool;
+typedef _Atomic(char) atomic_char;
+typedef _Atomic(signed char) atomic_schar;
+typedef _Atomic(unsigned char) atomic_uchar;
+typedef _Atomic(short) atomic_short;
+typedef _Atomic(unsigned short) atomic_ushort;
+typedef _Atomic(int) atomic_int;
+typedef _Atomic(unsigned int) atomic_uint;
+typedef _Atomic(long) atomic_long;
+typedef _Atomic(unsigned long) atomic_ulong;
+typedef _Atomic(long long) atomic_llong;
+typedef _Atomic(unsigned long long) atomic_ullong;
+typedef _Atomic(__CHAR16_TYPE__) atomic_char16_t;
+typedef _Atomic(__CHAR32_TYPE__) atomic_char32_t;
+typedef _Atomic(__WCHAR_TYPE__) atomic_wchar_t;
+
+typedef _Atomic(int_least8_t) atomic_int_least8_t;
+typedef _Atomic(uint_least8_t) atomic_uint_least8_t;
+typedef _Atomic(int_least16_t) atomic_int_least16_t;
+typedef _Atomic(uint_least16_t) atomic_uint_least16_t;
+typedef _Atomic(int_least32_t) atomic_int_least32_t;
+typedef _Atomic(uint_least32_t) atomic_uint_least32_t;
+typedef _Atomic(int_least64_t) atomic_int_least64_t;
+typedef _Atomic(uint_least64_t) atomic_uint_least64_t;
+typedef _Atomic(int_fast8_t) atomic_int_fast8_t;
+typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t;
+typedef _Atomic(int_fast16_t) atomic_int_fast16_t;
+typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t;
+typedef _Atomic(int_fast32_t) atomic_int_fast32_t;
+typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t;
+typedef _Atomic(int_fast64_t) atomic_int_fast64_t;
+typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t;
+typedef _Atomic(intptr_t) atomic_intptr_t;
+typedef _Atomic(uintptr_t) atomic_uintptr_t;
+typedef _Atomic(size_t) atomic_size_t;
+typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t;
+typedef _Atomic(intmax_t) atomic_intmax_t;
+typedef _Atomic(uintmax_t) atomic_uintmax_t;

include/stdckdint.h

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: ISC
+#pragma once
+
+#define ckd_add(result, a, b) __builtin_add_overflow(a, b, result)
+#define ckd_sub(result, a, b) __builtin_sub_overflow(a, b, result)
+#define ckd_mul(result, a, b) __builtin_mul_overflow(a, b, result)

include/stddef.h

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: ISC
+#pragma once
+
+typedef __SIZE_TYPE__ size_t;
+typedef __PTRDIFF_TYPE__ ptrdiff_t;
+
+typedef __WCHAR_TYPE__ wchar_t;
+typedef typeof(nullptr) nullptr_t;
+
+#undef offsetof
+#define offsetof(s, m) __builtin_offsetof(s, m)
+
+#undef unreachable
+#define unreachable() __builtin_unreachable()

include/stdint.h

@@ -0,0 +1,354 @@
+// SPDX-License-Identifier: ISC
+#pragma once
+
+#ifdef __UINT8_TYPE__
+typedef __UINT8_TYPE__ uint8_t;
+#endif
+#ifdef __UINT16_TYPE__
+typedef __UINT16_TYPE__ uint16_t;
+#endif
+#ifdef __UINT32_TYPE__
+typedef __UINT32_TYPE__ uint32_t;
+#endif
+#ifdef __UINT64_TYPE__
+typedef __UINT64_TYPE__ uint64_t;
+#endif
+
+typedef __UINT_LEAST8_TYPE__ uint_least8_t;
+typedef __UINT_LEAST16_TYPE__ uint_least16_t;
+typedef __UINT_LEAST32_TYPE__ uint_least32_t;
+typedef __UINT_LEAST64_TYPE__ uint_least64_t;
+
+typedef __UINT_FAST8_TYPE__ uint_fast8_t;
+typedef __UINT_FAST16_TYPE__ uint_fast16_t;
+typedef __UINT_FAST32_TYPE__ uint_fast32_t;
+typedef __UINT_FAST64_TYPE__ uint_fast64_t;
+
+#ifdef __INT8_TYPE__
+typedef __INT8_TYPE__ int8_t;
+#endif
+#ifdef __INT16_TYPE__
+typedef __INT16_TYPE__ int16_t;
+#endif
+#ifdef __INT32_TYPE__
+typedef __INT32_TYPE__ int32_t;
+#endif
+#ifdef __INT64_TYPE__
+typedef __INT64_TYPE__ int64_t;
+#endif
+
+typedef __INT_LEAST8_TYPE__ int_least8_t;
+typedef __INT_LEAST16_TYPE__ int_least16_t;
+typedef __INT_LEAST32_TYPE__ int_least32_t;
+typedef __INT_LEAST64_TYPE__ int_least64_t;
+
+typedef __INT_FAST8_TYPE__ int_fast8_t;
+typedef __INT_FAST16_TYPE__ int_fast16_t;
+typedef __INT_FAST32_TYPE__ int_fast32_t;
+typedef __INT_FAST64_TYPE__ int_fast64_t;
+
+#ifdef __UINTPTR_TYPE__
+typedef __UINTPTR_TYPE__ uintptr_t;
+#endif
+#ifdef __INTPTR_TYPE__
+typedef __INTPTR_TYPE__ intptr_t;
+#endif
+
+typedef __UINTMAX_TYPE__ uintmax_t;
+typedef __INTMAX_TYPE__ intmax_t;
+
+// Clang and GCC have different mechanisms for INT32_C and friends.
+#ifdef __clang__
+#ifndef __FREESTANDING_C_HDRS_C_JOIN
+#define __FREESTANDING_C_HDRS_C_EXPAND_JOIN(x, suffix) x##suffix
+#define __FREESTANDING_C_HDRS_C_JOIN(x, suffix)        __FREESTANDING_C_HDRS_C_EXPAND_JOIN(x, suffix)
+#endif
+
+#undef INT8_C
+#define INT8_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __INT8_C_SUFFIX__)
+#undef INT16_C
+#define INT16_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __INT16_C_SUFFIX__)
+#undef INT32_C
+#define INT32_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __INT32_C_SUFFIX__)
+#undef INT64_C
+#define INT64_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __INT64_C_SUFFIX__)
+
+#undef UINT8_C
+#define UINT8_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __UINT8_C_SUFFIX__)
+#undef UINT16_C
+#define UINT16_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __UINT16_C_SUFFIX__)
+#undef UINT32_C
+#define UINT32_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __UINT32_C_SUFFIX__)
+#undef UINT64_C
+#define UINT64_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __UINT64_C_SUFFIX__)
+
+#undef INTMAX_C
+#define INTMAX_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __INTMAX_C_SUFFIX__)
+#undef UINTMAX_C
+#define UINTMAX_C(x) __FREESTANDING_C_HDRS_C_JOIN(x, __UINTMAX_C_SUFFIX__)
+#else
+#undef INT8_C
+#define INT8_C(x) __INT8_C(x)
+#undef INT16_C
+#define INT16_C(x) __INT16_C(x)
+#undef INT32_C
+#define INT32_C(x) __INT32_C(x)
+#undef INT64_C
+#define INT64_C(x) __INT64_C(x)
+
+#undef UINT8_C
+#define UINT8_C(x) __UINT8_C(x)
+#undef UINT16_C
+#define UINT16_C(x) __UINT16_C(x)
+#undef UINT32_C
+#define UINT32_C(x) __UINT32_C(x)
+#undef UINT64_C
+#define UINT64_C(x) __UINT64_C(x)
+
+#undef INTMAX_C
+#define INTMAX_C(x) __INTMAX_C(x)
+#undef UINTMAX_C
+#define UINTMAX_C(x) __UINTMAX_C(x)
+#endif
+
+#ifdef __UINT8_MAX__
+#undef UINT8_MAX
+#define UINT8_MAX __UINT8_MAX__
+#endif
+#ifdef __UINT16_MAX__
+#undef UINT16_MAX
+#define UINT16_MAX __UINT16_MAX__
+#endif
+#ifdef __UINT32_MAX__
+#undef UINT32_MAX
+#define UINT32_MAX __UINT32_MAX__
+#endif
+#ifdef __UINT64_MAX__
+#undef UINT64_MAX
+#define UINT64_MAX __UINT64_MAX__
+#endif
+
+#ifdef __INT8_MAX__
+#undef INT8_MAX
+#define INT8_MAX __INT8_MAX__
+#endif
+#ifdef __INT16_MAX__
+#undef INT16_MAX
+#define INT16_MAX __INT16_MAX__
+#endif
+#ifdef __INT32_MAX__
+#undef INT32_MAX
+#define INT32_MAX __INT32_MAX__
+#endif
+#ifdef __INT64_MAX__
+#undef INT64_MAX
+#define INT64_MAX __INT64_MAX__
+#endif
+
+#ifdef __INT8_MAX__
+#undef INT8_MIN
+#define INT8_MIN (-INT8_MAX - 1)
+#endif
+#ifdef __INT16_MAX__
+#undef INT16_MIN
+#define INT16_MIN (-INT16_MAX - 1)
+#endif
+#ifdef __INT32_MAX__
+#undef INT32_MIN
+#define INT32_MIN (-INT32_MAX - 1)
+#endif
+#ifdef __INT64_MAX__
+#undef INT64_MIN
+#define INT64_MIN (-INT64_MAX - 1)
+#endif
+
+#undef UINT_LEAST8_MAX
+#define UINT_LEAST8_MAX __UINT_LEAST8_MAX__
+#undef UINT_LEAST16_MAX
+#define UINT_LEAST16_MAX __UINT_LEAST16_MAX__
+#undef UINT_LEAST32_MAX
+#define UINT_LEAST32_MAX __UINT_LEAST32_MAX__
+#undef UINT_LEAST64_MAX
+#define UINT_LEAST64_MAX __UINT_LEAST64_MAX__
+
+#undef INT_LEAST8_MAX
+#define INT_LEAST8_MAX __INT_LEAST8_MAX__
+#undef INT_LEAST16_MAX
+#define INT_LEAST16_MAX __INT_LEAST16_MAX__
+#undef INT_LEAST32_MAX
+#define INT_LEAST32_MAX __INT_LEAST32_MAX__
+#undef INT_LEAST64_MAX
+#define INT_LEAST64_MAX __INT_LEAST64_MAX__
+
+#undef INT_LEAST8_MIN
+#define INT_LEAST8_MIN (-INT_LEAST8_MAX - 1)
+#undef INT_LEAST16_MIN
+#define INT_LEAST16_MIN (-INT_LEAST16_MAX - 1)
+#undef INT_LEAST32_MIN
+#define INT_LEAST32_MIN (-INT_LEAST32_MAX - 1)
+#undef INT_LEAST64_MIN
+#define INT_LEAST64_MIN (-INT_LEAST64_MAX - 1)
+
+#undef UINT_FAST8_MAX
+#define UINT_FAST8_MAX __UINT_FAST8_MAX__
+#undef UINT_FAST16_MAX
+#define UINT_FAST16_MAX __UINT_FAST16_MAX__
+#undef UINT_FAST32_MAX
+#define UINT_FAST32_MAX __UINT_FAST32_MAX__
+#undef UINT_FAST64_MAX
+#define UINT_FAST64_MAX __UINT_FAST64_MAX__
+
+#undef INT_FAST8_MAX
+#define INT_FAST8_MAX __INT_FAST8_MAX__
+#undef INT_FAST16_MAX
+#define INT_FAST16_MAX __INT_FAST16_MAX__
+#undef INT_FAST32_MAX
+#define INT_FAST32_MAX __INT_FAST32_MAX__
+#undef INT_FAST64_MAX
+#define INT_FAST64_MAX __INT_FAST64_MAX__
+
+#undef INT_FAST8_MIN
+#define INT_FAST8_MIN (-INT_FAST8_MAX - 1)
+#undef INT_FAST16_MIN
+#define INT_FAST16_MIN (-INT_FAST16_MAX - 1)
+#undef INT_FAST32_MIN
+#define INT_FAST32_MIN (-INT_FAST32_MAX - 1)
+#undef INT_FAST64_MIN
+#define INT_FAST64_MIN (-INT_FAST64_MAX - 1)
+
+#ifdef __UINTPTR_MAX__
+#undef UINTPTR_MAX
+#define UINTPTR_MAX __UINTPTR_MAX__
+#endif
+#ifdef __INTPTR_MAX__
+#undef INTPTR_MAX
+#define INTPTR_MAX __INTPTR_MAX__
+#undef INTPTR_MIN
+#define INTPTR_MIN (-INTPTR_MAX - 1)
+#endif
+
+#undef UINTMAX_MAX
+#define UINTMAX_MAX __UINTMAX_MAX__
+#undef INTMAX_MAX
+#define INTMAX_MAX __INTMAX_MAX__
+#undef INTMAX_MIN
+#define INTMAX_MIN (-INTMAX_MAX - 1)
+
+#undef PTRDIFF_MAX
+#define PTRDIFF_MAX __PTRDIFF_MAX__
+#undef PTRDIFF_MIN
+#define PTRDIFF_MIN (-PTRDIFF_MAX - 1)
+
+#undef SIG_ATOMIC_MAX
+#define SIG_ATOMIC_MAX __SIG_ATOMIC_MAX__
+#undef SIG_ATOMIC_MIN
+#define SIG_ATOMIC_MIN (-SIG_ATOMIC_MAX - 1)
+
+#undef SIZE_MAX
+#define SIZE_MAX __SIZE_MAX__
+
+#undef WCHAR_MAX
+#define WCHAR_MAX __WCHAR_MAX__
+#undef WCHAR_MIN
+#define WCHAR_MIN (-WCHAR_MAX - 1)
+
+#undef WINT_MAX
+#define WINT_MAX __WINT_MAX__
+#undef WINT_MIN
+#define WINT_MIN (-WINT_MAX - 1)
+
+#ifdef __INT8_TYPE__
+#undef INT8_WIDTH
+#define INT8_WIDTH 8
+#endif
+#ifdef __UINT8_TYPE__
+#undef UINT8_WIDTH
+#define UINT8_WIDTH 8
+#endif
+#ifdef __INT16_TYPE__
+#undef INT16_WIDTH
+#define INT16_WIDTH 16
+#endif
+#ifdef __UINT16_TYPE__
+#undef UINT16_WIDTH
+#define UINT16_WIDTH 16
+#endif
+#ifdef __INT32_TYPE__
+#undef INT32_WIDTH
+#define INT32_WIDTH 32
+#endif
+#ifdef __UINT32_TYPE__
+#undef UINT32_WIDTH
+#define UINT32_WIDTH 32
+#endif
+#ifdef __INT64_TYPE__
+#undef INT64_WIDTH
+#define INT64_WIDTH 64
+#endif
+#ifdef __UINT64_TYPE__
+#undef UINT64_WIDTH
+#define UINT64_WIDTH 64
+#endif
+
+#undef INT_LEAST8_WIDTH
+#define INT_LEAST8_WIDTH __INT_LEAST8_WIDTH__
+#undef UINT_LEAST8_WIDTH
+#define UINT_LEAST8_WIDTH __INT_LEAST8_WIDTH__
+#undef INT_LEAST16_WIDTH
+#define INT_LEAST16_WIDTH __INT_LEAST16_WIDTH__
+#undef UINT_LEAST16_WIDTH
+#define UINT_LEAST16_WIDTH __INT_LEAST16_WIDTH__
+#undef INT_LEAST32_WIDTH
+#define INT_LEAST32_WIDTH __INT_LEAST32_WIDTH__
+#undef UINT_LEAST32_WIDTH
+#define UINT_LEAST32_WIDTH __INT_LEAST32_WIDTH__
+#undef INT_LEAST64_WIDTH
+#define INT_LEAST64_WIDTH __INT_LEAST64_WIDTH__
+#undef UINT_LEAST64_WIDTH
+#define UINT_LEAST64_WIDTH __INT_LEAST64_WIDTH__
+
+#undef INT_FAST8_WIDTH
+#define INT_FAST8_WIDTH __INT_FAST8_WIDTH__
+#undef UINT_FAST8_WIDTH
+#define UINT_FAST8_WIDTH __INT_FAST8_WIDTH__
+#undef INT_FAST16_WIDTH
+#define INT_FAST16_WIDTH __INT_FAST16_WIDTH__
+#undef UINT_FAST16_WIDTH
+#define UINT_FAST16_WIDTH __INT_FAST16_WIDTH__
+#undef INT_FAST32_WIDTH
+#define INT_FAST32_WIDTH __INT_FAST32_WIDTH__
+#undef UINT_FAST32_WIDTH
+#define UINT_FAST32_WIDTH __INT_FAST32_WIDTH__
+#undef INT_FAST64_WIDTH
+#define INT_FAST64_WIDTH __INT_FAST64_WIDTH__
+#undef UINT_FAST64_WIDTH
+#define UINT_FAST64_WIDTH __INT_FAST64_WIDTH__
+
+#ifdef __INTPTR_TYPE__
+#undef INTPTR_WIDTH
+#define INTPTR_WIDTH __INTPTR_WIDTH__
+#endif
+#ifdef __UINTPTR_TYPE__
+#undef UINTPTR_WIDTH
+#define UINTPTR_WIDTH __INTPTR_WIDTH__
+#endif
+
+#undef INTMAX_WIDTH
+#define INTMAX_WIDTH __INTMAX_WIDTH__
+#undef UINTMAX_WIDTH
+#define UINTMAX_WIDTH __INTMAX_WIDTH__
+
+#undef PTRDIFF_WIDTH
+#define PTRDIFF_WIDTH __PTRDIFF_WIDTH__
+
+#undef SIG_ATOMIC_WIDTH
+#define SIG_ATOMIC_WIDTH __SIG_ATOMIC_WIDTH__
+
+#undef SIZE_WIDTH
+#define SIZE_WIDTH __SIZE_WIDTH__
+
+#undef WCHAR_WIDTH
+#define WCHAR_WIDTH __WCHAR_WIDTH__
+
+#undef WINT_WIDTH
+#define WINT_WIDTH __WINT_WIDTH__

include/string.h

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: ISC
 #pragma once
 
 #include <stddef.h>
@@ -11,4 +12,3 @@ void* memset(void* dest, int value, size_t n);
 char* strncpy(char* restrict dst, const char* restrict src, size_t len);
 int strcmp(const char* str1, const char* str2);
 int strncmp(const char* str1, const char* str2, size_t len);
-char* strchr(const char* c, int s);

include/zinnia/alloc.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/types.h>
 #include <arch/mem.h>
 #include <stddef.h>
@@ -51,3 +52,5 @@ void mem_phys_init(struct phys_mem* map, size_t length);
 static inline size_t mem_high_shift() {
     return arch_mem_level_bits() * arch_mem_num_levels() + arch_mem_page_bits();
 }
+
+INIT_EXTERN(mem_stage);

include/zinnia/bio.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <zinnia/mman.h>
+
+// TODO
+
+enum bio_type : uint8_t {
+    BLOCK_READ,
+    BLOCK_WRITE,
+};
+
+struct bio {
+    enum bio_type type;
+    uint32_t num_lbas;
+    uint64_t lba;
+};

include/zinnia/cmdline.h

@@ -2,6 +2,7 @@
 
 #include <zinnia/compiler.h>
 #include <zinnia/utils.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #define _CMDLINE_OPTION(x, opt_name, opt_desc, opt_func) \
@@ -48,3 +49,10 @@ struct cmdline_option {
 
 // Parses the command line and invokes all options.
 void cmdline_parse(const char* cmdline);
+
+void cmdline_parse_buf(
+    char* buf,
+    size_t len,
+    const struct cmdline_option* const* start,
+    const struct cmdline_option* const* end
+);

include/zinnia/console.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <zinnia/compiler.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/utils.h>
 #include <stddef.h>
 
@@ -10,16 +11,16 @@ struct console {
     bool (*init)(struct console* con);
     // Writes to the console.
     void (*write)(struct console* con, const char* buf, size_t count);
+    struct console* next;
 };
 
-// Initializes a console for logging.
-void console_init();
+INIT_EXTERN(console_stage);
+
+// Registers a console.
+bool console_register(struct console* con);
+
+// Unregisters a regular console.
+void console_unregister(struct console* con);
 
 // Writes a message to a console.
 void console_write(const char* buf, size_t len);
-
-// Defines a new early console which can be used to log early boot messages.
-// An early console must not make any heap allocations.
-#define DEFINE_CONSOLE(con) \
-    [[__used, __section(".console")]] \
-    static struct console* UNIQUE_IDENT(console) = (con)

include/zinnia/elf.h

@@ -2,7 +2,7 @@
 
 #include <zinnia/errno.h>
 #include <zinnia/vfs.h>
-#include <bits/elf.h>
+#include <arch/elf.h>
 #include <stdint.h>
 
 constexpr char ELF_MAG[4] = {0x7F, 'E', 'L', 'F'};
@@ -74,6 +74,20 @@ enum : uint32_t {
     PT_TLS = 0x00000007,
 };
 
+enum : uint32_t {
+    PT_MOD_VERSION = 0x60000001,
+    PT_MOD_AUTHOR = 0x60000002,
+    PT_MOD_DESC = 0x60000003,
+    PT_MOD_LICENSE = 0x60000004,
+    PT_MOD_CMDLINE = 0x60000005,
+    PT_MOD_EXIT = 0x60000006,
+    PT_MOD_MATCH = 0x60000007,
+};
+
+enum : uint16_t {
+    SHN_UNDEF = 0,
+};
+
 enum : uint32_t {
     PF_X = 0x00000001,
     PF_W = 0x00000002,
@@ -110,6 +124,9 @@ enum : uint32_t {
     DT_FINI_ARRAY = 26,
     DT_INIT_ARRAYSZ = 27,
     DT_FINI_ARRAYSZ = 28,
+    DT_RELRSZ = 35,
+    DT_RELR = 36,
+    DT_RELRENT = 37,
     DT_LOOS = 0x60000000,
     DT_HIOS = 0x6FFFFFFF,
     DT_LOPROC = 0x70000000,
@@ -140,7 +157,7 @@ enum : uint32_t {
     AT_INIT_HANDLE = 0x1000,
 };
 
-struct elf64_ehdr {
+typedef struct {
     uint8_t e_ident[EI_NIDENT];
     uint16_t e_type;
     uint16_t e_machine;
@@ -155,9 +172,9 @@ struct elf64_ehdr {
     uint16_t e_shentsize;
     uint16_t e_shnum;
     uint16_t e_shstrndx;
-};
+} elf64_ehdr_t;
 
-struct elf32_ehdr {
+typedef struct {
     uint8_t e_ident[EI_NIDENT];
     uint16_t e_type;
     uint16_t e_machine;
@@ -172,9 +189,9 @@ struct elf32_ehdr {
     uint16_t e_shentsize;
     uint16_t e_shnum;
     uint16_t e_shstrndx;
-};
+} elf32_ehdr_t;
 
-struct elf64_phdr {
+typedef struct {
     uint32_t p_type;
     uint32_t p_flags;
     elf64_off_t p_offset;
@@ -183,9 +200,9 @@ struct elf64_phdr {
     uint64_t p_filesz;
     uint64_t p_memsz;
     uint64_t p_align;
-};
+} elf64_phdr_t;
 
-struct elf32_phdr {
+typedef struct {
     uint32_t p_type;
     elf32_off_t p_offset;
     elf32_addr_t p_vaddr;
@@ -194,68 +211,68 @@ struct elf32_phdr {
     uint32_t p_memsz;
     uint32_t p_flags;
     uint32_t p_align;
-};
+} elf32_phdr_t;
 
-struct elf64_dyn {
+typedef struct {
     int64_t d_tag;
     union {
         uint64_t d_val;
         elf64_addr_t d_ptr;
     } d_un;
-};
+} elf64_dyn_t;
 
-struct elf32_dyn {
+typedef struct {
     int32_t r_offset;
     union {
         uint32_t d_val;
         elf32_addr_t d_ptr;
     } d_un;
-};
+} elf32_dyn_t;
 
-struct elf64_nhdr {
+typedef struct {
     uint64_t n_namesz;
     uint64_t n_descsz;
     uint64_t n_type;
-};
+} elf64_nhdr_t;
 
-struct elf32_nhdr {
+typedef struct {
     uint32_t n_namesz;
     uint32_t n_descsz;
     uint32_t n_type;
-};
+} elf32_nhdr_t;
 
-struct elf64_auxv {
+typedef struct {
     uint64_t atype;
     uint64_t avalue;
-};
+} elf64_auxv_t;
 
-struct elf32_auxv {
+typedef struct {
     uint32_t atype;
     uint32_t avalue;
-};
+} elf32_auxv_t;
 
-struct elf_hash {
+typedef struct {
     uint32_t n_buckets;
     uint32_t n_chain;
-};
+} elf_hash_t;
 
-struct elf32_sym {
+typedef struct {
     elf32_off_t st_name;
     elf32_addr_t st_value;
     elf32_off_t st_size;
     uint8_t st_info;
     uint8_t st_other;
     uint16_t st_shndx;
-};
+} elf32_sym_t;
 
-struct elf64_sym {
+typedef struct {
     uint32_t st_name;
     uint8_t st_info;
     uint8_t st_other;
     uint16_t st_shndx;
     uint64_t st_value;
     uint64_t st_size;
-};
+} elf64_sym_t;
 
 static uint8_t ELF64_ST_BIND(uint8_t info) {
     return info >> 4;
@@ -272,7 +289,7 @@ static inline uint8_t ELF64_ST_INFO(uint8_t bind, uint8_t type) {
 typedef struct {
     uint16_t si_boundto;
     uint16_t si_flags;
-} elf64_syminfo;
+} elf64_syminfo_t;
 
 static inline uint8_t ELF32_ST_BIND(uint8_t info) {
     return info >> 4;
@@ -304,27 +321,30 @@ enum {
     STT_HIPROC = 15
 };
 
-struct elf32_rela {
+typedef struct {
     elf32_addr_t r_offset;
     uint32_t r_info;
     int32_t r_addend;
-};
+} elf32_rela_t;
 
-struct elf64_rela {
+typedef struct {
     elf64_addr_t r_offset;
     uint64_t r_info;
     int64_t r_addend;
-};
+} elf64_rela_t;
 
-struct elf32_rel {
+typedef uint64_t elf64_relr_t;
+typedef uint32_t elf32_relr_t;
+
+typedef struct {
     elf32_addr_t r_offset;
     uint32_t r_info;
-};
+} elf32_rel_t;
 
-struct elf64_rel {
+typedef struct {
     elf64_addr_t r_offset;
     uint64_t r_info;
-};
+} elf64_rel_t;
 
 static inline uint32_t ELF32_R_SYM(uint32_t info) {
     return info >> 8;

include/zinnia/errno.h

@@ -4,7 +4,6 @@
 #include <uapi/errno.h>
 
 typedef enum : int32_t {
-    EOK = 0, // Success! Not really an error though...
     EPERM = __EPERM,
     ENOENT = __ENOENT,
     ESRCH = __ESRCH,

include/zinnia/fdt.h

@@ -21,8 +21,18 @@ constexpr uint32_t FDT_END_NODE = 0x00000002;
 constexpr uint32_t FDT_PROP = 0x00000003;
 constexpr uint32_t FDT_NOP = 0x00000004;
 constexpr uint32_t FDT_END = 0x00000009;
+constexpr uint32_t FDT_MAGIC = 0xD00DFEED;
+
+struct fdt_match {
+    const char* compatible;
+    uintptr_t data;
+};
+
+#define FDT_MATCH_COMPATIBLE(compat) .compatible = (compat)
 
 struct fdt {
+    const uint8_t* blob;
+    size_t total_size;
     uint32_t version;
     uint32_t compat_version;
     uint32_t boot_cpuid;
@@ -35,11 +45,23 @@ struct fdt {
 
 struct fdt_node {
     struct fdt* tree;
+    size_t offset;
 };
 
 struct fdt_prop {
     struct fdt* tree;
+    const void* data;
+    size_t length;
 };
 
+uint32_t fdt_read_u32(const void* data);
+uint64_t fdt_read_u64(const void* data);
+bool fdt_init(struct fdt* tree, const void* data, size_t length);
+struct fdt_node fdt_find(struct fdt* tree, const char* path);
+struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible);
+struct fdt_prop fdt_property(struct fdt_node node, const char* name);
+bool fdt_property_u32(struct fdt_node node, const char* name, uint32_t* value);
+bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value);
+
 // Root of the device tree.
 struct fdt_node fdt_root(struct fdt* tree);

include/zinnia/hashmap.h

@@ -4,7 +4,7 @@
 #include <zinnia/errno.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <zinnia/string.h>
+#include <string.h>
 
 #define hashmap(K, V) \
     struct { \

include/zinnia/init.h

@@ -38,6 +38,7 @@ struct boot_info {
     struct boot_file* files;
     size_t num_files;
     struct boot_framebuffer fb;
+    struct boot_file kernel_file;
     void* dtb;
     size_t dtb_size;
 #if CONFIG_ACPI

include/zinnia/initgraph.h

@@ -0,0 +1,82 @@
+#pragma once
+
+#include <zinnia/compiler.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+
+enum init_action {
+    INIT_EMPTY,
+    INIT_CALLBACK,
+    INIT_GATE,
+};
+
+struct boot_info;
+struct init_node;
+
+struct init_edge {
+    struct init_node* source;
+    struct init_node* target;
+    bool propagates_disable;
+    struct init_edge* next_in;
+    struct init_edge* next_out;
+};
+
+struct init_edges {
+    struct init_node* node;
+    bool depends;
+    size_t count;
+    struct init_node* const* nodes;
+    struct init_edge* edges;
+};
+
+struct init_node {
+    const char* name;
+    enum init_action action;
+    union {
+        void (*callback)(struct boot_info* info);
+        bool (*gate)(struct boot_info* info);
+        void* func;
+    };
+    size_t unsatisfied;
+    bool wanted;
+    bool done;
+    bool disabled;
+    struct init_edge* in_edges;
+    struct init_edge* out_edges;
+    struct init_node* queue_next;
+};
+
+#define INIT_EXTERN(node) extern struct init_node node[1]
+
+#define INIT_TASK(node, display, ...) \
+    struct init_node node[1] = {{ \
+        .name = (display), \
+        __VA_OPT__( \
+            .action = _Generic( \
+                (__VA_ARGS__), \
+                void (*)(struct boot_info*): INIT_CALLBACK, \
+                bool (*)(struct boot_info*): INIT_GATE \
+            ), \
+            .func = (void*)(__VA_ARGS__), \
+        ) \
+    }}; \
+    [[__used, __section(".initgraph.nodes")]] \
+    static struct init_node* const CONCAT(node, _initgraph_node) = node
+
+#define _INIT_EDGES(x, from, is_depends, ...) \
+    static struct init_node* const CONCAT(x, _nodes)[] = {__VA_ARGS__}; \
+    static struct init_edge CONCAT(x, _edges)[ARRAY_SIZE(CONCAT(x, _nodes))]; \
+    static const struct init_edges x = { \
+        .node = (from), \
+        .depends = (is_depends), \
+        .count = ARRAY_SIZE(CONCAT(x, _nodes)), \
+        .nodes = CONCAT(x, _nodes), \
+        .edges = CONCAT(x, _edges), \
+    }; \
+    [[__used, __section(".initgraph.edges")]] \
+    static const struct init_edges* const CONCAT(x, _ptr) = &x
+
+#define INIT_DEPENDS(node, ...) _INIT_EDGES(UNIQUE_IDENT(init_edges), node, true, __VA_ARGS__)
+#define INIT_ENTAILS(node, ...) _INIT_EDGES(UNIQUE_IDENT(init_edges), node, false, __VA_ARGS__)
+
+void initgraph_run(struct init_node* goal, struct boot_info* info);

include/zinnia/iovec.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <zinnia/errno.h>
-#include <zinnia/string.h>
 #include <zinnia/usercopy.h>
 #include <zinnia/utils.h>
 #include <stddef.h>

include/zinnia/ksyms.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <zinnia/initgraph.h>
+#include <stddef.h>
+#include <stdint.h>
+
+void ksym_init();
+
+// Finds the function containing `addr`. Returns false if no symbol is known.
+bool ksym_lookup(uintptr_t addr, const char** name, size_t* offset);
+
+// Writes `name+0xoffset` for `addr` to `buf`, or `<???>` if the address can't be resolved.
+void ksym_format(uintptr_t addr, char* buf, size_t size);
+
+bool ksym_find(const char* name, uintptr_t* addr);
+
+const char* ksym_kernel_name();
+
+INIT_EXTERN(ksyms_stage);

include/zinnia/module.h

@@ -1,7 +1,11 @@
 #pragma once
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/tailq.h>
+#include <zinnia/types.h>
 #include <zinnia/utils.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #define MODULE_PROPERTY(name, data) \
@@ -13,36 +17,83 @@
 #define MODULE_VERSION(version) MODULE_PROPERTY("version", version)
 #define MODULE_LICENSE(license) MODULE_PROPERTY("license", license)
 
-#ifdef IN_MODULE
+struct match_table {
+    const char* name;
+    const void* entries;
+    size_t size;
+};
+
+#ifdef __IN_MODULE
+#define MATCH_TABLE(match_name, table) \
+    [[__used, __section(".mod.match"), __aligned(8)]] \
+    static const struct match_table CONCAT(table, _match_table) = { \
+        .name = (match_name), \
+        .entries = (table), \
+        .size = ARRAY_SIZE(table), \
+    }
+#else
+#define MATCH_TABLE(match_name, table) static_assert(true)
+#endif
+
+// Module entries are used
+#ifdef __IN_MODULE
 #define MODULE_ENTRY(func) \
-    [[__used]] \
+    [[__used, __visibility("hidden")]] \
     errno_t __module_entry(struct module* __module) { \
         return func(__module); \
-    } \
+    }
+#else
+#define MODULE_ENTRY(func) \
     [[__used, __section(".mod.entry")]] \
-    static const void* __entry_fn = &__module_entry
+    static errno_t (*const UNIQUE_IDENT(module_entry))(struct module*) = &(func)
+#endif
 
+#ifdef __IN_MODULE
 #define MODULE_EXIT(func) \
-    [[__used]] \
+    [[__used, __visibility("hidden")]] \
     errno_t __module_exit(struct module* __module) { \
         return func(__module); \
     } \
     [[__used, __section(".mod.exit")]] \
-    static const void* __exit_fn = &__module_exit
+    static errno_t (*const __exit_fn)(struct module*) = &__module_exit
 #else
-#define MODULE_ENTRY(func) \
-    [[__used, __section(".mod.entry")]] \
-    static const void* UNIQUE_IDENT(module_entry) = &(func)
-
-#define MODULE_EXIT(func) \
-    [[__used, __section(".mod.exit")]] \
-    static const void* UNIQUE_IDENT(module_exit) = &(func)
+// Built-in drivers can't be unloaded.
+#define MODULE_EXIT(func) static_assert(&(func) != nullptr)
 #endif
 
+struct module_segment {
+    phys_t phys;
+    uintptr_t virt;
+    size_t size;
+};
+
 struct module {
     const char* name;
+    const char* version;
+    const char* author;
+    const char* desc;
+    const char* license;
+    uintptr_t base;
+    size_t size;
+    uintptr_t bias;
+    const void* symtab;
+    size_t num_syms;
+    const char* strtab;
+    size_t strtab_size;
+    struct module_segment* segments;
+    size_t num_segments;
+    errno_t (*exit)(struct module*);
+    char* cmdline;
+    tailq_link(struct module) link;
 };
 
 #define THIS_MODULE (struct module*)nullptr
 
-void module_init();
+struct boot_info;
+void module_init(struct boot_info* info);
+
+errno_t module_load(const void* data, size_t length, const char* cmdline, struct module** out);
+errno_t module_unload(struct module* mod);
+struct module* module_find(const char* name);
+
+INIT_EXTERN(module_stage);

include/zinnia/pci.h

@@ -0,0 +1,187 @@
+#pragma once
+
+#include <zinnia/alloc.h>
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct pci_address {
+    uint16_t domain;
+    uint8_t bus;
+    uint8_t device;
+    uint8_t function;
+};
+
+#define PCI_ADDR_FMT     "%.4hx:%.2hhx:%.2hhx.%hhx"
+#define PCI_ADDR_ARGS(a) (a)->domain, (a)->bus, (a)->device, (a)->function
+
+struct pci_id {
+    uint16_t vendor_id;
+    uint16_t device_id;
+    uint16_t subvendor_id;
+    uint16_t subdevice_id;
+};
+
+struct pci_class {
+    uint8_t class;
+    uint8_t subclass;
+    uint8_t prog_if;
+};
+
+enum pci_cfg : uint16_t {
+    PCI_VENDOR_ID = 0x00,
+    PCI_DEVICE_ID = 0x02,
+    PCI_COMMAND = 0x04,
+    PCI_STATUS = 0x06,
+    PCI_PROG_IF = 0x09,
+    PCI_SUBCLASS = 0x0a,
+    PCI_CLASS = 0x0b,
+    PCI_HEADER_TYPE = 0x0e,
+    PCI_BAR0 = 0x10,
+    PCI_SECONDARY_BUS = 0x19,
+    PCI_SUBSYS_VENDOR_ID = 0x2c,
+    PCI_SUBSYS_ID = 0x2e,
+    PCI_CAP_PTR = 0x34,
+    PCI_INTERRUPT_LINE = 0x3c,
+    PCI_INTERRUPT_PIN = 0x3d,
+};
+
+enum pci_command : uint16_t {
+    PCI_COMMAND_IO = 1 << 0,
+    PCI_COMMAND_MEM = 1 << 1,
+    PCI_COMMAND_MASTER = 1 << 2,
+    PCI_COMMAND_INTX_DISABLE = 1 << 10,
+};
+
+constexpr uint16_t PCI_STATUS_CAP_LIST = 1 << 4;
+
+constexpr uint8_t PCI_HEADER_TYPE_MASK = 0x7f;
+constexpr uint8_t PCI_HEADER_TYPE_MULTIFUNCTION = 0x80;
+constexpr uint8_t PCI_HEADER_TYPE_BRIDGE = 0x01;
+
+constexpr size_t PCI_NUM_BARS = 6;
+constexpr uint8_t PCI_NUM_SLOTS = 32;
+constexpr uint8_t PCI_NUM_FUNCTIONS = 8;
+
+enum pci_cap_id : uint8_t {
+    PCI_CAP_MSI = 0x05,
+    PCI_CAP_PCIE = 0x10,
+    PCI_CAP_MSIX = 0x11,
+};
+
+struct pci_access {
+    const char* name;
+    uint16_t domain;
+    uint8_t start_bus;
+    uint8_t end_bus;
+    uint8_t priority;
+    uint32_t (*read32)(struct pci_access* self, const struct pci_address* addr, uint16_t offset);
+    void (*write32)(struct pci_access* self, const struct pci_address* addr, uint16_t offset, uint32_t value);
+};
+
+enum pci_access_priority : uint8_t {
+    PCI_ACCESS_PRIO_LEGACY = 0,
+    PCI_ACCESS_PRIO_ECAM = 128,
+};
+
+static bool pci_access_decodes(const struct pci_access* access, const struct pci_address* addr);
+
+errno_t pci_access_register(struct pci_access* access);
+struct pci_access* pci_access_find(const struct pci_address* addr);
+
+errno_t pci_ecam_register(phys_t base, uint16_t domain, uint8_t start_bus, uint8_t end_bus);
+
+struct pci_driver;
+
+struct pci_device {
+    struct pci_address address;
+    struct pci_id id;
+    struct pci_class class;
+    struct pci_access* access;
+    uint8_t header_type;
+    struct pci_driver* driver;
+};
+
+uint8_t pci_read8(struct pci_device* dev, uint16_t offset);
+uint16_t pci_read16(struct pci_device* dev, uint16_t offset);
+uint32_t pci_read32(struct pci_device* dev, uint16_t offset);
+void pci_write8(struct pci_device* dev, uint16_t offset, uint8_t value);
+void pci_write16(struct pci_device* dev, uint16_t offset, uint16_t value);
+void pci_write32(struct pci_device* dev, uint16_t offset, uint32_t value);
+
+enum pci_bar_type : uint8_t {
+    PCI_BAR_NONE,
+    PCI_BAR_MMIO32,
+    PCI_BAR_MMIO64,
+    PCI_BAR_IO,
+};
+
+struct pci_bar {
+    enum pci_bar_type type;
+    bool prefetchable;
+    phys_t address;
+    size_t size;
+};
+
+errno_t pci_bar_get(struct pci_device* dev, size_t index, struct pci_bar* out);
+
+uint8_t pci_cap_next(struct pci_device* dev, uint8_t cur);
+uint8_t pci_cap_find(struct pci_device* dev, uint8_t id);
+
+#define PCI_ANY_ID    0xffff
+#define PCI_ANY_CLASS 0xff
+
+// Matches a device by ID and class.
+struct pci_match {
+    uint16_t vendor_id;
+    uint16_t device_id;
+    uint8_t class;
+    uint8_t subclass;
+    uint8_t prog_if;
+
+    // Not used to match, but can be used to pass device-specific context data.
+    uintptr_t data;
+};
+
+#define PCI_MATCH_DEVICE(vendor, device) \
+    .vendor_id = (vendor), .device_id = (device), .class = PCI_ANY_CLASS, .subclass = PCI_ANY_CLASS, \
+    .prog_if = PCI_ANY_CLASS
+
+#define PCI_MATCH_CLASS(c, sc, pi) \
+    .vendor_id = PCI_ANY_ID, .device_id = PCI_ANY_ID, .class = (c), .subclass = (sc), .prog_if = (pi)
+
+// Writes the address to `buf`.
+static void pci_device_name(const struct pci_device* dev, char buf[static 18]);
+
+struct pci_driver {
+    const char* name;
+
+    // Match table, array of `num_matches` entries, checked sequentially.
+    // When writing a match table, the most specific matches should come first, then more generic cases.
+    const struct pci_match* match;
+    const size_t num_matches;
+
+    errno_t (*probe)(struct pci_device* dev, const struct pci_match* match);
+};
+
+errno_t pci_driver_register(struct pci_driver* driver);
+
+INIT_EXTERN(pci_stage);
+
+#ifdef __MODULE_NAME
+#define PCI_LOG_NAME __MODULE_NAME
+#else
+#define PCI_LOG_NAME "pci"
+#endif
+
+#define pci_log_level(level, dev, message, ...) \
+    log_write(level, PCI_LOG_NAME " " PCI_ADDR_FMT ": " message, PCI_ADDR_ARGS(&(dev)->address), ##__VA_ARGS__)
+
+#define pci_log_debug(dev, message, ...) pci_log_level(LOG_DEBUG, dev, message, ##__VA_ARGS__)
+#define pci_log(dev, message, ...)       pci_log_level(LOG_INFO, dev, message, ##__VA_ARGS__)
+#define pci_log_warn(dev, message, ...)  pci_log_level(LOG_WARN, dev, message, ##__VA_ARGS__)
+#define pci_log_err(dev, message, ...)   pci_log_level(LOG_ERR, dev, message, ##__VA_ARGS__)

include/zinnia/percpu.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <zinnia/compiler.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/sched.h>
 #include <zinnia/usercopy.h>
 #include <stddef.h>
@@ -40,7 +41,9 @@ struct percpu* percpu_new();
 size_t percpu_max_cpus();
 struct percpu* percpu_get();
 void percpu_bsp_init();
-void percpu_init();
+void percpu_init(struct boot_info* info);
+
+INIT_EXTERN(percpu_stage);
 
 // Per-CPU data for the bootstrap processor.
 extern struct percpu percpu_bsp;

include/zinnia/print.h

@@ -2,19 +2,45 @@
 
 #include <zinnia/compiler.h>
 #include <stdarg.h>
+#include <stddef.h>
 
 [[__format(printf, 1, 2)]]
 void kprintf(const char* message, ...);
 
 void kvprintf(const char* message, va_list args);
 
-#include <zinnia/clock.h>
-#include <zinnia/percpu.h>
+[[__format(printf, 3, 4)]]
+size_t ksnprintf(char* buf, size_t size, const char* format, ...);
 
-#define log(message, ...) \
-    do { \
-        auto _time = clock_get_elapsed_ns(); \
-        auto _cur = percpu_get()->sched.current; \
-        auto _tid = _cur ? _cur->id : 0; \
-        kprintf("[%5lu.%06lu] [%5lu] " message, _time / 1'000'000'000, _time / 1'000 % 1'000'000, _tid, ##__VA_ARGS__); \
-    } while (0)
+size_t kvsnprintf(char* buf, size_t size, const char* format, va_list args);
+
+typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
+void print_fmt(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args);
+
+enum log_level {
+    LOG_DEBUG,
+    LOG_STATUS,
+    LOG_INFO,
+    LOG_WARN,
+    LOG_ERR,
+    LOG_PANIC,
+};
+
+[[__format(printf, 2, 3)]]
+void log_write(enum log_level level, const char* message, ...);
+
+void log_vwrite(enum log_level level, const char* message, va_list args);
+
+// Like log_vwrite(), but prepends a `prefix` to the message.
+void log_vwrite_prefixed(enum log_level level, const char* prefix, const char* message, va_list args);
+
+#ifndef pr_fmt
+#define pr_fmt(fmt) fmt
+#endif
+
+#define log(message, ...)        log_write(LOG_INFO, pr_fmt(message), ##__VA_ARGS__)
+#define log_debug(message, ...)  log_write(LOG_DEBUG, pr_fmt(message), ##__VA_ARGS__)
+#define log_status(message, ...) log_write(LOG_STATUS, pr_fmt(message), ##__VA_ARGS__)
+#define log_warn(message, ...)   log_write(LOG_WARN, pr_fmt(message), ##__VA_ARGS__)
+#define log_err(message, ...)    log_write(LOG_ERR, pr_fmt(message), ##__VA_ARGS__)
+#define log_panic(message, ...)  log_write(LOG_PANIC, pr_fmt(message), ##__VA_ARGS__)

include/zinnia/sched.h

@@ -2,10 +2,12 @@
 
 #include <zinnia/errno.h>
 #include <zinnia/init.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/stack.h>
 #include <zinnia/tailq.h>
 #include <zinnia/types.h>
 #include <bits/sched.h>
+#include <stdatomic.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -34,6 +36,7 @@ struct task {
     uintptr_t user_stack;
     size_t time_slice;
     int8_t priority;
+    atomic_bool wake_pending;
 
     tailq_link(struct task) next;
 };
@@ -57,6 +60,13 @@ void sched_init(struct scheduler* sched);
 void sched_reschedule(struct scheduler* sched);
 void sched_yield(struct scheduler* sched);
 void sched_add_task(struct scheduler* sched, struct task* task);
+
+// Blocks the current task until sched_wake() is called on it.
+// Returns immediately if a wake is already pending. Must not hold any spinlock.
+void sched_block(struct scheduler* sched);
+
+// Makes a blocked task runnable again, or marks the wake as pending if it hasn't blocked yet.
+void sched_wake(struct task* task);
 [[noreturn]]
 void sched_to_user(uintptr_t ctx);
 [[noreturn]]
@@ -68,3 +78,5 @@ struct task_start_info {
     uintptr_t sp;
     uintptr_t arg;
 };
+
+INIT_EXTERN(sched_stage);

include/zinnia/spin.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <stdatomic.h>
 #include <stdint.h>
 
 // Busy-waits in a loop until the lock is freed.

include/zinnia/usb.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <stdint.h>
+
+#define USB_ANY_ID    0xffff
+#define USB_ANY_CLASS 0xff
+
+struct usb_match {
+    uint16_t vendor_id;
+    uint16_t product_id;
+    uint8_t class;
+    uint8_t subclass;
+    uint8_t protocol;
+    uintptr_t data;
+};
+
+#define USB_MATCH_DEVICE(vendor, product) \
+    .vendor_id = (vendor), .product_id = (product), .class = USB_ANY_CLASS, .subclass = USB_ANY_CLASS, \
+    .protocol = USB_ANY_CLASS
+
+#define USB_MATCH_CLASS(c, sc, p) \
+    .vendor_id = USB_ANY_ID, .product_id = USB_ANY_ID, .class = (c), .subclass = (sc), .protocol = (p)

include/zinnia/vec.h

@@ -3,7 +3,7 @@
 #include <zinnia/alloc.h>
 #include <zinnia/errno.h>
 #include <stddef.h> // IWYU pragma: keep
-#include <zinnia/string.h> // IWYU pragma: keep
+#include <string.h> // IWYU pragma: keep
 
 #define vec(T) \
     struct { \

meson.build

@@ -29,22 +29,25 @@ assert(c_compiler.has_define('__GNUC__'))
 add_project_arguments(
     '-Wno-unused',
     '-Wno-unused-parameter',
-    '-fno-stack-protector',
-    '-ffunction-sections',
-    '-fdata-sections',
-    '-ffreestanding',
-    '-fno-stack-protector',
     language: 'c',
 )
 
-add_project_link_arguments(
+common_c_args = [
+    '-fno-stack-protector',
+    '-fno-omit-frame-pointer',
+    '-ffunction-sections',
+    '-fdata-sections',
+    '-ffreestanding',
+    '-nostdinc',
+]
+
+common_link_args = [
     '-ffreestanding',
     '-nostdlib',
     '-znoexecstack',
     '-Wl,--gc-sections',
     '-Wl,--build-id=none',
-    language: 'c',
-)
+]
 
 common_includes = [
     include_directories('include'),
@@ -70,12 +73,14 @@ foreach k, v : kconfig
     endif
 endforeach
 
-kernel_c_args = ['-DIN_KERNEL']
+kernel_c_args = ['-D__IN_KERNEL']
 
 kernel_lds = meson.current_source_dir() / 'boot' / 'kernel.ld'
 kernel_link_args = [
     '-T' + kernel_lds,
-    '-Wl,--export-dynamic',
+    '-shared',
+    '-Wl,-Bsymbolic',
+    '-Wl,-soname,zinnia',
 ]
 
 kernel_includes = []
@@ -92,71 +97,107 @@ subdir('syscalls')
 subdir('dev')
 subdir('fs')
 
+subdir('tests')
+
 configure_file(
     configuration: cfg,
     output: 'config.h',
 )
 
+zinnia_datadir = get_option('datadir') / 'zinnia'
+
 mod_summary = {}
+modular_modules = {}
+builtin_libs = []
+
 foreach name, v : modules
-    is_modular = v.get('modular', false)
-    mod_sources = v.get('sources')
-    mod_c_args = v.get('c_args', [])
-    mod_link_args = v.get('link_args', [])
-    mod_includes = v.get('include_directories', [])
-    mod_link = v.get('link_with', [])
-
-    if is_modular
-        shared_library(
-            name,
-            sources: mod_sources,
-            c_args: ['-fPIC', '-fPIE', '-DIN_MODULE', mod_c_args],
-            link_args: [
-                '-T' + module_lds, mod_link_args,
-            ],
-            link_depends: module_lds,
-            include_directories: [
-                common_includes,
-                kernel_includes, # We want this, actually
-                mod_includes,
-            ],
-            name_prefix: '',
-            name_suffix: 'kso',
-            link_with: mod_link,
-            override_options: ['b_lundef=false'],
-            install_dir: get_option('datadir') / 'zinnia' / 'modules',
-            #link_depends: kernel.full_path(),
+    foreach dep : v.get('depends', [])
+        assert(
+            dep in modules,
+            'Module "' + name + '" depends on unknown module "' + dep + '"',
         )
-    else
-        kernel_sources += mod_sources
-        kernel_c_args += mod_c_args
-        kernel_includes += mod_includes
-        kernel_link_args += mod_link_args
+        assert(
+            not modules[dep].get('modular', false),
+            'Module "'
+            + name
+            + '" can\'t depend on modular module "'
+            + dep
+            + '"',
+        )
+    endforeach
+
+    if v.get('modular', false)
+        modular_modules += {name: v}
+        mod_summary += {name: 'module'}
+        continue
     endif
 
-    if is_modular
-        mod_summary += {name: 'mod'}
-    else
-        mod_summary += {name: 'builtin'}
-    endif
+    mod_c_args = v.get('c_args', [])
+    mod_includes = v.get('include_directories', [])
+    builtin_libs += static_library(
+        name,
+        sources: v.get('sources'),
+        c_args: [
+            common_c_args,
+            kernel_c_args,
+            mod_c_args,
+            '-D__MODULE_NAME="' + name + '"',
+        ],
+        include_directories: [common_includes, kernel_includes, mod_includes],
+        pic: true,
+    )
+    kernel_c_args += mod_c_args
+    kernel_includes += mod_includes
+    kernel_link_args += v.get('link_args', [])
+    mod_summary += {name: 'builtin'}
 endforeach
 
 # Kernel
-executable(
+kernel = executable(
     'zinnia',
     include_directories: [
         common_includes,
         kernel_includes,
     ],
-    c_args: [kernel_c_args],
-    link_args: kernel_link_args,
+    c_args: [common_c_args, kernel_c_args],
+    link_args: [common_link_args, kernel_link_args],
     link_depends: kernel_lds,
     sources: kernel_sources,
-    install: true,
-    install_dir: get_option('datadir') / 'zinnia',
+    link_whole: builtin_libs,
     pie: true,
+    install: true,
+    install_dir: zinnia_datadir,
 )
 
+foreach name, v : modular_modules
+    shared_library(
+        name,
+        sources: v.get('sources'),
+        c_args: [
+            common_c_args,
+            '-fPIC',
+            '-D__IN_MODULE',
+            '-D__MODULE_NAME="' + name + '"',
+            v.get('c_args', []),
+        ],
+        link_args: [
+            common_link_args,
+            '-T' + module_lds, kernel.full_path(),
+            v.get('link_args', []),
+        ],
+        link_depends: [module_lds, kernel],
+        include_directories: [
+            common_includes,
+            kernel_includes, # We want this, actually
+            v.get('include_directories', []),
+        ],
+        name_prefix: '',
+        name_suffix: 'kso',
+        link_with: v.get('link_with', []),
+        install_dir: zinnia_datadir / 'modules',
+    )
+endforeach
+
 # Print a summary
 summary(mod_summary, section: 'Driver configuration')
 summary(kconfig, bool_yn: true, section: 'Kernel configuration')

scripts/get_modinfo.py

@@ -1,14 +1,134 @@
 #!/usr/bin/env python3
-# Small script to print all options available inside a built kernel.
+# Small script to print all module information available.
 # fmt: off
+import struct
 import sys
 
+from elftools.elf.dynamic import DynamicSegment
 from elftools.elf.elffile import ELFFile
+from elftools.elf.relocation import RelrRelocationTable
 
 SIZEOF_PTR = 8
 SIZEOF_STRUCT = 40
 ENDIAN = "little"
 
+MATCH_SEGMENT = 0x60000007
+TABLE_FORMAT = "QQQ"
+PTR_SIZE = 8
+
+
+def format_id(value):
+    return "****" if value == 0xFFFF else f"{value:04x}"
+
+
+def format_class(value):
+    return "**" if value == 0xFF else f"{value:02x}"
+
+
+def format_ids(module, addr):
+    first, second, cls, subcls, iface = struct.unpack(
+        module.endian + "HHBBB", module.read(addr, 7)
+    )
+    ids = f"{format_id(first)}:{format_id(second)}"
+    classes = f"{format_class(cls)}:{format_class(subcls)}:{format_class(iface)}"
+    return f"<id = {ids}, class = {classes}>"
+
+
+def format_fdt(module, addr):
+    compatible = module.read_string(module.read_pointer(addr))
+    if compatible is None:
+        return "<compatible = *>"
+    return f'<compatible = "{compatible}">'
+
+
+def format_acpi(module, addr):
+    hid = module.read_string(module.read_pointer(addr))
+    if hid is None:
+        return "<hid = *>"
+    return f'<hid = "{hid}">'
+
+
+KINDS = {
+    "pci": {"size": 16, "format": format_ids},
+    "usb": {"size": 16, "format": format_ids},
+    "fdt": {"size": 16, "format": format_fdt},
+    "acpi": {"size": 16, "format": format_acpi},
+}
+
+
+class Module:
+    def __init__(self, stream):
+        self.stream = stream
+        self.file = ELFFile(stream)
+        self.byteorder = "little" if self.file.little_endian else "big"
+        self.endian = "<" if self.file.little_endian else ">"
+        self.addends = {}
+        for segment in self.file.iter_segments():
+            if isinstance(segment, DynamicSegment):
+                for table in segment.get_relocation_tables().values():
+                    if not isinstance(table, RelrRelocationTable) and table.is_RELA():
+                        for reloc in table.iter_relocations():
+                            self.addends[reloc["r_offset"]] = reloc["r_addend"]
+
+    def match_region(self):
+        for segment in self.file.iter_segments():
+            if segment["p_type"] == MATCH_SEGMENT:
+                return segment["p_vaddr"], segment.data()
+        return 0, b""
+
+    def read(self, addr, size):
+        self.stream.seek(next(self.file.address_offsets(addr, size)))
+        return self.stream.read(size)
+
+    def pointer(self, raw, addr):
+        return raw if raw != 0 else self.addends.get(addr, 0)
+
+    def read_pointer(self, addr):
+        return self.pointer(
+            int.from_bytes(self.read(addr, PTR_SIZE), self.byteorder), addr
+        )
+
+    def read_string(self, addr):
+        if addr == 0:
+            return None
+        self.stream.seek(next(self.file.address_offsets(addr)))
+        raw = b""
+        while (byte := self.stream.read(1)) not in (b"", b"\0"):
+            raw += byte
+        return raw.decode(errors="replace")
+
+
+def parse_matches(module):
+    base, data = module.match_region()
+    table_size = struct.calcsize(module.endian + TABLE_FORMAT)
+
+    tables = []
+    offset = 0
+    while offset + table_size <= len(data):
+        addr = base + offset
+        raw_name, raw_entries, count = struct.unpack_from(
+            module.endian + TABLE_FORMAT, data, offset
+        )
+        name_addr = module.pointer(raw_name, addr)
+        if name_addr == 0:
+            offset += PTR_SIZE
+            continue
+        offset += table_size
+
+        name = module.read_string(name_addr)
+        kind = KINDS.get(name)
+        if kind is None:
+            tables.append((name, count, None))
+            continue
+
+        entries_addr = module.pointer(raw_entries, addr + PTR_SIZE)
+        entries = [
+            kind["format"](module, entries_addr + i * kind["size"])
+            for i in range(count)
+        ]
+        tables.append((name, count, entries))
+    return tables
+
 
 if __name__ == "__main__":
     if len(sys.argv) < 2:
@@ -16,7 +136,8 @@ if __name__ == "__main__":
         sys.exit(1)
 
     with open(sys.argv[1], "rb") as elffile:
-        file = ELFFile(elffile)
+        module = Module(elffile)
+        file = module.file
 
         version = "?"
         author = "?"
@@ -35,3 +156,17 @@ if __name__ == "__main__":
         print(f"Version:\t{version}")
         print(f"Author(s):\t{author}")
         print(f"License:\t{license}")
+
+        tables = parse_matches(module)
+        if not tables:
+            print("Devices:\tnone")
+        else:
+            print("Devices:")
+        for name, count, entries in tables:
+            rules = f"{count} rule{'' if count == 1 else 's'}"
+            if entries is None:
+                print(f"  {name} ({rules}, unknown type)")
+                continue
+            print(f"  {name} ({rules})")
+            for j, entry in enumerate(entries):
+                print(f"    [{j}] = {entry}")

syscalls/uname.c

@@ -1,8 +1,8 @@
 #include <zinnia/spin.h>
-#include <zinnia/string.h>
 #include <zinnia/syscall.h>
 #include <zinnia/usercopy.h>
 #include <config.h>
+#include <string.h>
 #include <uapi/utsname.h>
 
 static struct spinlock uname_lock = {};