Rename dirs

lib/clock.c

@@ -0,0 +1,64 @@
+#define pr_fmt(fmt) "clock: " fmt
+
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
+#include <zinnia/compiler.h>
+#include <zinnia/print.h>
+#include <arch/cpu.h>
+#include <stdint.h>
+
+static _Atomic(struct clock*) active_clock;
+static _Atomic(int64_t) realtime_offset_ns;
+
+static uint64_t clock_read_elapsed(struct clock* clock) {
+    if (__unlikely(!clock))
+        return 0;
+
+    return clock->base + clock->get_elapsed_ns(clock);
+}
+
+bool clock_available() {
+    return atomic_load(&active_clock, ATOMIC_ACQUIRE) != nullptr;
+}
+
+uint64_t clock_get_elapsed_ns() {
+    return clock_read_elapsed(atomic_load(&active_clock, ATOMIC_ACQUIRE));
+}
+
+uint64_t clock_get_realtime_ns() {
+    int64_t ns = (int64_t)clock_get_elapsed_ns() + atomic_load(&realtime_offset_ns, ATOMIC_RELAXED);
+    return ns < 0 ? 0 : ns;
+}
+
+void clock_set_realtime_ns(uint64_t ns) {
+    atomic_store(&realtime_offset_ns, (int64_t)ns - (int64_t)clock_get_elapsed_ns(), ATOMIC_RELAXED);
+}
+
+bool clock_switch(struct clock* clock) {
+    struct clock* old = atomic_load(&active_clock, ATOMIC_ACQUIRE);
+
+    do {
+        if (old && clock->priority <= old->priority)
+            return false;
+
+        uint64_t elapsed = clock_read_elapsed(old);
+        clock->reset(clock);
+        clock->base = elapsed;
+    } while (!atomic_compare_exchange(&active_clock, &old, clock, ATOMIC_RELEASE, ATOMIC_ACQUIRE));
+
+    log("Switching to clock source \"%s\"\n", clock->name);
+
+    return true;
+}
+
+void clock_spin_ns(uint64_t ns) {
+    if (!clock_available()) {
+        log_err("Unable to sleep for %lu ns, no clock available!\n", ns);
+        return;
+    }
+
+    uint64_t target = clock_get_elapsed_ns() + ns;
+    while (clock_get_elapsed_ns() < target) {
+        arch_cpu_pause();
+    }
+}

lib/console.c

@@ -0,0 +1,47 @@
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+
+static struct console* registered = nullptr;
+static struct spinlock console_lock;
+
+void console_write(const char* buf, size_t len) {
+    spin_lock(&console_lock);
+    for (struct console* con = registered; con; con = con->next)
+        con->write(con, buf, len);
+    spin_unlock(&console_lock);
+}
+
+static void console_replay_write(void* ctx, const char* buf, size_t len) {
+    struct console* con = ctx;
+    con->write(con, buf, len);
+}
+
+bool console_register(struct console* con) {
+    if (!con->init(con))
+        return false;
+
+    con->write(con, "\ec", 2);
+    log_replay(console_replay_write, con);
+    spin_lock(&console_lock);
+    con->next = registered;
+    registered = con;
+    spin_unlock(&console_lock);
+
+    log("console: Registered console \"%s\"\n", con->name);
+    return true;
+}
+
+void console_unregister(struct console* con) {
+    spin_lock(&console_lock);
+    for (struct console** link = &registered; *link; link = &(*link)->next) {
+        if (*link == con) {
+            *link = con->next;
+            break;
+        }
+    }
+    spin_unlock(&console_lock);
+}
+
+INIT_TASK(console_stage, "core.console");

lib/dpc.c

@@ -0,0 +1,3 @@
+#include <zinnia/dpc.h>
+
+// TODO

lib/guid.c

@@ -0,0 +1,31 @@
+#include <zinnia/guid.h>
+#include <zinnia/print.h>
+#include <string.h>
+
+bool guid_is_zero(const struct guid* guid) {
+    static const struct guid zero = {};
+    return guid_equal(guid, &zero);
+}
+
+bool guid_equal(const struct guid* a, const struct guid* b) {
+    return memcmp(a, b, sizeof(struct guid)) == 0;
+}
+
+void guid_format(const struct guid* guid, char out[static GUID_STR_LEN]) {
+    ksnprintf(
+        out,
+        GUID_STR_LEN,
+        "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+        guid->time_low,
+        guid->time_mid,
+        guid->time_hi_version,
+        guid->clock_seq[0],
+        guid->clock_seq[1],
+        guid->node[0],
+        guid->node[1],
+        guid->node[2],
+        guid->node[3],
+        guid->node[4],
+        guid->node[5]
+    );
+}

lib/initgraph.c

@@ -0,0 +1,194 @@
+#define pr_fmt(fmt) "init: " fmt
+
+#include <zinnia/assert.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+
+extern struct init_node* const __ld_initgraph_nodes_start[];
+extern struct init_node* const __ld_initgraph_nodes_end[];
+extern const struct init_edges* const __ld_initgraph_edges_start[];
+extern const struct init_edges* const __ld_initgraph_edges_end[];
+
+static bool dump_graph = false;
+static void initgraph_option(bool value) {
+    dump_graph = value;
+}
+CMDLINE_OPTION("initgraph", "Print the init graph in DOT format after running", initgraph_option);
+
+static bool edges_linked = false;
+static struct init_node* failed_node = nullptr;
+
+struct init_queue {
+    struct init_node* head;
+    struct init_node* tail;
+};
+
+static void queue_push(struct init_queue* queue, struct init_node* node) {
+    node->queue_next = nullptr;
+    if (queue->tail)
+        queue->tail->queue_next = node;
+    else
+        queue->head = node;
+    queue->tail = node;
+}
+
+static struct init_node* queue_pop(struct init_queue* queue) {
+    struct init_node* node = queue->head;
+    if (node) {
+        queue->head = node->queue_next;
+        if (!queue->head)
+            queue->tail = nullptr;
+        node->queue_next = nullptr;
+    }
+    return node;
+}
+
+static void link_edges() {
+    for (auto cur = __ld_initgraph_edges_start; cur < __ld_initgraph_edges_end; cur++) {
+        const struct init_edges* group = *cur;
+        for (size_t i = 0; i < group->count; i++) {
+            struct init_edge* edge = &group->edges[i];
+            edge->source = group->depends ? group->nodes[i] : group->node;
+            edge->target = group->depends ? group->node : group->nodes[i];
+            edge->propagates_disable = group->depends;
+
+            edge->next_out = edge->source->out_edges;
+            edge->source->out_edges = edge;
+            edge->next_in = edge->target->in_edges;
+            edge->target->in_edges = edge;
+            edge->target->unsatisfied++;
+        }
+    }
+    edges_linked = true;
+}
+
+static void mark_wanted(struct init_node* goal) {
+    if (!goal) {
+        for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++)
+            (*cur)->wanted = true;
+        return;
+    }
+
+    struct init_queue queue = {};
+
+    if (!goal->wanted) {
+        goal->wanted = true;
+        queue_push(&queue, goal);
+    }
+
+    struct init_node* node;
+    while ((node = queue_pop(&queue))) {
+        for (struct init_edge* edge = node->in_edges; edge; edge = edge->next_in) {
+            if (!edge->source->wanted) {
+                edge->source->wanted = true;
+                queue_push(&queue, edge->source);
+            }
+        }
+    }
+}
+
+static bool is_ready(struct init_node* node) {
+    return node->wanted && !node->done && node->unsatisfied == 0;
+}
+
+static void reach_node(struct init_node* node, struct boot_info* info) {
+    if (node->disabled) {
+        log_status("Skipping stage \"%s\"\n", node->name);
+    } else {
+        log_status("Running stage \"%s\"\n", node->name);
+        failed_node = node;
+        switch (node->action) {
+        case INIT_EMPTY:
+            break;
+        case INIT_CALLBACK:
+            node->callback(info);
+            break;
+        case INIT_GATE:
+            if (!node->gate(info)) {
+                node->disabled = true;
+                node->failed = true;
+            }
+            break;
+        }
+        failed_node = nullptr;
+        node->executed = true;
+    }
+    node->done = true;
+}
+
+static void dump() {
+    kprintf("digraph initgraph {\n");
+    for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
+        struct init_node* node = *cur;
+        const char* style = "";
+        if (node->failed || node == failed_node)
+            style = ", color=red, fontcolor=red";
+        else if (!node->executed)
+            style = ", color=gray, fontcolor=gray";
+        kprintf("\tn%p [label=\"%s\"%s];\n", node, node->name, style);
+        for (struct init_edge* edge = node->in_edges; edge; edge = edge->next_in)
+            kprintf(
+                "\tn%p -> n%p%s;\n",
+                edge->source,
+                edge->target,
+                edge->source->executed && edge->target->executed ? "" : " [color=gray]"
+            );
+    }
+    kprintf("}\n");
+}
+
+void initgraph_panic() {
+    static bool dumping = false;
+    if (!dump_graph || !failed_node || dumping)
+        return;
+    dumping = true;
+    dump();
+}
+
+void initgraph_run(struct init_node* goal, struct boot_info* info) {
+    if (!edges_linked)
+        link_edges();
+
+    mark_wanted(goal);
+
+    struct init_queue pending = {};
+
+    for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
+        if (is_ready(*cur))
+            queue_push(&pending, *cur);
+    }
+
+    struct init_node* node;
+    while ((node = queue_pop(&pending))) {
+        reach_node(node, info);
+
+        for (struct init_edge* edge = node->out_edges; edge; edge = edge->next_out) {
+            struct init_node* successor = edge->target;
+            assert(successor->unsatisfied != 0, "Stage \"%s\" has no unsatisfied dependencies left", successor->name);
+            successor->unsatisfied--;
+
+            if (node->disabled && edge->propagates_disable)
+                successor->disabled = true;
+
+            if (is_ready(successor))
+                queue_push(&pending, successor);
+        }
+    }
+
+    for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
+        if ((*cur)->wanted && !(*cur)->done)
+            failed_node = *cur;
+        assert(
+            !(*cur)->wanted || (*cur)->done,
+            "The dependencies for stage \"%s\" could not be resolved",
+            (*cur)->name
+        );
+    }
+
+    if (!goal) {
+        log_status("All stages are complete!\n");
+        if (dump_graph)
+            dump();
+    }
+}

lib/iovec.c

@@ -0,0 +1,69 @@
+#include <zinnia/iovec.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <string.h>
+
+// Copies between `buf` and the iterator. If `to_iter` is set, data flows from `buf` into the iovecs.
+static errno_t iovec_iter_copy(iovec_iter_t* iter, void* buf, size_t len, bool to_iter, size_t* out_copied) {
+    size_t progress = 0;
+    errno_t status = 0;
+
+    while (progress < len && iter->current_idx < iter->iovecs_len) {
+        iovec_t* vec = &iter->iovecs[iter->current_idx];
+        size_t remaining = vec->len - iter->current_offset;
+        if (remaining == 0) {
+            iter->current_idx++;
+            iter->current_offset = 0;
+            continue;
+        }
+
+        size_t chunk = MIN(remaining, len - progress);
+        void* ptr = (void*)(vec->base + iter->current_offset);
+        uint8_t* kbuf = (uint8_t*)buf + progress;
+
+        if (iter->user) {
+            bool ok = to_iter ? usercopy_write((__user void*)ptr, kbuf, chunk)
+                              : usercopy_read(kbuf, (const __user void*)ptr, chunk);
+            if (!ok) {
+                status = EFAULT;
+                break;
+            }
+        } else if (to_iter) {
+            memcpy(ptr, kbuf, chunk);
+        } else {
+            memcpy(kbuf, ptr, chunk);
+        }
+
+        progress += chunk;
+        iter->current_offset += chunk;
+        iter->total_offset += chunk;
+    }
+
+    if (out_copied)
+        *out_copied = progress;
+    return status;
+}
+
+iovec_iter_t iovec_iter_kernel(iovec_t* iovecs, size_t count) {
+    size_t total = 0;
+    for (size_t i = 0; i < count; i++)
+        total += iovecs[i].len;
+
+    return (iovec_iter_t){
+        .iovecs = iovecs,
+        .iovecs_len = count,
+        .total_len = total,
+        .total_offset = 0,
+        .current_idx = 0,
+        .current_offset = 0,
+        .user = false,
+    };
+}
+
+errno_t iovec_iter_copy_to(iovec_iter_t* iter, const void* src, size_t len, size_t* out_copied) {
+    return iovec_iter_copy(iter, (void*)src, len, true, out_copied);
+}
+
+errno_t iovec_iter_copy_from(iovec_iter_t* iter, void* dst, size_t len, size_t* out_copied) {
+    return iovec_iter_copy(iter, dst, len, false, out_copied);
+}

lib/irq.c

@@ -0,0 +1,69 @@
+#include <zinnia/assert.h>
+#include <zinnia/irq.h>
+#include <zinnia/list.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <arch/irq.h>
+
+[[__percpu]]
+struct irq_percpu irq_data;
+
+bool irq_set_state(bool state) {
+    return arch_irq_set_state(state);
+}
+
+bool irq_get_state() {
+    return arch_irq_get_state();
+}
+
+void irq_wait() {
+    arch_irq_wait();
+}
+
+void irq_lock() {
+    bool prev = arch_irq_set_state(false);
+    struct irq_percpu* state = percpu_ref(irq_data);
+    if (state->depth++ == 0)
+        state->saved_state = prev;
+}
+
+void irq_unlock() {
+    struct irq_percpu* state = percpu_ref(irq_data);
+    if (--state->depth == 0 && state->saved_state)
+        arch_irq_set_state(true);
+}
+
+void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
+    irq_lock();
+    line->mode = line->set_config(line, trigger, polarity);
+    irq_unlock();
+}
+
+void irq_line_raise(struct irq_line* line) {
+    assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
+
+    irq_lock();
+    line->is_busy = true;
+
+    bool claimed = false;
+    struct irq_handler* handler;
+    list_foreach(handler, &line->handlers, next) {
+        if (handler->raise(handler) == IRQ_STATUS_HANDLED)
+            claimed = true;
+    }
+
+    if (!claimed)
+        log("irq: Spurious interrupt\n");
+
+    if ((line->mode & IRQ_MODE_EOI) && line->eoi)
+        line->eoi(line);
+
+    line->is_busy = false;
+    irq_unlock();
+}
+
+void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
+    irq_lock();
+    list_insert_head(&line->handlers, handler, next);
+    irq_unlock();
+}

lib/ksyms.c

@@ -0,0 +1,135 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/compiler.h>
+#include <zinnia/elf.h>
+#include <zinnia/hashmap.h>
+#include <zinnia/init.h>
+#include <zinnia/ksyms.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <bits/elf.h>
+#include <string.h>
+
+[[__visibility("hidden")]]
+extern const elf_sym_t __ld_dynsym_start[];
+[[__visibility("hidden")]]
+extern const elf_sym_t __ld_dynsym_end[];
+[[__visibility("hidden")]]
+extern const char __ld_dynstr_start[];
+[[__visibility("hidden")]]
+extern const char __ld_dynstr_end[];
+[[__visibility("hidden")]]
+extern const elf_dyn_t _DYNAMIC[];
+
+static uintptr_t slide;
+static bool slide_known;
+static const char* kernel_name;
+static hashmap(const char*, uintptr_t) symbols;
+
+static uint64_t sym_hash(const void* key, size_t) {
+    const char* name = *(const char* const*)key;
+    return hashmap_hash_default(name, strlen(name));
+}
+
+static bool sym_eq(const void* a, const void* b, size_t) {
+    return !strcmp(*(const char* const*)a, *(const char* const*)b);
+}
+
+static size_t num_syms() {
+    return __ld_dynsym_end - __ld_dynsym_start;
+}
+
+static const char* dynstr(uint64_t offset) {
+    if (offset >= (size_t)(__ld_dynstr_end - __ld_dynstr_start))
+        return nullptr;
+    return &__ld_dynstr_start[offset];
+}
+
+static uintptr_t kernel_slide() {
+    if (slide_known)
+        return slide;
+
+    for (size_t i = 0; i < num_syms(); i++) {
+        const char* name = dynstr(__ld_dynsym_start[i].st_name);
+        if (name != nullptr && !strcmp(name, "ksym_lookup")) {
+            slide = (uintptr_t)ksym_lookup - __ld_dynsym_start[i].st_value;
+            break;
+        }
+    }
+    slide_known = true;
+    return slide;
+}
+
+void ksym_init() {
+    const uintptr_t base = kernel_slide();
+
+    for (const elf_dyn_t* dyn = _DYNAMIC; dyn->d_tag != DT_NULL; dyn++) {
+        if (dyn->d_tag == DT_SONAME)
+            kernel_name = dynstr(dyn->d_un.d_val);
+    }
+
+    hashmap_init(&symbols);
+
+    size_t count = 0;
+    for (size_t i = 0; i < num_syms(); i++) {
+        const elf_sym_t* sym = &__ld_dynsym_start[i];
+        if (sym->st_shndx == SHN_UNDEF)
+            continue;
+
+        const char* name = dynstr(sym->st_name);
+        if (name == nullptr || *name == 0)
+            continue;
+
+        const errno_t status = hashmap_insert(&symbols, name, sym->st_value + base, sym_hash, sym_eq);
+        assert(status == 0, "Failed to register kernel symbol \"%s\" (%i)", name, status);
+        count++;
+    }
+
+    log("ksyms: Registered %zu kernel symbols\n", count);
+}
+
+bool ksym_find(const char* name, uintptr_t* addr) {
+    uintptr_t* value = hashmap_get(&symbols, name, sym_hash, sym_eq);
+    if (value == nullptr)
+        return false;
+    *addr = *value;
+    return true;
+}
+
+const char* ksym_kernel_name() {
+    return kernel_name;
+}
+
+bool ksym_lookup(uintptr_t addr, const char** name, size_t* offset) {
+    addr -= kernel_slide();
+    for (size_t i = 0; i < num_syms(); i++) {
+        const elf_sym_t* sym = &__ld_dynsym_start[i];
+        if (ELF_ST_TYPE(sym->st_info) != STT_FUNC || sym->st_shndx == SHN_UNDEF)
+            continue;
+        if (addr >= sym->st_value && addr - sym->st_value < sym->st_size) {
+            const char* sym_name = dynstr(sym->st_name);
+            if (sym_name == nullptr)
+                return false;
+            *name = sym_name;
+            *offset = addr - sym->st_value;
+            return true;
+        }
+    }
+    return false;
+}
+
+void ksym_format(uintptr_t addr, char* buf, size_t size) {
+    const char* name;
+    size_t offset;
+    if (ksym_lookup(addr, &name, &offset))
+        ksnprintf(buf, size, "%s+0x%zx", name, offset);
+    else
+        ksnprintf(buf, size, "<\?\?\?>");
+}
+
+[[__init]]
+static void ksyms_stage_init(struct boot_info*) {
+    ksym_init();
+}
+INIT_TASK(ksyms_stage, "core.ksyms", ksyms_stage_init);
+INIT_DEPENDS(ksyms_stage, mem_stage);

lib/meson.build

@@ -0,0 +1,20 @@
+kernel_sources += files(
+    'clock.c',
+    'console.c',
+    'guid.c',
+    'initgraph.c',
+    'iovec.c',
+    'irq.c',
+    'ksyms.c',
+    'module.c',
+    'panic.c',
+    'percpu.c',
+    'print.c',
+    'radix.c',
+    'random.c',
+    'ringbuf.c',
+    'runtime.c',
+    'spin.c',
+    'stdlib.c',
+    'string.c',
+)

lib/module.c

@@ -0,0 +1,615 @@
+#define pr_fmt(fmt) "module: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/elf.h>
+#include <zinnia/ksyms.h>
+#include <zinnia/module.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <bits/elf.h>
+#include <stdckdint.h>
+#include <string.h>
+
+extern errno_t (*__ld_mod_entry_start[])(struct module*);
+extern errno_t (*__ld_mod_entry_end[])(struct module*);
+
+static struct spinlock module_lock;
+static tailq_head(struct module) module_list;
+
+static bool in_range(size_t total, uint64_t offset, uint64_t size) {
+    return offset <= total && size <= total - offset;
+}
+
+static void* image_ptr(struct module* mod, uint64_t vaddr, uint64_t size) {
+    uintptr_t addr, end;
+    if (ckd_add(&addr, mod->bias, vaddr) || ckd_add(&end, addr, size))
+        return nullptr;
+
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
+        if (addr >= seg->virt && end <= seg->virt + seg->size)
+            return (void*)addr;
+    }
+    return nullptr;
+}
+
+static const char* image_cstr(struct module* mod, uint64_t vaddr, uint64_t size) {
+    const char* str = image_ptr(mod, vaddr, size);
+    if (str == nullptr || size == 0 || strnlen(str, size) >= size)
+        return nullptr;
+    return str;
+}
+
+static const char* table_str(const char* table, size_t size, uint64_t offset) {
+    if (offset >= size)
+        return nullptr;
+    const char* str = table + offset;
+    if (strnlen(str, size - offset) >= size - offset)
+        return nullptr;
+    return str;
+}
+
+static const char* sym_name(struct module* mod, uint64_t offset) {
+    return table_str(mod->strtab, mod->strtab_size, offset);
+}
+
+static struct module* find_locked(const char* name) {
+    for (struct module* m = tailq_first(&module_list); m != nullptr; m = tailq_next(m, link)) {
+        if (!strcmp(m->name, name))
+            return m;
+    }
+    return nullptr;
+}
+
+void module_init() {
+    spin_init(&module_lock);
+    tailq_init(&module_list);
+
+    for (auto start = __ld_mod_entry_start; start < __ld_mod_entry_end; start += 1) {
+        errno_t mod_status = (*start)(nullptr);
+        assert(mod_status == 0, "Built-in entry point returned %i", mod_status);
+    }
+}
+
+static errno_t resolve_symbol(struct module* mod, uint32_t index, uintptr_t* out) {
+    if (index >= mod->num_syms)
+        return ENOEXEC;
+
+    const elf_sym_t* sym = &((const elf_sym_t*)mod->symtab)[index];
+    if (sym->st_shndx != SHN_UNDEF) {
+        *out = mod->bias + sym->st_value;
+        return 0;
+    }
+
+    const char* name = sym_name(mod, sym->st_name);
+    if (name == nullptr)
+        return ENOEXEC;
+
+    if (ksym_find(name, out))
+        return 0;
+
+    if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
+        *out = 0;
+        return 0;
+    }
+
+    log_err("%s: Unresolved symbol \"%s\"\n", mod->name, name);
+    return ENOENT;
+}
+
+static errno_t store_word(struct module* mod, uint64_t vaddr, uintptr_t value) {
+    uintptr_t* where = image_ptr(mod, vaddr, sizeof(uintptr_t));
+    if (where == nullptr)
+        return ENOEXEC;
+    *where = value;
+    return 0;
+}
+
+static errno_t relocate_relative(struct module* mod, uint64_t vaddr) {
+    uintptr_t* where = image_ptr(mod, vaddr, sizeof(uintptr_t));
+    if (where == nullptr)
+        return ENOEXEC;
+    *where += mod->bias;
+    return 0;
+}
+
+static errno_t apply_rela(struct module* mod, uint64_t addr, uint64_t size, uint64_t entsize) {
+    if (entsize == 0)
+        entsize = sizeof(elf_rela_t);
+    if (entsize < sizeof(elf_rela_t))
+        return ENOEXEC;
+
+    const uint8_t* table = image_ptr(mod, addr, size);
+    if (table == nullptr)
+        return ENOEXEC;
+
+    for (uint64_t off = 0; off + sizeof(elf_rela_t) <= size; off += entsize) {
+        elf_rela_t rela;
+        memcpy(&rela, table + off, sizeof(rela));
+
+        errno_t status = 0;
+        uintptr_t value;
+        switch (ELF_R_TYPE(rela.r_info)) {
+        case R_NONE:
+            break;
+        case R_64:
+        case R_GLOB_DAT:
+        case R_JUMP_SLOT:
+            status = resolve_symbol(mod, ELF_R_SYM(rela.r_info), &value);
+            if (status == 0)
+                status = store_word(mod, rela.r_offset, value + rela.r_addend);
+            break;
+        case R_RELATIVE:
+            status = store_word(mod, rela.r_offset, mod->bias + rela.r_addend);
+            break;
+        default:
+            log_err("%s: Unsupported relocation type %u\n", mod->name, ELF_R_TYPE(rela.r_info));
+            status = ENOEXEC;
+            break;
+        }
+
+        if (status)
+            return status;
+    }
+    return 0;
+}
+
+static errno_t apply_relr(struct module* mod, uint64_t addr, uint64_t size, uint64_t entsize) {
+    if (entsize != 0 && entsize != sizeof(elf_relr_t))
+        return ENOEXEC;
+
+    const elf_relr_t* table = image_ptr(mod, addr, size);
+    if (table == nullptr)
+        return ENOEXEC;
+
+    uint64_t where = 0;
+    for (size_t i = 0; i < size / sizeof(elf_relr_t); i++) {
+        elf_relr_t entry = table[i];
+        errno_t status;
+
+        if ((entry & 1) == 0) {
+            status = relocate_relative(mod, entry);
+            if (status)
+                return status;
+            where = entry + sizeof(elf_relr_t);
+        } else {
+            for (size_t bit = 0; (entry >>= 1) != 0; bit++) {
+                if ((entry & 1) == 0)
+                    continue;
+                status = relocate_relative(mod, where + bit * sizeof(elf_relr_t));
+                if (status)
+                    return status;
+            }
+            where += (sizeof(elf_relr_t) * 8 - 1) * sizeof(elf_relr_t);
+        }
+    }
+    return 0;
+}
+
+static void module_free(struct module* mod) {
+    const size_t page_size = arch_mem_page_size();
+
+    mem_free(mod->cmdline);
+
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
+        for (size_t off = 0; off < seg->size; off += page_size)
+            pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+    }
+    pmap_shootdown(&kernel_vma.pmap);
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
+        mem_phys_free(seg->phys, seg->size / page_size);
+    }
+    mem_free(mod->segments);
+
+    if (mod->base != 0)
+        vm_free(mod->base, mod->size);
+
+    mem_free(mod);
+}
+
+static enum pte_flags prot_from_phdr(const elf_phdr_t* phdr) {
+    enum pte_flags prot = 0;
+    if (phdr->p_flags & PF_R)
+        prot |= PTE_READ;
+    if (phdr->p_flags & PF_W)
+        prot |= PTE_WRITE;
+    if (phdr->p_flags & PF_X)
+        prot |= PTE_EXEC;
+    return prot;
+}
+
+static errno_t read_headers(const void* data, size_t length, elf_ehdr_t* ehdr, elf_phdr_t** out) {
+    const uint8_t* file = data;
+
+    if (data == nullptr || length < sizeof(elf_ehdr_t))
+        return ENOEXEC;
+
+    memcpy(ehdr, file, sizeof(*ehdr));
+
+    if (memcmp(ehdr->e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0 || ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
+        ehdr->e_ident[EI_CLASS] != ELF_ARCH_CLASS || ehdr->e_ident[EI_DATA] != ELF_ARCH_DATA ||
+        ehdr->e_machine != ELF_ARCH_MACHINE || ehdr->e_type != ET_DYN)
+        return ENOEXEC;
+
+    if (ehdr->e_phentsize < sizeof(elf_phdr_t) || ehdr->e_phnum == 0 ||
+        !in_range(length, ehdr->e_phoff, (uint64_t)ehdr->e_phnum * ehdr->e_phentsize))
+        return ENOEXEC;
+
+    elf_phdr_t* phdrs = mem_alloc(ehdr->e_phnum * sizeof(elf_phdr_t), 0);
+    if (phdrs == nullptr)
+        return ENOMEM;
+    for (size_t i = 0; i < ehdr->e_phnum; i++)
+        memcpy(&phdrs[i], file + ehdr->e_phoff + i * ehdr->e_phentsize, sizeof(elf_phdr_t));
+
+    *out = phdrs;
+    return 0;
+}
+
+errno_t module_load(const void* data, size_t length, const char* cmdline, struct module** out) {
+    const uint8_t* file = data;
+    const size_t page_size = arch_mem_page_size();
+
+    elf_ehdr_t ehdr;
+    elf_phdr_t* phdrs;
+    errno_t header_status = read_headers(data, length, &ehdr, &phdrs);
+    if (header_status)
+        return header_status;
+
+    struct module* mod = mem_alloc(sizeof(struct module), 0);
+    if (mod == nullptr) {
+        mem_free(phdrs);
+        return ENOMEM;
+    }
+    mod->version = "";
+    mod->author = "";
+    mod->desc = "";
+    mod->license = "";
+
+    errno_t status = ENOEXEC;
+    bool listed = false;
+
+    uintptr_t load_min = UINTPTR_MAX;
+    uintptr_t load_end = 0;
+    size_t num_loads = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
+            continue;
+        if (phdr->p_filesz > phdr->p_memsz || !in_range(length, phdr->p_offset, phdr->p_filesz))
+            goto fail;
+
+        uintptr_t end;
+        if (ckd_add(&end, phdr->p_vaddr, phdr->p_memsz) || end > UINTPTR_MAX - page_size)
+            goto fail;
+
+        const uintptr_t start = ALIGN_DOWN(phdr->p_vaddr, page_size);
+        if (start < load_end)
+            goto fail;
+        if (load_min == UINTPTR_MAX)
+            load_min = start;
+        load_end = ALIGN_UP(end, page_size);
+        num_loads++;
+    }
+
+    if (num_loads == 0)
+        goto fail;
+
+    status = ENOMEM;
+    mod->segments = mem_alloc(num_loads * sizeof(struct module_segment), 0);
+    if (mod->segments == nullptr)
+        goto fail;
+
+    mod->base = vm_alloc(load_end - load_min);
+    if (mod->base == 0)
+        goto fail;
+    mod->size = load_end - load_min;
+    mod->bias = mod->base - load_min;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
+            continue;
+
+        const uintptr_t start = ALIGN_DOWN(phdr->p_vaddr, page_size);
+        const size_t size = ALIGN_UP(phdr->p_vaddr + phdr->p_memsz, page_size) - start;
+
+        phys_t phys;
+        status = mem_phys_alloc(size / page_size, 0, &phys);
+        if (status)
+            goto fail;
+
+        struct module_segment* seg = &mod->segments[mod->num_segments++];
+        *seg = (struct module_segment){.phys = phys, .virt = mod->bias + start, .size = size};
+
+        for (size_t off = 0; off < size; off += page_size) {
+            status = pmap_map(&kernel_vma.pmap, seg->virt + off, phys + off, PTE_READ | PTE_WRITE, CACHE_WRITE_BACK);
+            if (status)
+                goto fail;
+        }
+
+        memcpy((void*)(mod->bias + phdr->p_vaddr), file + phdr->p_offset, phdr->p_filesz);
+    }
+
+    status = ENOEXEC;
+
+    const elf_dyn_t* dyn = nullptr;
+    size_t num_dyn = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        if (phdrs[i].p_type != PT_DYNAMIC)
+            continue;
+        dyn = image_ptr(mod, phdrs[i].p_vaddr, phdrs[i].p_memsz);
+        num_dyn = phdrs[i].p_memsz / sizeof(elf_dyn_t);
+        break;
+    }
+    if (dyn == nullptr)
+        goto fail;
+
+    uint64_t dt_strtab = 0, dt_strsz = 0, dt_symtab = 0, dt_hash = 0, dt_soname = 0;
+    uint64_t dt_rela = 0, dt_relasz = 0, dt_relaent = 0;
+    uint64_t dt_jmprel = 0, dt_pltrelsz = 0, dt_pltrel = DT_RELA;
+    uint64_t dt_relr = 0, dt_relrsz = 0, dt_relrent = 0;
+    bool has_strtab = false, has_symtab = false, has_hash = false, has_soname = false;
+    bool has_rela = false, has_jmprel = false, has_relr = false, has_needed = false;
+
+    for (size_t i = 0; i < num_dyn && dyn[i].d_tag != DT_NULL; i++) {
+        const uint64_t val = dyn[i].d_un.d_val;
+        switch (dyn[i].d_tag) {
+        case DT_STRTAB:
+            dt_strtab = val;
+            has_strtab = true;
+            break;
+        case DT_STRSZ:
+            dt_strsz = val;
+            break;
+        case DT_SYMTAB:
+            dt_symtab = val;
+            has_symtab = true;
+            break;
+        case DT_HASH:
+            dt_hash = val;
+            has_hash = true;
+            break;
+        case DT_SONAME:
+            dt_soname = val;
+            has_soname = true;
+            break;
+        case DT_NEEDED:
+            has_needed = true;
+            break;
+        case DT_RELA:
+            dt_rela = val;
+            has_rela = true;
+            break;
+        case DT_RELASZ:
+            dt_relasz = val;
+            break;
+        case DT_RELAENT:
+            dt_relaent = val;
+            break;
+        case DT_JMPREL:
+            dt_jmprel = val;
+            has_jmprel = true;
+            break;
+        case DT_PLTRELSZ:
+            dt_pltrelsz = val;
+            break;
+        case DT_PLTREL:
+            dt_pltrel = val;
+            break;
+        case DT_RELR:
+            dt_relr = val;
+            has_relr = true;
+            break;
+        case DT_RELRSZ:
+            dt_relrsz = val;
+            break;
+        case DT_RELRENT:
+            dt_relrent = val;
+            break;
+        default:
+            break;
+        }
+    }
+
+    if (!has_strtab || !has_symtab || !has_hash || !has_soname || dt_strsz == 0 || dt_pltrel != DT_RELA)
+        goto fail;
+
+    const elf_hash_t* hash = image_ptr(mod, dt_hash, sizeof(elf_hash_t));
+    if (hash == nullptr)
+        goto fail;
+
+    mod->num_syms = hash->n_chain;
+    mod->symtab = image_ptr(mod, dt_symtab, (uint64_t)mod->num_syms * sizeof(elf_sym_t));
+    mod->strtab = image_ptr(mod, dt_strtab, dt_strsz);
+    mod->strtab_size = dt_strsz;
+    if (mod->symtab == nullptr || mod->strtab == nullptr) {
+        mod->num_syms = 0;
+        goto fail;
+    }
+
+    mod->name = sym_name(mod, dt_soname);
+    if (mod->name == nullptr || *mod->name == 0)
+        goto fail;
+
+    for (size_t i = 0; has_needed && i < num_dyn && dyn[i].d_tag != DT_NULL; i++) {
+        if (dyn[i].d_tag != DT_NEEDED)
+            continue;
+
+        // TODO: Modules can technically claim they have dependencies on other modules, but we don't support that yet.
+        const char* needed = sym_name(mod, dyn[i].d_un.d_val);
+        const char* kernel_name = ksym_kernel_name();
+        if (needed == nullptr || kernel_name == nullptr || strcmp(needed, kernel_name) != 0) {
+            log_err("%s: Modules can only depend on the kernel, not \"%s\"\n", mod->name, needed ? needed : "?");
+            goto fail;
+        }
+    }
+
+    if (has_relr) {
+        status = apply_relr(mod, dt_relr, dt_relrsz, dt_relrent);
+        if (status)
+            goto fail;
+    }
+    if (has_rela) {
+        status = apply_rela(mod, dt_rela, dt_relasz, dt_relaent);
+        if (status)
+            goto fail;
+    }
+    if (has_jmprel) {
+        status = apply_rela(mod, dt_jmprel, dt_pltrelsz, dt_relaent);
+        if (status)
+            goto fail;
+    }
+
+    spin_lock(&module_lock);
+    if (find_locked(mod->name) != nullptr) {
+        spin_unlock(&module_lock);
+        status = EEXIST;
+        goto fail;
+    }
+    tailq_insert_tail(&module_list, mod, link);
+    listed = true;
+    spin_unlock(&module_lock);
+
+    const struct cmdline_option* const* opts_start = nullptr;
+    const struct cmdline_option* const* opts_end = nullptr;
+
+    status = ENOEXEC;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        const char* str;
+        switch (phdr->p_type) {
+        case PT_MOD_VERSION:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->version = str;
+            break;
+        case PT_MOD_AUTHOR:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->author = str;
+            break;
+        case PT_MOD_DESC:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->desc = str;
+            break;
+        case PT_MOD_LICENSE:
+            if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
+                mod->license = str;
+            break;
+        case PT_MOD_CMDLINE:
+            if (phdr->p_memsz < sizeof(void*))
+                break;
+            opts_start = image_ptr(mod, phdr->p_vaddr, phdr->p_memsz);
+            if (opts_start == nullptr)
+                goto fail;
+            opts_end = opts_start + (phdr->p_memsz / sizeof(void*));
+            break;
+        case PT_MOD_EXIT: {
+            if (phdr->p_memsz < sizeof(void*))
+                break;
+            const void* slot = image_ptr(mod, phdr->p_vaddr, sizeof(void*));
+            if (slot == nullptr)
+                goto fail;
+            memcpy(&mod->exit, slot, sizeof(void*));
+            break;
+        }
+        default:
+            break;
+        }
+    }
+
+    if (image_ptr(mod, ehdr.e_entry, 1) == nullptr)
+        goto fail;
+
+    size_t seg_idx = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf_phdr_t* phdr = &phdrs[i];
+        if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
+            continue;
+
+        const struct module_segment* seg = &mod->segments[seg_idx++];
+        const enum pte_flags prot = prot_from_phdr(phdr);
+        for (size_t off = 0; off < seg->size; off += page_size) {
+            pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+            status = pmap_map(&kernel_vma.pmap, seg->virt + off, seg->phys + off, prot, CACHE_WRITE_BACK);
+            if (status)
+                goto fail;
+        }
+    }
+
+    if (cmdline == nullptr)
+        cmdline = "";
+    const size_t cmdline_len = strlen(cmdline);
+    mod->cmdline = mem_alloc(cmdline_len + 1, 0);
+    if (mod->cmdline == nullptr) {
+        status = ENOMEM;
+        goto fail;
+    }
+    memcpy(mod->cmdline, cmdline, cmdline_len);
+    if (opts_start != nullptr)
+        cmdline_parse_buf(mod->cmdline, cmdline_len, opts_start, opts_end);
+
+    log("Loaded module \"%s\" (%s, %s) at %p\n", mod->name, mod->desc, mod->version, (void*)mod->base);
+
+    errno_t (*entry)(struct module*) = (errno_t (*)(struct module*))(mod->bias + ehdr.e_entry);
+    status = entry(mod);
+    if (status) {
+        log_err("%s: Entry point returned %i\n", mod->name, status);
+        goto fail;
+    }
+
+    mem_free(phdrs);
+
+    if (out != nullptr)
+        *out = mod;
+    return 0;
+
+fail:
+    if (listed) {
+        spin_lock(&module_lock);
+        tailq_remove(&module_list, mod, link);
+        spin_unlock(&module_lock);
+    }
+    module_free(mod);
+    mem_free(phdrs);
+    return status;
+}
+
+errno_t module_unload(struct module* mod) {
+    if (mod == nullptr)
+        return EINVAL;
+
+    if (mod->exit == nullptr)
+        return ENOTSUP;
+
+    spin_lock(&module_lock);
+    tailq_remove(&module_list, mod, link);
+    spin_unlock(&module_lock);
+
+    errno_t status = mod->exit(mod);
+    if (status) {
+        log_err("%s: Exit function returned %i\n", mod->name, status);
+        spin_lock(&module_lock);
+        tailq_insert_tail(&module_list, mod, link);
+        spin_unlock(&module_lock);
+        return status;
+    }
+
+    log("Unloaded module \"%s\"\n", mod->name);
+    module_free(mod);
+    return 0;
+}
+
+struct module* module_find(const char* name) {
+    spin_lock(&module_lock);
+    struct module* mod = find_locked(name);
+    spin_unlock(&module_lock);
+    return mod;
+}

lib/panic.c

@@ -0,0 +1,73 @@
+#include <zinnia/alloc.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/ksyms.h>
+#include <zinnia/panic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <stdarg.h>
+#include <stdint.h>
+
+#define panic_log(msg, ...) kprintf("[    !!!!    ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
+
+static void print_backtrace() {
+    constexpr size_t max_depth = 32;
+    void** fp = __builtin_frame_address(0);
+
+    panic_log("Backtrace:\n");
+    for (size_t i = 0; i < max_depth; i++) {
+        if (!fp || (uintptr_t)fp < mem_hhdm_base || (uintptr_t)fp % alignof(void*))
+            break;
+
+        void* ret = fp[1];
+        if (!ret)
+            break;
+
+        const char* name;
+        size_t offset;
+        // The return address may already point past the end of a noreturn call.
+        if (ksym_lookup((uintptr_t)ret - 1, &name, &offset))
+            panic_log("  #%zu [%p] %s+0x%zx\n", i, ret, name, offset + 1);
+        else
+            panic_log("  #%zu [%p] <\?\?\?>\n", i, ret);
+
+        void** next = fp[0];
+        if (next <= fp)
+            break;
+        fp = next;
+    }
+}
+
+[[noreturn]]
+void panic(const char* msg, ...) {
+    // TODO: Stop all other CPUs with IPI.
+    irq_set_state(false);
+
+    struct percpu* cpu = percpu_get();
+    struct task* current = cpu->sched.current;
+    kprintf(
+        "[    !!!!    ] \e[31m"
+        "Kernel panic on CPU %zu (T%zu, \"%s\"):\e[0m\n",
+        cpu->id,
+        current ? current->id : 0,
+        current ? current->name : nullptr
+    );
+
+    kprintf("[    !!!!    ] \e[31m");
+
+    va_list args;
+    va_start(args, msg);
+    kvprintf(msg, args);
+    va_end(args);
+
+    kprintf("\e[0m");
+
+    print_backtrace();
+    panic_log("End of panic message\n");
+
+    initgraph_panic();
+
+    while (true) {
+        irq_wait();
+    }
+}

lib/percpu.c

@@ -0,0 +1,77 @@
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <arch/percpu.h>
+
+static _Atomic(size_t) num_cpus = 1;
+
+static size_t max_cpus = SIZE_MAX;
+static void smp_option(uintptr_t value) {
+    max_cpus = value;
+}
+CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option);
+
+[[__used, __section(".percpu.init")]]
+struct percpu percpu_bsp = {
+    .self = &percpu_bsp,
+    .id = 0,
+    .online = false,
+};
+
+struct percpu* percpu_new() {
+    const size_t pgsz = arch_mem_page_size();
+    const size_t stride = percpu_stride();
+    const size_t id = atomic_load(&num_cpus, ATOMIC_RELAXED);
+    if (id >= (UINTPTR_MAX - (uintptr_t)__ld_percpu_start) / stride)
+        return nullptr;
+
+    phys_t mem;
+    if (mem_phys_alloc(stride / pgsz, 0, &mem))
+        return nullptr;
+
+    const uintptr_t base = (uintptr_t)percpu_cpu(id);
+    for (size_t offset = 0; offset < stride; offset += pgsz) {
+        if (pmap_map(&kernel_vma.pmap, base + offset, mem + offset, PTE_READ | PTE_WRITE, CACHE_ANY)) {
+            while (offset) {
+                offset -= pgsz;
+                pmap_unmap(&kernel_vma.pmap, base + offset);
+            }
+            mem_phys_free(mem, stride / pgsz);
+            return nullptr;
+        }
+    }
+
+    struct percpu* cpu = (struct percpu*)base;
+    cpu->self = cpu;
+    cpu->id = id;
+    cpu->online = false;
+    atomic_store(&num_cpus, id + 1, ATOMIC_RELEASE);
+    return cpu;
+}
+
+size_t percpu_max_cpus() {
+    return max_cpus;
+}
+
+size_t percpu_count() {
+    return atomic_load(&num_cpus, ATOMIC_RELAXED);
+}
+
+struct percpu* percpu_get() {
+    return arch_percpu_get();
+}
+
+void percpu_bsp_init() {
+    arch_percpu_bsp_init();
+}
+
+void percpu_init(struct boot_info*) {
+    arch_percpu_init();
+}
+INIT_TASK(percpu_stage, "core.percpu", percpu_init);
+INIT_DEPENDS(percpu_stage, mem_stage);

lib/print.c

@@ -0,0 +1,495 @@
+#include <zinnia/clock.h>
+#include <zinnia/console.h>
+#include <zinnia/panic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/spin.h>
+#include <zinnia/stdlib.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) {
+    for (size_t i = 0; i < count; i++)
+        callback(ctx, &c, 1);
+}
+
+void print_fmt(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
+    // Amount of bytes written.
+    int32_t written = 0;
+
+    while (*fmt != '\0') {
+        size_t maxrem = INT32_MAX - written;
+
+        if (fmt[0] != '%' || fmt[1] == '%') {
+            if (fmt[0] == '%')
+                fmt++;
+            size_t amount = 1;
+            while (fmt[amount] && fmt[amount] != '%')
+                amount++;
+            if (maxrem < amount) {
+                return;
+            }
+            callback(ctx, fmt, amount);
+            fmt += amount;
+            written += amount;
+            continue;
+        }
+
+        const char* format_begun_at = fmt++;
+        bool force_sign = false;
+        bool blank_sign = false;
+        bool write_prefix = false;
+        bool pad_zero = false;
+        bool left_align = false;
+        size_t width = 0;
+        bool has_precision = false;
+        size_t precision = 0;
+        size_t size = sizeof(uint32_t);
+        char number[32];
+check_fmt:
+        memset(number, 0, sizeof(number));
+
+        switch (*fmt) {
+        // Flags
+        case '+': {
+            force_sign = true;
+            fmt++;
+            goto check_fmt;
+        }
+        case ' ': {
+            blank_sign = true;
+            fmt++;
+            goto check_fmt;
+        }
+        case '#': {
+            write_prefix = true;
+            fmt++;
+            goto check_fmt;
+        }
+        case '0': {
+            pad_zero = true;
+            fmt++;
+            goto check_fmt;
+        }
+        case '-': {
+            left_align = true;
+            fmt++;
+            goto check_fmt;
+        }
+        // Width
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+        case '5':
+        case '6':
+        case '7':
+        case '8':
+        case '9': {
+            width = 0;
+            while (*fmt >= '0' && *fmt <= '9') {
+                width = width * 10 + (*fmt - '0');
+                fmt++;
+            }
+
+            goto check_fmt;
+        }
+        // Precision
+        case '.': {
+            has_precision = true;
+            fmt++;
+            precision = 0;
+            if (*fmt == '*') {
+                const int32_t arg = va_arg(args, int32_t);
+                if (arg < 0)
+                    has_precision = false;
+                else
+                    precision = arg;
+                fmt++;
+            } else {
+                while (*fmt >= '0' && *fmt <= '9') {
+                    precision = precision * 10 + (*fmt - '0');
+                    fmt++;
+                }
+            }
+            goto check_fmt;
+        }
+        // Length
+        case 'h': {
+            if (fmt[1] == 'h') {
+                size = sizeof(uint8_t);
+                fmt++;
+            } else
+                size = sizeof(uint16_t);
+            fmt++;
+            goto check_fmt;
+        }
+        case 'l': {
+            // There is no difference between `long int` and `long long int`.
+            if (fmt[1] == 'l')
+                fmt++;
+            fmt++;
+            size = sizeof(uint64_t);
+
+            goto check_fmt;
+        }
+        case 'z': {
+            size = sizeof(size_t);
+            fmt++;
+            goto check_fmt;
+        }
+        // Character
+        case 'c': {
+            char c = (char)va_arg(args, int32_t);
+            const size_t pad = width > 1 ? width - 1 : 0;
+
+            if (!left_align)
+                print_pad(callback, ctx, ' ', pad);
+            callback(ctx, &c, sizeof(c));
+            if (left_align)
+                print_pad(callback, ctx, ' ', pad);
+
+            written += 1 + pad;
+            fmt++;
+            break;
+        }
+        // String of characters
+        case 's': {
+            const char* str = va_arg(args, const char*);
+            if (!str)
+                str = "(null)";
+            size_t len = 0;
+            if (has_precision)
+                len = strnlen(str, precision);
+            else
+                len = strlen(str);
+            const size_t pad = width > len ? width - len : 0;
+
+            if (!left_align)
+                print_pad(callback, ctx, ' ', pad);
+            callback(ctx, str, len);
+            if (left_align)
+                print_pad(callback, ctx, ' ', pad);
+
+            written += len + pad;
+            fmt++;
+            break;
+        }
+        // Signed decimal integer
+        case 'i':
+        case 'd': {
+            switch (size) {
+            case sizeof(int8_t):
+                ltoa((int8_t)va_arg(args, int32_t), number, 10);
+                break;
+            case sizeof(int16_t):
+                ltoa((int16_t)va_arg(args, int32_t), number, 10);
+                break;
+            case sizeof(int32_t):
+                ltoa((int32_t)va_arg(args, int32_t), number, 10);
+                break;
+            default:
+                ltoa(va_arg(args, int64_t), number, 10);
+                break;
+            }
+            goto print_num;
+        }
+        // Unsigned decimal integer
+        case 'u': {
+            switch (size) {
+            case sizeof(uint8_t):
+                lutoa((uint8_t)va_arg(args, uint32_t), number, 10);
+                break;
+            case sizeof(uint16_t):
+                lutoa((uint16_t)va_arg(args, uint32_t), number, 10);
+                break;
+            case sizeof(uint32_t):
+                lutoa(va_arg(args, uint32_t), number, 10);
+                break;
+            case sizeof(uint64_t):
+                lutoa(va_arg(args, uint64_t), number, 10);
+                break;
+            }
+            goto print_num;
+        }
+        // Unsigned hexadecimal integer
+        case 'X':
+        case 'x': {
+            switch (size) {
+            case sizeof(uint8_t):
+                lutoa((uint8_t)va_arg(args, uint32_t), number, 16);
+                break;
+            case sizeof(uint16_t):
+                lutoa((uint16_t)va_arg(args, uint32_t), number, 16);
+                break;
+            case sizeof(uint32_t):
+                lutoa(va_arg(args, uint32_t), number, 16);
+                break;
+            case sizeof(uint64_t):
+                lutoa(va_arg(args, uint64_t), number, 16);
+                break;
+            }
+print_num:
+            const char* digits = number;
+            char sign = 0;
+            if (digits[0] == '-') {
+                sign = '-';
+                digits++;
+            } else if (force_sign) {
+                sign = '+';
+            } else if (blank_sign) {
+                sign = ' ';
+            }
+
+            const size_t len = strlen(digits);
+            size_t zeros = (has_precision && precision > len) ? precision - len : 0;
+            const size_t total = (sign ? 1 : 0) + (write_prefix ? 2 : 0) + zeros + len;
+            size_t pad = width > total ? width - total : 0;
+            if (pad_zero && !left_align && !has_precision) {
+                zeros += pad;
+                pad = 0;
+            }
+
+            if (!left_align)
+                print_pad(callback, ctx, ' ', pad);
+            if (sign)
+                callback(ctx, &sign, 1);
+            if (write_prefix)
+                callback(ctx, "0x", 2);
+            print_pad(callback, ctx, '0', zeros);
+            callback(ctx, digits, len);
+            if (left_align)
+                print_pad(callback, ctx, ' ', pad);
+
+            written += width > total ? width : total;
+            fmt++;
+            break;
+        }
+        // Pointer address
+        case 'p': {
+            const size_t num = va_arg(args, size_t);
+            const size_t buf_size = sizeof(size_t) * 2 + 1;
+            char str[buf_size];
+            lutoa(num, number, 0x10);
+            const size_t len = strlen(number); // Get the length of the final number.
+            for (size_t i = 0; i < buf_size; i++)
+                str[i] = '0';
+            size_t offset = sizeof(str) - len - 1;
+
+            callback(ctx, str, offset);
+            callback(ctx, number, len);
+
+            fmt++;
+            written += len;
+            break;
+        }
+        // The number of characters written so far is stored in the pointed location.
+        case 'n': {
+            int32_t* const ptr = va_arg(args, int32_t*);
+            *ptr = written;
+            fmt++;
+            break;
+        }
+        // No format, just normal text.
+        default: {
+            size_t len = fmt - format_begun_at;
+            if (*fmt)
+                len++;
+
+            callback(ctx, format_begun_at, len);
+
+            written += len;
+            fmt = format_begun_at + len;
+            break;
+        }
+        }
+    }
+}
+
+static void printf_console(void*, const char* msg, size_t len) {
+    console_write(msg, len);
+}
+
+static struct spinlock print_lock = {};
+
+void kprintf(const char* message, ...) {
+    spin_lock(&print_lock);
+
+    va_list args;
+    va_start(args, message);
+    print_fmt(printf_console, nullptr, message, args);
+    va_end(args);
+
+    spin_unlock(&print_lock);
+}
+
+struct snprintf_ctx {
+    char* buf;
+    size_t size;
+    size_t len;
+};
+
+static void snprintf_write(void* ctx, const char* str, size_t len) {
+    struct snprintf_ctx* c = ctx;
+    // Always keep room for the terminator.
+    if (c->len < c->size) {
+        size_t room = c->size - c->len - 1;
+        memcpy(c->buf + c->len, str, len < room ? len : room);
+    }
+    c->len += len;
+}
+
+size_t kvsnprintf(char* buf, size_t size, const char* format, va_list args) {
+    struct snprintf_ctx ctx = {.buf = buf, .size = size};
+    print_fmt(snprintf_write, &ctx, format, args);
+    if (size > 0)
+        buf[ctx.len < size ? ctx.len : size - 1] = '\0';
+    return ctx.len;
+}
+
+size_t ksnprintf(char* buf, size_t size, const char* format, ...) {
+    va_list args;
+    va_start(args, format);
+    size_t len = kvsnprintf(buf, size, format, args);
+    va_end(args);
+    return len;
+}
+
+void kvprintf(const char* message, va_list args) {
+    spin_lock(&print_lock);
+
+    print_fmt(printf_console, nullptr, message, args);
+
+    spin_unlock(&print_lock);
+}
+
+struct log_record {
+    uint64_t time_ns;
+    uint32_t tid;
+    uint16_t len;
+    uint8_t level;
+};
+
+#define LOG_BUFFER_SIZE (64 * 1024)
+#define LOG_MESSAGE_MAX 255
+
+static uint8_t log_buffer[LOG_BUFFER_SIZE];
+
+// Protected by print_lock.
+static size_t log_head = 0;
+static size_t log_tail = 0;
+
+static void log_ring_copy_in(const void* src, size_t len) {
+    for (size_t i = 0; i < len; i++)
+        log_buffer[(log_head + i) % LOG_BUFFER_SIZE] = ((const uint8_t*)src)[i];
+    log_head += len;
+}
+
+static void log_ring_peek(size_t offset, void* dst, size_t len) {
+    for (size_t i = 0; i < len; i++)
+        ((uint8_t*)dst)[i] = log_buffer[(offset + i) % LOG_BUFFER_SIZE];
+}
+
+static void log_store(const struct log_record* rec, const char* msg) {
+    const size_t total = sizeof(*rec) + rec->len;
+
+    // Drop the oldest records until the new one fits.
+    while (LOG_BUFFER_SIZE - (log_head - log_tail) < total) {
+        struct log_record old;
+        log_ring_peek(log_tail, &old, sizeof(old));
+        log_tail += sizeof(old) + old.len;
+    }
+
+    log_ring_copy_in(rec, sizeof(*rec));
+    log_ring_copy_in(msg, rec->len);
+}
+
+struct log_line {
+    char buf[LOG_MESSAGE_MAX];
+    size_t len;
+};
+
+static void log_line_append(void* ctx, const char* str, size_t len) {
+    struct log_line* line = ctx;
+    if (len > sizeof(line->buf) - line->len)
+        len = sizeof(line->buf) - line->len;
+    memcpy(line->buf + line->len, str, len);
+    line->len += len;
+}
+
+// ANSI colors for each level.
+static const char* const log_level_colors[] = {
+    [LOG_DEBUG] = "\e[90m",
+    [LOG_STATUS] = "\e[32m",
+    [LOG_INFO] = "",
+    [LOG_WARN] = "\e[33m",
+    [LOG_ERR] = "\e[31m",
+    [LOG_PANIC] = "\e[1;31m",
+};
+
+static void log_cb_printf(log_fn_t callback, void* ctx, const char* fmt, ...) {
+    va_list args;
+    va_start(args, fmt);
+    print_fmt(callback, ctx, fmt, args);
+    va_end(args);
+}
+
+// Formats a single record and passes it to `callback`.
+static void log_emit(log_fn_t callback, void* ctx, const struct log_record* rec, const char* msg) {
+    log_cb_printf(
+        callback,
+        ctx,
+        "[%5lu.%06lu] %s",
+        rec->time_ns / 1'000'000'000,
+        rec->time_ns / 1'000 % 1'000'000,
+        log_level_colors[rec->level]
+    );
+    callback(ctx, msg, rec->len);
+    callback(ctx, "\e[0m\n", 5);
+}
+
+void log_replay(log_fn_t callback, void* ctx) {
+    char msg[LOG_MESSAGE_MAX];
+    for (size_t offset = log_tail; offset != log_head;) {
+        struct log_record rec;
+        log_ring_peek(offset, &rec, sizeof(rec));
+        log_ring_peek(offset + sizeof(rec), msg, rec.len);
+        log_emit(callback, ctx, &rec, msg);
+        offset += sizeof(rec) + rec.len;
+    }
+}
+
+void log_vwrite(enum log_level level, const char* message, va_list args) {
+    log_vwrite_prefixed(level, "", message, args);
+}
+
+void log_vwrite_prefixed(enum log_level level, const char* prefix, const char* message, va_list args) {
+    struct log_line line = {};
+    log_line_append(&line, prefix, strlen(prefix));
+    print_fmt(log_line_append, &line, message, args);
+    if (line.len > 0 && line.buf[line.len - 1] == '\n')
+        line.len--;
+
+    auto cur = percpu_get()->sched.current;
+    const struct log_record rec = {
+        .time_ns = clock_get_elapsed_ns(),
+        .tid = cur ? cur->id : 0,
+        .len = line.len,
+        .level = level,
+    };
+
+    spin_lock(&print_lock);
+    log_store(&rec, line.buf);
+    log_emit(printf_console, nullptr, &rec, line.buf);
+    spin_unlock(&print_lock);
+}
+
+void log_write(enum log_level level, const char* message, ...) {
+    va_list args;
+    va_start(args, message);
+    log_vwrite(level, message, args);
+    va_end(args);
+}

lib/radix.c

@@ -0,0 +1,255 @@
+#include <zinnia/alloc.h>
+#include <zinnia/radix.h>
+
+static bool covers(const struct radix_node* node, uint64_t index) {
+    return node->shift + RADIX_BITS >= 64 || (index >> (node->shift + RADIX_BITS)) == 0;
+}
+
+static size_t slot_of(const struct radix_node* node, uint64_t index) {
+    return (index >> node->shift) & (RADIX_SLOTS - 1);
+}
+
+static struct radix_node* node_alloc(uint8_t shift, struct radix_node* parent, uint8_t offset) {
+    struct radix_node* node = mem_alloc(sizeof(struct radix_node), 0);
+    if (!node)
+        return nullptr;
+    node->shift = shift;
+    node->parent = parent;
+    node->offset = offset;
+    return node;
+}
+
+static void free_subtree(struct radix_node* node) {
+    if (node->shift > 0) {
+        for (size_t i = 0; i < RADIX_SLOTS; i++) {
+            if (node->slots[i])
+                free_subtree(node->slots[i]);
+        }
+    }
+    mem_free(node);
+}
+
+void radix_init(struct radix_tree* tree) {
+    tree->root = nullptr;
+}
+
+void radix_destroy(struct radix_tree* tree) {
+    if (tree->root)
+        free_subtree(tree->root);
+    tree->root = nullptr;
+}
+
+bool radix_empty(struct radix_tree* tree) {
+    return tree->root == nullptr;
+}
+
+static struct radix_node* leaf_of(struct radix_tree* tree, uint64_t index) {
+    struct radix_node* node = tree->root;
+    if (!node || !covers(node, index))
+        return nullptr;
+    while (node->shift > 0) {
+        node = node->slots[slot_of(node, index)];
+        if (!node)
+            return nullptr;
+    }
+    return node;
+}
+
+void* radix_lookup(struct radix_tree* tree, uint64_t index) {
+    struct radix_node* leaf = leaf_of(tree, index);
+    return leaf ? leaf->slots[slot_of(leaf, index)] : nullptr;
+}
+
+void** radix_slot(struct radix_tree* tree, uint64_t index) {
+    struct radix_node* leaf = leaf_of(tree, index);
+    return leaf ? &leaf->slots[slot_of(leaf, index)] : nullptr;
+}
+
+static errno_t grow(struct radix_tree* tree, uint64_t index) {
+    if (!tree->root) {
+        uint8_t shift = 0;
+        while (shift + RADIX_BITS < 64 && (index >> (shift + RADIX_BITS)) != 0)
+            shift += RADIX_BITS;
+        tree->root = node_alloc(shift, nullptr, 0);
+        return tree->root ? 0 : ENOMEM;
+    }
+
+    while (!covers(tree->root, index)) {
+        struct radix_node* old = tree->root;
+        struct radix_node* top = node_alloc(old->shift + RADIX_BITS, nullptr, 0);
+        if (!top)
+            return ENOMEM;
+        top->slots[0] = old;
+        top->count = 1;
+        for (size_t t = 0; t < RADIX_NUM_TAGS; t++) {
+            if (old->tags[t])
+                top->tags[t] = 1;
+        }
+        old->parent = top;
+        old->offset = 0;
+        tree->root = top;
+    }
+    return 0;
+}
+
+errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value) {
+    errno_t status = grow(tree, index);
+    if (status)
+        return status;
+
+    struct radix_node* node = tree->root;
+    while (node->shift > 0) {
+        const size_t slot = slot_of(node, index);
+        struct radix_node* child = node->slots[slot];
+        if (!child) {
+            child = node_alloc(node->shift - RADIX_BITS, node, slot);
+            if (!child)
+                return ENOMEM;
+            node->slots[slot] = child;
+            node->count++;
+        }
+        node = child;
+    }
+
+    const size_t slot = slot_of(node, index);
+    if (node->slots[slot])
+        return EEXIST;
+    node->slots[slot] = value;
+    node->count++;
+    return 0;
+}
+
+static void clear_tag_upwards(struct radix_node* node, size_t slot, enum radix_tag tag) {
+    while (node) {
+        node->tags[tag] &= ~((uint64_t)1 << slot);
+        if (node->tags[tag])
+            return;
+        slot = node->offset;
+        node = node->parent;
+    }
+}
+
+void* radix_remove(struct radix_tree* tree, uint64_t index) {
+    struct radix_node* node = leaf_of(tree, index);
+    if (!node)
+        return nullptr;
+
+    size_t slot = slot_of(node, index);
+    void* value = node->slots[slot];
+    if (!value)
+        return nullptr;
+
+    for (size_t t = 0; t < RADIX_NUM_TAGS; t++) {
+        if (node->tags[t] & ((uint64_t)1 << slot))
+            clear_tag_upwards(node, slot, t);
+    }
+
+    node->slots[slot] = nullptr;
+    node->count--;
+    while (node->count == 0) {
+        struct radix_node* parent = node->parent;
+        if (!parent) {
+            tree->root = nullptr;
+            mem_free(node);
+            break;
+        }
+        slot = node->offset;
+        parent->slots[slot] = nullptr;
+        parent->count--;
+        mem_free(node);
+        node = parent;
+    }
+    return value;
+}
+
+void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
+    struct radix_node* node = leaf_of(tree, index);
+    if (!node || !node->slots[slot_of(node, index)])
+        return;
+    size_t slot = slot_of(node, index);
+    while (node) {
+        const uint64_t bit = (uint64_t)1 << slot;
+        if (node->tags[tag] & bit)
+            return;
+        node->tags[tag] |= bit;
+        slot = node->offset;
+        node = node->parent;
+    }
+}
+
+void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
+    struct radix_node* node = leaf_of(tree, index);
+    if (!node)
+        return;
+    const size_t slot = slot_of(node, index);
+    if (node->tags[tag] & ((uint64_t)1 << slot))
+        clear_tag_upwards(node, slot, tag);
+}
+
+bool radix_tag_get(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
+    struct radix_node* node = leaf_of(tree, index);
+    return node && (node->tags[tag] & ((uint64_t)1 << slot_of(node, index)));
+}
+
+bool radix_tagged(struct radix_tree* tree, enum radix_tag tag) {
+    return tree->root && tree->root->tags[tag];
+}
+
+struct gang {
+    size_t max;
+    size_t found;
+    void** out;
+    uint64_t* out_index;
+    int tag;
+};
+
+static void gang_walk(struct radix_node* node, uint64_t base, uint64_t first, struct gang* g) {
+    size_t start = 0;
+    if (first > base) {
+        const uint64_t delta = (first - base) >> node->shift;
+        if (delta >= RADIX_SLOTS)
+            return;
+        start = delta;
+    }
+
+    for (size_t i = start; i < RADIX_SLOTS && g->found < g->max; i++) {
+        if (!node->slots[i])
+            continue;
+        if (g->tag >= 0 && !(node->tags[g->tag] & ((uint64_t)1 << i)))
+            continue;
+        const uint64_t index = base + ((uint64_t)i << node->shift);
+        if (node->shift == 0) {
+            if (g->out)
+                g->out[g->found] = node->slots[i];
+            if (g->out_index)
+                g->out_index[g->found] = index;
+            g->found++;
+        } else {
+            gang_walk(node->slots[i], index, first > index ? first : index, g);
+        }
+    }
+}
+
+static size_t gang_lookup(struct radix_tree* tree, uint64_t first, int tag, size_t max, void** out, uint64_t* idx) {
+    struct radix_node* root = tree->root;
+    if (!root || max == 0 || !covers(root, first))
+        return 0;
+    struct gang g = {.max = max, .out = out, .out_index = idx, .tag = tag};
+    gang_walk(root, 0, first, &g);
+    return g.found;
+}
+
+size_t radix_gang_lookup(struct radix_tree* tree, uint64_t first, size_t max, void** out, uint64_t* out_index) {
+    return gang_lookup(tree, first, -1, max, out, out_index);
+}
+
+size_t radix_gang_lookup_tag(
+    struct radix_tree* tree,
+    uint64_t first,
+    enum radix_tag tag,
+    size_t max,
+    void** out,
+    uint64_t* out_index
+) {
+    return gang_lookup(tree, first, tag, max, out, out_index);
+}

lib/random.c

@@ -0,0 +1,61 @@
+#define pr_fmt(fmt) "random: " fmt
+
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
+#include <zinnia/print.h>
+#include <zinnia/random.h>
+#include <zinnia/spin.h>
+#include <string.h>
+
+static _Atomic(uint64_t) state = 0x9E37'79B9'7F4A'7C15;
+
+static bool dumb_random(struct random_source*, uint64_t* out) {
+    uint64_t z = atomic_fetch_add(&state, 0x9E37'79B9'7F4A'7C15, ATOMIC_RELAXED);
+    z ^= clock_get_elapsed_ns() * 0x2545'F491'4F6C'DD1D;
+    z = (z ^ (z >> 30)) * 0xBF58'476D'1CE4'E5B9;
+    z = (z ^ (z >> 27)) * 0x94D0'49BB'1331'11EB;
+    *out = z ^ (z >> 31);
+    return true;
+}
+
+static struct random_source fallback_source = {
+    .name = "fallback",
+    .quality = 0,
+    .get_u64 = dumb_random,
+};
+
+static _Atomic(struct random_source*) active_source = &fallback_source;
+static struct spinlock lock = {};
+
+bool random_register(struct random_source* source) {
+    spin_lock(&lock);
+
+    struct random_source* active = atomic_load(&active_source, ATOMIC_RELAXED);
+    if (source->quality <= active->quality) {
+        spin_unlock(&lock);
+        return false;
+    }
+    atomic_store(&active_source, source, ATOMIC_RELEASE);
+
+    spin_unlock(&lock);
+
+    log("Switching to random source \"%s\"\n", source->name);
+
+    return true;
+}
+
+void random_fill(void* buf, size_t len) {
+    struct random_source* source = atomic_load(&active_source, ATOMIC_ACQUIRE);
+    uint8_t* out = buf;
+    while (len) {
+        uint64_t value;
+        source->get_u64(source, &value);
+
+        const size_t chunk = len < sizeof(value) ? len : sizeof(value);
+        memcpy(out, &value, chunk);
+        out += chunk;
+        len -= chunk;
+    }
+}
+
+INIT_TASK(random_stage, "core.random");

lib/ringbuf.c

@@ -0,0 +1,94 @@
+#include <zinnia/alloc.h>
+#include <zinnia/ringbuf.h>
+#include <zinnia/utils.h>
+#include <string.h>
+
+errno_t ringbuf_init(struct ringbuf* rb, size_t capacity) {
+    rb->data = mem_alloc(capacity, ALLOC_NOZERO);
+    if (!rb->data)
+        return ENOMEM;
+    rb->capacity = capacity;
+    rb->head = 0;
+    rb->len = 0;
+    return 0;
+}
+
+void ringbuf_fini(struct ringbuf* rb) {
+    mem_free(rb->data);
+    rb->data = nullptr;
+    rb->capacity = 0;
+    rb->len = 0;
+}
+
+static size_t tail(const struct ringbuf* rb) {
+    return (rb->head + rb->len) % rb->capacity;
+}
+
+static size_t write_chunk(const struct ringbuf* rb, size_t len) {
+    return MIN(len, MIN(ringbuf_space(rb), rb->capacity - tail(rb)));
+}
+
+static size_t read_chunk(const struct ringbuf* rb, size_t len) {
+    return MIN(len, MIN(rb->len, rb->capacity - rb->head));
+}
+
+static void advance_read(struct ringbuf* rb, size_t len) {
+    rb->head = (rb->head + len) % rb->capacity;
+    rb->len -= len;
+    if (rb->len == 0)
+        rb->head = 0;
+}
+
+size_t ringbuf_write(struct ringbuf* rb, const void* src, size_t len) {
+    size_t done = 0;
+    size_t chunk;
+    while ((chunk = write_chunk(rb, len - done))) {
+        memcpy(rb->data + tail(rb), (const uint8_t*)src + done, chunk);
+        rb->len += chunk;
+        done += chunk;
+    }
+    return done;
+}
+
+size_t ringbuf_read(struct ringbuf* rb, void* dst, size_t len) {
+    size_t done = 0;
+    size_t chunk;
+    while ((chunk = read_chunk(rb, len - done))) {
+        memcpy((uint8_t*)dst + done, rb->data + rb->head, chunk);
+        advance_read(rb, chunk);
+        done += chunk;
+    }
+    return done;
+}
+
+errno_t ringbuf_write_iter(struct ringbuf* rb, iovec_iter_t* iter, size_t len, size_t* out_written) {
+    size_t done = 0;
+    errno_t status = 0;
+    size_t chunk;
+    while (!status && (chunk = write_chunk(rb, len - done))) {
+        size_t copied;
+        status = iovec_iter_copy_from(iter, rb->data + tail(rb), chunk, &copied);
+        rb->len += copied;
+        done += copied;
+        if (copied < chunk)
+            break;
+    }
+    *out_written = done;
+    return done ? 0 : status;
+}
+
+errno_t ringbuf_read_iter(struct ringbuf* rb, iovec_iter_t* iter, size_t len, size_t* out_read) {
+    size_t done = 0;
+    errno_t status = 0;
+    size_t chunk;
+    while (!status && (chunk = read_chunk(rb, len - done))) {
+        size_t copied;
+        status = iovec_iter_copy_to(iter, rb->data + rb->head, chunk, &copied);
+        advance_read(rb, copied);
+        done += copied;
+        if (copied < chunk)
+            break;
+    }
+    *out_read = done;
+    return done ? 0 : status;
+}

lib/runtime.c

@@ -0,0 +1,93 @@
+// Compiler runtime support.
+
+#include <stdint.h>
+
+uint64_t __udivdi3(uint64_t dividend, uint64_t divisor) {
+    if (divisor == 0)
+        return 0;
+
+    if (dividend < divisor)
+        return 0;
+
+    unsigned shift = __builtin_clzll(divisor) - __builtin_clzll(dividend);
+
+    divisor <<= shift;
+
+    uint64_t quotient = 0;
+
+    for (;;) {
+        if (dividend >= divisor) {
+            dividend -= divisor;
+            quotient |= (uint64_t)1 << shift;
+        }
+
+        if (shift == 0)
+            break;
+
+        shift--;
+        divisor >>= 1;
+    }
+
+    return quotient;
+}
+
+static unsigned clz128(__uint128_t value) {
+    uint64_t high = (uint64_t)(value >> 64);
+
+    if (high != 0)
+        return (unsigned)__builtin_clzll(high);
+
+    return 64 + (unsigned)__builtin_clzll((uint64_t)value);
+}
+
+__uint128_t __udivmodti4(__uint128_t dividend, __uint128_t divisor, __uint128_t* remainder) {
+    if (divisor == 0) {
+        if (remainder)
+            *remainder = dividend;
+
+        return 0;
+    }
+
+    if (dividend < divisor) {
+        if (remainder)
+            *remainder = dividend;
+
+        return 0;
+    }
+
+    unsigned shift = clz128(divisor) - clz128(dividend);
+
+    divisor <<= shift;
+
+    __uint128_t quotient = 0;
+
+    for (;;) {
+        if (dividend >= divisor) {
+            dividend -= divisor;
+            quotient |= (__uint128_t)1 << shift;
+        }
+
+        if (shift == 0)
+            break;
+
+        shift--;
+        divisor >>= 1;
+    }
+
+    if (remainder)
+        *remainder = dividend;
+
+    return quotient;
+}
+
+__uint128_t __udivti3(__uint128_t dividend, __uint128_t divisor) {
+    return __udivmodti4(dividend, divisor, nullptr);
+}
+
+__uint128_t __umodti3(__uint128_t dividend, __uint128_t divisor) {
+    __uint128_t remainder;
+
+    __udivmodti4(dividend, divisor, &remainder);
+
+    return remainder;
+}

lib/spin.c

@@ -0,0 +1,34 @@
+#include <zinnia/atomic.h>
+#include <zinnia/irq.h>
+#include <zinnia/spin.h>
+#include <arch/cpu.h>
+
+void spin_init(struct spinlock* spin) {
+    spin->next = 0;
+    spin->owner = 0;
+}
+
+void spin_lock(struct spinlock* spin) {
+    irq_lock();
+    uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED);
+
+    while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) {
+        arch_cpu_pause();
+    }
+}
+
+bool spin_trylock(struct spinlock* spin) {
+    irq_lock();
+    uint32_t owner = atomic_load(&spin->owner, ATOMIC_RELAXED);
+    uint32_t expected = owner;
+    if (atomic_compare_exchange(&spin->next, &expected, owner + 1, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return true;
+    irq_unlock();
+    return false;
+}
+
+void spin_unlock(struct spinlock* spin) {
+    uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED);
+    atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE);
+    irq_unlock();
+}

lib/stdlib.c

@@ -0,0 +1,102 @@
+#include <zinnia/stdlib.h>
+#include <stdint.h>
+#include <string.h>
+
+static void reverse(char* s) {
+    int i, j;
+    char c;
+
+    for (i = 0, j = strlen(s) - 1; i < j; i++, j--) {
+        c = s[i];
+        s[i] = s[j];
+        s[j] = c;
+    }
+}
+
+static int char_to_digit(char c) {
+    if (c >= '0' && c <= '9')
+        return c - '0';
+    if (c >= 'a' && c <= 'z')
+        return c - 'a' + 10;
+    if (c >= 'A' && c <= 'Z')
+        return c - 'A' + 10;
+    return -1;
+}
+
+unsigned long atolu(const char* str, int base) {
+    unsigned long result = 0;
+
+    if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
+        str += 2;
+    }
+
+    while (*str) {
+        int digit = char_to_digit(*str++);
+        if (digit < 0 || digit >= base)
+            break;
+
+        result = result * base + digit;
+    }
+
+    return result;
+}
+
+long atol(const char* str, int base) {
+    int neg = 0;
+
+    if (*str == '-') {
+        neg = 1;
+        str++;
+    }
+
+    unsigned long val = atolu(str, base);
+    return neg ? -(long)val : (long)val;
+}
+
+char* lutoa(uint64_t value, char* str, int base) {
+    if (base < 2 || base > 36)
+        return nullptr;
+
+    size_t i = 0;
+
+    do {
+        uint64_t digit = value % base;
+        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
+        value /= base;
+    } while (value);
+
+    str[i] = '\0';
+    reverse(str);
+    return str;
+}
+
+char* ltoa(int64_t value, char* str, int base) {
+    if (base < 2 || base > 36)
+        return nullptr;
+
+    uint64_t magnitude;
+    int negative = 0;
+
+    if (value < 0) {
+        negative = 1;
+        magnitude = (uint64_t)(-(value + 1)) + 1;
+        // avoids INT64_MIN overflow
+    } else {
+        magnitude = (uint64_t)value;
+    }
+
+    size_t i = 0;
+
+    do {
+        uint64_t digit = magnitude % base;
+        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
+        magnitude /= base;
+    } while (magnitude);
+
+    if (negative)
+        str[i++] = '-';
+
+    str[i] = '\0';
+    reverse(str);
+    return str;
+}

lib/string.c

@@ -0,0 +1,141 @@
+#include <zinnia/compiler.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+#include <stdint.h>
+
+[[__hot]]
+size_t strlen(const char* str) {
+    size_t result = 0;
+    while (*str++) {
+        result++;
+    }
+    return result;
+}
+
+[[__hot]]
+size_t strnlen(const char* str, size_t len) {
+    size_t result = 0;
+    while (result < len && *str++) {
+        result++;
+    }
+    return result;
+}
+
+[[__hot]]
+int memcmp(const void* s1, const void* s2, size_t size) {
+    const uint8_t* s1ptr = s1;
+    const uint8_t* s2ptr = s2;
+
+    for (size_t i = 0; i < size; i++) {
+        if (s1ptr[i] != s2ptr[i])
+            return (int)s1ptr[i] - (int)s2ptr[i];
+    }
+    return 0;
+}
+
+typedef size_t word_t [[__may_alias]]
+;
+
+[[__hot]]
+void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t* s = src;
+
+    // Word copies are only possible if both pointers can be aligned at the same time.
+    if (((uintptr_t)d - (uintptr_t)s) % sizeof(word_t) == 0) {
+        while (n && (uintptr_t)d % sizeof(word_t) != 0) {
+            *d++ = *s++;
+            n--;
+        }
+
+        word_t* word_dest = (word_t*)d;
+        const word_t* word_src = (const word_t*)s;
+        for (; n >= sizeof(word_t); n -= sizeof(word_t))
+            *word_dest++ = *word_src++;
+
+        d = (uint8_t*)word_dest;
+        s = (const uint8_t*)word_src;
+    }
+
+    while (n--)
+        *d++ = *s++;
+
+    return dest;
+}
+
+[[__hot]]
+void* memmove(void* dstptr, const void* srcptr, size_t size) {
+    uint8_t* dst = (uint8_t*)dstptr;
+    const uint8_t* src = (const uint8_t*)srcptr;
+
+    if ((uintptr_t)dst < (uintptr_t)src) {
+        for (size_t i = 0; i < size; i++)
+            dst[i] = src[i];
+    } else {
+        for (size_t i = size; i != 0; i--)
+            dst[i - 1] = src[i - 1];
+    }
+    return dstptr;
+}
+
+[[__hot]]
+void* memset(void* dest, int value, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t byte = (uint8_t)value;
+
+    while (n && (uintptr_t)d % sizeof(word_t) != 0) {
+        *d++ = byte;
+        n--;
+    }
+
+    const word_t word_value = (SIZE_MAX / 0xFF) * byte;
+    word_t* word_dest = (word_t*)d;
+    for (; n >= sizeof(word_t); n -= sizeof(word_t))
+        *word_dest++ = word_value;
+
+    d = (uint8_t*)word_dest;
+    while (n--)
+        *d++ = byte;
+
+    return dest;
+}
+
+char* strncpy(char* restrict dst, const char* restrict src, size_t len) {
+    size_t src_len = strnlen(src, len);
+    memcpy(dst, src, src_len);
+    memset(dst + src_len, 0, len - src_len);
+    return dst;
+}
+
+int strncmp(const char* str1, const char* str2, size_t len) {
+    while (len && *str1 && (*str1 == *str2)) {
+        ++str1;
+        ++str2;
+        --len;
+    }
+    if (len == 0) {
+        return 0;
+    } else {
+        return (*(unsigned char*)str1 - *(unsigned char*)str2);
+    }
+}
+
+int strcmp(const char* str1, const char* str2) {
+    while (*str1 && (*str1 == *str2)) {
+        ++str1;
+        ++str2;
+    }
+    return (*(unsigned char*)str1 - *(unsigned char*)str2);
+}
+
+char* strchr(const char* s, int c) {
+    char ch = (char)c;
+    while (true) {
+        char cur = *s;
+        if (cur == ch)
+            return (char*)s;
+        if (cur == 0)
+            return nullptr;
+        s++;
+    }
+}

lib/wait.c

@@ -0,0 +1,17 @@
+#include <zinnia/waitq.h>
+
+void waitq_init(struct waitq* wq) {
+    tailq_init(&wq->waiters);
+}
+
+void waitq_wait(struct waitq* wq) {
+    // TODO
+}
+
+void waitq_wake_one(struct waitq* wq) {
+    // TODO
+}
+
+void waitq_wake_all(struct waitq* wq) {
+    // TODO
+}