Rename dirs

mem/Kconfig

@@ -0,0 +1,12 @@
+menu "Memory Management options"
+
+config HAVE_MMU
+  bool
+config HAVE_MMU_GENERIC
+  bool
+config HAVE_MMU_SPLIT_ROOT
+  bool
+config HAVE_TLB_SHOOTDOWN_IPI
+  bool
+
+endmenu

mem/init.c

@@ -0,0 +1,178 @@
+#define pr_fmt(fmt) "mem: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/panic.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+
+extern uint8_t __ld_text_start[];
+extern uint8_t __ld_text_end[];
+extern uint8_t __ld_rodata_start[];
+extern uint8_t __ld_rodata_end[];
+extern uint8_t __ld_data_start[];
+extern uint8_t __ld_data_end[];
+extern uint8_t __ld_kernel_start[];
+
+struct vma kernel_vma = {0};
+
+[[__init]]
+void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) {
+    // This function creates a kernel page table and initializes all memory managers.
+
+    const size_t pgsz = arch_mem_page_size();
+
+    log("Memory map:\n");
+    for (size_t i = 0; i < map_len; i++) {
+        if (map[i].length == 0)
+            continue;
+
+        const char* label = nullptr;
+        switch (map[i].usage) {
+        case PHYS_RESERVED:
+            label = "Reserved";
+            break;
+        case PHYS_USABLE:
+            label = "Usable";
+            break;
+        case PHYS_STATIC:
+            label = "Static";
+            break;
+        case PHYS_RECLAIMABLE:
+            label = "Reclaimable";
+            break;
+        }
+
+        log("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
+    }
+
+    // Set up the bootstrap allocator.
+    struct phys_mem* largest = nullptr;
+    for (size_t i = 0; i < map_len; i++) {
+        if (map[i].usage == PHYS_USABLE && (!largest || map[i].length > largest->length))
+            largest = &map[i];
+    }
+    assert(largest && largest->length >= pgsz, "No usable memory for bootstrap allocator");
+    mem_phys_bootstrap(largest);
+    log("Using region [%p - %p] for bootstrap allocator\n",
+        (void*)largest->address,
+        (void*)(largest->address + largest->length - 1));
+
+    // Set the HHDM base address to the address given by the loader.
+    // We must not keep any virtual addresses to this region,
+    // since we're likely going to map it at a different base address.
+    mem_hhdm_base = tmp_hhdm;
+
+    assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table");
+    tailq_init(&kernel_vma.mappings);
+
+    // text
+    log("Mapping text segment at %p\n", __ld_text_start);
+    for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
+        errno_t status = pmap_map(
+            &kernel_vma.pmap,
+            (uintptr_t)p,
+            (phys_t)(p - __ld_kernel_start + kernel_phys),
+            PTE_READ | PTE_EXEC,
+            CACHE_ANY
+        );
+        assert(!status, "Failed to map %p with error %i", p, status);
+    }
+
+    // rodata
+    log("Mapping rodata segment at %p\n", __ld_rodata_start);
+    for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
+        errno_t status = pmap_map(
+            &kernel_vma.pmap,
+            (uintptr_t)p,
+            (phys_t)(p - __ld_kernel_start + kernel_phys),
+            PTE_READ,
+            CACHE_ANY
+        );
+        assert(!status, "Failed to map %p with error %i", p, status);
+    }
+
+    // data
+    log("Mapping data segment at %p\n", __ld_data_start);
+    for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
+        errno_t status = pmap_map(
+            &kernel_vma.pmap,
+            (uintptr_t)p,
+            (phys_t)(p - __ld_kernel_start + kernel_phys),
+            PTE_READ | PTE_WRITE,
+            CACHE_ANY
+        );
+        assert(!status, "Failed to map %p with error %i", p, status);
+    }
+
+    // The kernel address space is divided into 4 segments.
+    // For example, they could live at:
+    // HHDM     FFFF'8000'0000'0000
+    // PFNDB    FFFF'A000'0000'0000
+    // Mappings FFFF'C000'0000'0000
+    // Kernel   FFFF'FFFF'8000'0000
+
+    // Map all physical memory to the HHDM address.
+    tmp_hhdm = arch_mem_hhdm_addr();
+    for (size_t i = 0; i < map_len; i++) {
+        if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC && map[i].usage != PHYS_RECLAIMABLE)
+            continue;
+        for (size_t p = 0; p < map[i].length; p += pgsz) {
+            uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
+            phys_t paddr = (phys_t)(map[i].address + p);
+            errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
+            assert(!status, "Failed to map HHDM page %p to %p with error %i", (void*)vaddr, (void*)paddr, status);
+        }
+    }
+    mem_hhdm_base = tmp_hhdm;
+
+    // Switch to our own page table.
+    pmap_set(&kernel_vma.pmap);
+
+    // We record metadata for every single page of available memory in a large array.
+    // This array is contiguous in virtual memory, but is sparsely populated.
+    // Only those array entries which represent usable memory are mapped.
+    for (size_t i = 0; i < map_len; i++) {
+        if (map[i].length == 0 || map[i].usage != PHYS_USABLE)
+            continue;
+
+        const size_t num_pages = (map[i].length + pgsz - 1) / pgsz;
+        const size_t metadata_size = num_pages * sizeof(struct page);
+        const uintptr_t metadata_start_vaddr = arch_mem_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
+        const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size;
+
+        // Find the page-aligned boundaries that contain this metadata.
+        const uintptr_t aligned_start = ALIGN_DOWN(metadata_start_vaddr, pgsz);
+        const uintptr_t aligned_end = ALIGN_UP(metadata_end_vaddr, pgsz);
+        const size_t length = aligned_end - aligned_start;
+        const uintptr_t vaddr = aligned_start;
+
+        for (size_t page = 0; page < length; page += pgsz) {
+            phys_t paddr;
+            assert(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!");
+            assert(
+                !pmap_map(&kernel_vma.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_ANY),
+                "Failed to map PFNDB memory"
+            );
+        }
+    }
+    vm_pfndb = (struct page*)arch_mem_pfndb_addr();
+
+    // We don't need the bootstrap allocator from this point on.
+    // Initialize the real page allocator.
+    mem_phys_init(map, map_len);
+
+    slab_init();
+
+    log("Memory initialization complete\n");
+}
+
+[[__init]]
+static void mem_stage_init(struct boot_info* info) {
+    mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
+}
+INIT_TASK(mem_stage, "core.mem", mem_stage_init);
+INIT_DEPENDS(mem_stage, console_stage);

mem/meson.build

@@ -0,0 +1,21 @@
+kernel_sources += files(
+    'fault.c',
+    'init.c',
+    'pmm.c',
+    'slab.c',
+    'stack.c',
+    'usercopy.c',
+    'virt.c',
+    'vma.c',
+    'vmo.c',
+)
+
+if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y'
+    kernel_sources += files(
+        'pmap.c',
+    )
+endif
+
+if kconfig.get('CONFIG_HAVE_TLB_SHOOTDOWN_IPI', 'n') == 'y'
+    kernel_sources += files('shootdown.c')
+endif

mem/pmap.c

@@ -0,0 +1,216 @@
+#include <zinnia/alloc.h>
+#include <zinnia/errno.h>
+#include <zinnia/pmap.h>
+#include <zinnia/spin.h>
+#include <zinnia/vma.h>
+#include <config.h>
+
+// TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
+static_assert(CONFIG_HAVE_MMU_GENERIC);
+
+errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
+    flags &= ~ALLOC_NOZERO;
+
+    phys_t addr;
+    errno_t status = mem_phys_alloc(1, flags, &addr);
+    if (status)
+        return status;
+
+#if !CONFIG_HAVE_MMU_SPLIT_ROOT
+    pte_t* root = HHDM_PTR(addr);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+    for (size_t i = num_entries / 2; i < num_entries; i++) {
+        phys_t dir;
+        status = mem_phys_alloc(1, flags, &dir);
+        if (status)
+            return status;
+        pte_write(&root[i], pte_build(dir, PTE_DIR, CACHE_ANY));
+    }
+#endif
+
+    struct pmap result = {
+        .root = addr,
+        .lock = (struct spinlock){0},
+        .is_user = false,
+    };
+
+    *pt = result;
+    return 0;
+}
+
+errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
+    phys_t user_l1;
+
+    // If the kernel and userspace PTEs are on separate page tables,
+    // allocate them individually. Otherwise, copy over the higher half PTEs
+    // into the new user page table from the kernel VMA.
+#if CONFIG_HAVE_MMU_SPLIT_ROOT
+    errno_t status = mem_phys_alloc(1, flags & ~ALLOC_NOZERO, &user_l1);
+    if (status)
+        return status;
+#else
+    errno_t status = mem_phys_alloc(1, flags, &user_l1);
+    if (status)
+        return status;
+
+    void* user_l1_ptr = HHDM_PTR(user_l1);
+    void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
+    memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
+#endif
+
+    struct pmap result = {
+        .root = user_l1,
+        .lock = (struct spinlock){0},
+        .is_user = true,
+    };
+
+    *pt = result;
+    return 0;
+}
+
+// Gets a reference to the PTE at the given virtual address.
+// If `check_only` is set, only checks if the PTE exists,
+// and doesn't allocate new levels if they don't already exist.
+// If it can't allocate a page if it has to, returns `nullptr`.
+static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) {
+    pte_t* current_head = HHDM_PTR(pt->root);
+    size_t index = 0;
+
+    for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) {
+        const size_t addr_mask = (1 << arch_mem_level_bits()) - 1;
+        const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level);
+        const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0);
+
+        index = (vaddr >> addr_shift) & addr_mask;
+
+        // The last level is used to access the actual PTE, so break the loop then.
+        // We still need to update the index beforehand, that's why we can't just end early.
+        if (level == 0)
+            break;
+
+        pte_t* pte = &current_head[index];
+
+        if (pte_is_present(pte)) {
+            // Get the next level.
+            const pte_t upgraded = *pte | pte_build(0, level_flags, CACHE_ANY);
+            if (upgraded != *pte)
+                pte_write(pte, upgraded);
+            current_head = HHDM_PTR(pte_address(pte));
+        } else {
+            // If the current level isn't present, we can skip the rest.
+            if (check_only)
+                return nullptr;
+
+            phys_t addr;
+            if (mem_phys_alloc(1, 0, &addr) != 0)
+                return nullptr;
+
+            pte_write(pte, pte_build(addr, level_flags, CACHE_ANY));
+            current_head = HHDM_PTR(addr);
+        }
+    }
+
+    return &current_head[index];
+}
+
+errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
+    if (pte == nullptr) {
+        status = ENOMEM;
+        goto fail;
+    }
+
+    if (pte_is_present(pte)) {
+        pte_clear(pte);
+        pte_flush(vaddr);
+    }
+    pte_write(pte, pte_build(paddr, flags, cache));
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte)) {
+        status = ENOENT;
+        goto fail;
+    }
+
+    pte_clear(pte);
+    pte_flush(vaddr);
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte))
+        status = ENOENT;
+    else
+        *out = pte_address(pte);
+
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte)) {
+        status = ENOENT;
+        goto fail;
+    }
+
+    pte_write(pte, pte_protect(*pte, flags));
+    pte_flush(vaddr);
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+static void free_table(phys_t table, int8_t level) {
+    if (level > 0) {
+        pte_t* entries = HHDM_PTR(table);
+        const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+        for (size_t i = 0; i < num_entries; i++) {
+            if (pte_is_present(&entries[i]))
+                free_table(pte_address(&entries[i]), level - 1);
+        }
+    }
+    mem_phys_free(table, 1);
+}
+
+void pmap_destroy(struct pmap* pt) {
+    if (!pt->is_user)
+        return;
+
+    pte_t* root = HHDM_PTR(pt->root);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+#if CONFIG_HAVE_MMU_SPLIT_ROOT
+    const size_t user_entries = num_entries;
+#else
+    const size_t user_entries = num_entries / 2;
+#endif
+    for (size_t i = 0; i < user_entries; i++) {
+        if (pte_is_present(&root[i]))
+            free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
+    }
+    mem_phys_free(pt->root, 1);
+    pt->root = 0;
+}

mem/pmm.c

@@ -0,0 +1,193 @@
+#include <zinnia/alloc.h>
+#include <zinnia/compiler.h>
+#include <zinnia/panic.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/virt.h>
+#include <stdint.h>
+
+uintptr_t mem_hhdm_base = 0;
+
+static errno_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
+static errno_t (*free_fn)(phys_t start, size_t num_pages);
+
+errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
+    return alloc_fn(num_pages, flags, out);
+}
+
+errno_t mem_phys_free(phys_t start, size_t num_pages) {
+    return free_fn(start, num_pages);
+}
+
+static struct phys_mem bump_mem = {0};
+static struct phys_mem* bump_region = nullptr;
+
+static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
+    const size_t bytes = num_pages * arch_mem_page_size();
+    if (bytes > bump_mem.length)
+        return ENOMEM;
+
+    *out = bump_mem.address;
+    bump_mem.length -= bytes;
+    bump_mem.address += bytes;
+
+    if (!(flags & ALLOC_NOZERO)) {
+        memset(HHDM_PTR(*out), 0, bytes);
+    }
+
+    return 0;
+}
+
+static errno_t bump_free(phys_t addr, size_t num_pages) {
+    // We don't free bootstrap memory.
+    panic("Attempted to free bootstrap memory!");
+}
+
+void mem_phys_bootstrap(struct phys_mem* mem) {
+    alloc_fn = bump_alloc;
+    free_fn = bump_free;
+
+    // Save the region by reference and value.
+    // We can't directly modify the region by reference, because that
+    // interferes with e.g. HHDM mapping logic.
+    bump_mem = *mem;
+    bump_region = mem;
+}
+
+static struct page* pmm_head = nullptr;
+static struct spinlock pmm_lock = {0};
+static size_t pmm_total_free = 0;
+
+static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
+    spin_lock(&pmm_lock);
+
+    const size_t bytes = num_pages * arch_mem_page_size();
+    phys_t limit = UINTPTR_MAX;
+    if (flags & ALLOC_MEM20) {
+        limit = (phys_t)1 << 20;
+    } else if (flags & ALLOC_MEM32) {
+        limit = (phys_t)1 << 32;
+    }
+
+    struct page* it = pmm_head;
+    struct page* prev_it = nullptr;
+
+    while (it) {
+        const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
+        // If the region is too high up or doesn't have enough pages left, continue searching.
+        if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) {
+            prev_it = it;
+            it = it->freelist.next;
+            continue;
+        }
+
+        // If the current block has exactly enough pages left, consume it entirely.
+        if (__unlikely(it->freelist.count == num_pages)) {
+            if (prev_it)
+                prev_it->freelist.next = it->freelist.next;
+            else
+                pmm_head = it->freelist.next;
+
+            it->freelist.next = nullptr;
+            it->freelist.count = 0;
+
+            if (__likely(out))
+                *out = page_addr;
+        }
+        // If not, we can just shrink the block.
+        else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) {
+            it->freelist.count -= num_pages;
+
+            if (__likely(out))
+                *out = page_addr + (it->freelist.count * arch_mem_page_size());
+        } else {
+            struct page* rest = it + num_pages;
+            rest->freelist.count = it->freelist.count - num_pages;
+            rest->freelist.next = it->freelist.next;
+
+            if (prev_it)
+                prev_it->freelist.next = rest;
+            else
+                pmm_head = rest;
+
+            it->freelist.next = nullptr;
+            it->freelist.count = 0;
+
+            if (__likely(out))
+                *out = page_addr;
+        }
+        goto success;
+    }
+
+    spin_unlock(&pmm_lock);
+    return ENOMEM;
+
+success:
+    // If the NOZERO flag is *not* specified, zero memory.
+    if (__likely(out) && __likely(!(flags & ALLOC_NOZERO))) {
+        memset(HHDM_PTR(*out), 0, bytes);
+    }
+    pmm_total_free -= num_pages;
+    spin_unlock(&pmm_lock);
+    return 0;
+}
+
+static void freelist_insert(struct page* page, size_t num_pages) {
+    struct page* prev = nullptr;
+    struct page* next = pmm_head;
+    while (next && next < page) {
+        prev = next;
+        next = next->freelist.next;
+    }
+
+    page->freelist.count = num_pages;
+    page->freelist.next = next;
+
+    if (next && page + page->freelist.count == next) {
+        page->freelist.count += next->freelist.count;
+        page->freelist.next = next->freelist.next;
+        next->freelist.next = nullptr;
+        next->freelist.count = 0;
+    }
+
+    if (prev && prev + prev->freelist.count == page) {
+        prev->freelist.count += page->freelist.count;
+        prev->freelist.next = page->freelist.next;
+        page->freelist.next = nullptr;
+        page->freelist.count = 0;
+    } else if (prev) {
+        prev->freelist.next = page;
+    } else {
+        pmm_head = page;
+    }
+}
+
+static errno_t freelist_free(phys_t addr, size_t num_pages) {
+    spin_lock(&pmm_lock);
+
+    freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages);
+    pmm_total_free += num_pages;
+
+    spin_unlock(&pmm_lock);
+    return 0;
+}
+
+void mem_phys_init(struct phys_mem* map, size_t length) {
+    alloc_fn = freelist_alloc;
+    free_fn = freelist_free;
+    bump_region->address = bump_mem.address;
+    bump_region->length = bump_mem.length;
+
+    for (size_t i = 0; i < length; i++) {
+        // Regions smaller than a page are useless.
+        if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
+            continue;
+
+        const size_t num_pages = map[i].length / arch_mem_page_size();
+        freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages);
+        pmm_total_free += num_pages;
+    }
+
+    const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
+    log("pmm: Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
+}

mem/shootdown.c

@@ -0,0 +1,47 @@
+#include <zinnia/atomic.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <arch/cpu.h>
+
+[[__percpu]]
+static atomic_size_t shootdown_req;
+
+[[__percpu]]
+static atomic_size_t shootdown_ack;
+
+void pmap_shootdown_handle() {
+    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
+    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
+        return;
+    pte_flush_all();
+    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
+}
+
+void pmap_shootdown(struct pmap* pt) {
+    const size_t count = percpu_count();
+    if (count < 2)
+        return;
+
+    irq_lock();
+    struct percpu* self = percpu_get();
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
+        pte_shootdown_kick(cpu);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
+        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
+            pmap_shootdown_handle();
+            arch_cpu_pause();
+        }
+    }
+    irq_unlock();
+}

mem/slab.c

@@ -0,0 +1,143 @@
+#include <zinnia/alloc.h>
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/spin.h>
+#include <string.h>
+
+typedef struct {
+    size_t num_pages; // Amount of pages connected to this slab.
+    size_t size;      // Size of this slab.
+} slab_info;
+
+struct slab {
+    struct spinlock lock; // Access lock.
+    size_t ent_size;      // Size of one entry.
+    void** head;
+};
+
+typedef struct {
+    struct slab* slab;
+} slab_header;
+
+// Initializes the SLAB structures.
+
+static struct slab slabs[8] = {0};
+
+static errno_t slab_new(struct slab* slab, size_t size) {
+    // Allocate a new page for the head.
+    phys_t addr;
+    errno_t status = mem_phys_alloc(1, 0, &addr);
+    if (status)
+        return status;
+
+    slab->head = (void**)(addr + arch_mem_hhdm_addr());
+    slab->ent_size = size;
+
+    const size_t offset = ALIGN_UP(sizeof(slab_header), size);
+    const size_t available_size = arch_mem_page_size() - offset;
+
+    slab_header* ptr = (slab_header*)slab->head;
+    ptr->slab = slab;
+    slab->head = (void**)((void*)slab->head + offset);
+
+    void** arr = slab->head;
+    const size_t max = available_size / size - 1;
+    const size_t fact = size / sizeof(void*);
+
+    for (size_t i = 0; i < max; i++) {
+        arr[i * fact] = &arr[(i + 1) * fact];
+    }
+    arr[max * fact] = nullptr;
+
+    return 0;
+}
+
+void slab_init(void) {
+    slab_new(&slabs[0], 16);
+    slab_new(&slabs[1], 32);
+    slab_new(&slabs[2], 64);
+    slab_new(&slabs[3], 128);
+    slab_new(&slabs[4], 256);
+    slab_new(&slabs[5], 512);
+    slab_new(&slabs[6], 1024);
+    slab_new(&slabs[7], 2048);
+}
+
+static void* slab_do_alloc(struct slab* slab) {
+    spin_lock(&slab->lock);
+
+    if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) {
+        spin_unlock(&slab->lock);
+        return nullptr;
+    }
+
+    void** old_free = slab->head;
+    slab->head = *old_free;
+    memset(old_free, 0, slab->ent_size);
+
+    spin_unlock(&slab->lock);
+    return old_free;
+}
+
+static void slab_do_free(struct slab* slab, void* addr) {
+    spin_lock(&slab->lock);
+
+    if (__unlikely(addr == nullptr))
+        goto cleanup;
+
+    void** new_head = addr;
+    *new_head = slab->head;
+    slab->head = new_head;
+
+cleanup:
+    spin_unlock(&slab->lock);
+}
+
+static inline struct slab* slab_find_size(size_t size) {
+    for (size_t i = 0; i < ARRAY_SIZE(slabs); i++) {
+        if (slabs[i].ent_size >= size)
+            return &slabs[i];
+    }
+    return nullptr;
+}
+
+void* mem_alloc(size_t size, enum alloc_flags flags) {
+    if (__unlikely(size == 0))
+        return nullptr;
+
+    struct slab* slab = slab_find_size(size);
+    if (slab != nullptr) {
+        return slab_do_alloc(slab);
+    }
+
+    size_t num_pages = ROUND_UP(size, arch_mem_page_size());
+
+    // Allocate the pages plus an additional page for metadata.
+    phys_t ret;
+    errno_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
+    if (__unlikely(status != 0))
+        return nullptr;
+
+    ret = ret + (phys_t)arch_mem_hhdm_addr();
+    // Write metadata into the first page.
+    slab_info* info = (slab_info*)ret;
+    info->num_pages = num_pages;
+    info->size = size;
+    // Skip the first page and return the next one.
+    return (void*)(ret + arch_mem_page_size());
+}
+
+void mem_free(void* addr) {
+    if (__unlikely(addr == nullptr))
+        return;
+
+    // If the address is page aligned.
+    if ((size_t)addr == ALIGN_DOWN((size_t)addr, arch_mem_page_size())) {
+        slab_info* info = (slab_info*)(addr - arch_mem_page_size());
+        mem_phys_free(((phys_t)info - (phys_t)arch_mem_hhdm_addr()), info->num_pages + 1);
+        return;
+    }
+
+    slab_header* header = (slab_header*)(ALIGN_DOWN((size_t)addr, arch_mem_page_size()));
+    slab_do_free(header->slab, addr);
+}

mem/stack.c

@@ -0,0 +1,61 @@
+#include <zinnia/alloc.h>
+#include <zinnia/errno.h>
+#include <zinnia/pmap.h>
+#include <zinnia/stack.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+
+errno_t stack_init(struct stack* ks, size_t size) {
+    if (!ks || size == 0)
+        return EINVAL;
+
+    const size_t page_size = arch_mem_page_size();
+    size = ALIGN_UP(size, page_size);
+
+    uintptr_t va = vm_alloc(size + page_size);
+    if (!va)
+        return ENOMEM;
+
+    phys_t phys;
+    errno_t status = mem_phys_alloc(size / page_size, 0, &phys);
+    if (status)
+        goto err_vm;
+
+    const uintptr_t base = va + page_size;
+    size_t mapped = 0;
+    for (; mapped < size; mapped += page_size) {
+        status = pmap_map(&kernel_vma.pmap, base + mapped, phys + mapped, PTE_READ | PTE_WRITE, CACHE_WRITE_BACK);
+        if (status)
+            goto err_map;
+    }
+
+    ks->base = base;
+    ks->phys = phys;
+    ks->size = size;
+    return 0;
+
+err_map:
+    for (size_t i = 0; i < mapped; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, base + i);
+    mem_phys_free(phys, size / page_size);
+err_vm:
+    vm_free(va, size + page_size);
+    return status;
+}
+
+void stack_destroy(struct stack* ks) {
+    if (!ks || !ks->base)
+        return;
+
+    const size_t page_size = arch_mem_page_size();
+    for (size_t i = 0; i < ks->size; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, ks->base + i);
+    pmap_shootdown(&kernel_vma.pmap);
+
+    mem_phys_free(ks->phys, ks->size / page_size);
+    vm_free(ks->base - page_size, ks->size + page_size);
+
+    *ks = (struct stack){0};
+}

mem/usercopy.c

@@ -0,0 +1,44 @@
+#include <zinnia/percpu.h>
+#include <zinnia/usercopy.h>
+#include <arch/mem.h>
+#include <arch/usercopy.h>
+
+static bool is_user_range(uintptr_t addr, size_t len) {
+    if (len == 0)
+        return arch_mem_addr_is_user(addr);
+    return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1);
+}
+
+bool usercopy_read(void* dst, const __user void* src, size_t len) {
+    if (!is_user_range((uintptr_t)src, len))
+        return false;
+
+    arch_usercopy_allow(true);
+    bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
+    arch_usercopy_allow(false);
+    return success;
+}
+
+bool usercopy_write(__user void* dst, const void* src, size_t len) {
+    if (!is_user_range((uintptr_t)dst, len))
+        return false;
+
+    arch_usercopy_allow(true);
+    bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
+    arch_usercopy_allow(false);
+    return success;
+}
+
+bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
+    if (!arch_mem_addr_is_user((uintptr_t)str))
+        return false;
+
+    size_t unused;
+    if (len == nullptr)
+        len = &unused;
+
+    arch_usercopy_allow(true);
+    bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
+    arch_usercopy_allow(false);
+    return success;
+}

mem/virt.c

@@ -0,0 +1,26 @@
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <stddef.h>
+
+struct page* vm_pfndb = nullptr;
+
+// TODO: Replace with actual allocator.
+
+static _Atomic(size_t) mapping_offset = 0;
+
+uintptr_t vm_alloc(size_t length) {
+    uintptr_t base = arch_mem_mapping_addr();
+    size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE);
+
+    return base + offset;
+}
+
+void vm_free(uintptr_t ptr, size_t length) {
+    // TODO
+}
+
+uintptr_t vm_page_addr(struct page* page) {
+    return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
+}

mem/vma.c

@@ -0,0 +1,158 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/errno.h>
+#include <zinnia/mman.h>
+#include <zinnia/pmap.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stdckdint.h>
+
+// Try to create a new, empty virtual address space.
+errno_t vma_new(struct vma** out) {
+    debug_assert(out != nullptr, "Missing out");
+
+    errno_t res;
+    if (!out)
+        return EINVAL;
+
+    struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
+    if (!vmspace)
+        return ENOMEM;
+
+    res = pmap_new_user(&vmspace->pmap, 0);
+    if (res)
+        goto err1;
+
+    tailq_init(&vmspace->mappings);
+
+    *out = vmspace;
+    return 0;
+
+err1:
+    mem_free(vmspace);
+err0:
+    return res;
+}
+
+// Delete a virtual address space.
+void vma_free(struct vma* v) {
+    struct vma_mapping* mapping;
+    while ((mapping = tailq_first(&v->mappings)) != nullptr) {
+        tailq_remove(&v->mappings, mapping, link);
+        vmo_unref(mapping->object);
+        mem_free(mapping);
+    }
+}
+
+errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
+    panic("TODO");
+}
+
+errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
+    panic("TODO");
+}
+
+static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
+    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (page >= m->start_page && page < m->end_page)
+            return m;
+    }
+    return nullptr;
+}
+
+static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) {
+    if (info->caused_by_write)
+        return prot & PROT_WRITE;
+    if (info->caused_by_fetch)
+        return prot & PROT_EXEC;
+    return prot & (PROT_READ | PROT_WRITE);
+}
+
+static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
+    enum pte_flags pf = 0;
+    if (prot & PROT_READ)
+        pf |= PTE_READ;
+    if (prot & PROT_WRITE)
+        pf |= PTE_WRITE;
+    if (prot & PROT_EXEC)
+        pf |= PTE_EXEC;
+    if (v->pmap.is_user)
+        pf |= PTE_USER;
+    return pf;
+}
+
+bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t faulty_page = info->addr / page_size;
+
+    spin_lock(&v->lock);
+    struct vma_mapping* mapping = find_mapping(v, faulty_page);
+    if (!mapping || !can_access(mapping->prot, info)) {
+        spin_unlock(&v->lock);
+        return false;
+    }
+
+    struct vmo* object = mapping->object;
+    const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
+    const enum prot_flags prot = mapping->prot;
+    vmo_ref(object);
+    spin_unlock(&v->lock);
+
+    struct page* page;
+    if (object->get_page(object, object_page, &page)) {
+        vmo_unref(object);
+        return false;
+    }
+
+    spin_lock(&v->lock);
+    mapping = find_mapping(v, faulty_page);
+    const bool still_valid = mapping && mapping->object == object &&
+        (faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info);
+
+    if (still_valid) {
+        errno_t status =
+            pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
+        assert(status == 0, "Failed to map a demand-loaded page");
+    }
+    spin_unlock(&v->lock);
+
+    vmo_unref(object);
+    return true;
+}
+
+errno_t vma_map(
+    struct vma* vma,
+    struct vmo* vmo,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags flags,
+    uintptr_t vmo_offset
+) {
+    if (!vma || !vmo)
+        return EINVAL;
+
+    uintptr_t end;
+    if (ckd_add(&end, addr, len))
+        return ENOMEM;
+
+    const size_t page_size = arch_mem_page_size();
+    if (addr % page_size != vmo_offset % page_size)
+        return EINVAL;
+
+    struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
+    if (!mapping)
+        return ENOMEM;
+
+    vmo_ref(vmo);
+    mapping->start_page = addr / page_size;
+    mapping->end_page = ALIGN_UP(end, page_size) / page_size;
+    mapping->offset_page = vmo_offset / page_size;
+    mapping->object = vmo;
+    mapping->prot = flags;
+
+    spin_lock(&vma->lock);
+    tailq_insert_head(&vma->mappings, mapping, link);
+    spin_unlock(&vma->lock);
+
+    return 0;
+}

mem/vmo.c

@@ -0,0 +1,263 @@
+#include <zinnia/alloc.h>
+#include <zinnia/errno.h>
+#include <zinnia/list.h>
+#include <zinnia/utils.h>
+#include <zinnia/vmo.h>
+
+void vmo_ref(struct vmo* obj) {
+    refcount_inc(&obj->rc);
+}
+
+void vmo_unref(struct vmo* obj) {
+    if (refcount_dec(&obj->rc))
+        obj->destroy(obj);
+}
+
+errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
+    const size_t page_size = arch_mem_page_size();
+    size_t progress = 0;
+    errno_t status = 0;
+
+    while (progress < len) {
+        const size_t misalign = (progress + offset) % page_size;
+        const size_t page_index = (progress + offset) / page_size;
+        const size_t copy_size = MIN(page_size - misalign, len - progress);
+
+        struct page* p;
+        status = obj->get_page(obj, page_index, &p);
+        if (status != 0)
+            break;
+
+        uintptr_t page_addr = vm_page_addr(p);
+
+        memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
+        progress += copy_size;
+    }
+
+    if (actual_read)
+        *actual_read = progress;
+    return status;
+}
+
+errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
+    const size_t page_size = arch_mem_page_size();
+    size_t progress = 0;
+    errno_t status = 0;
+
+    while (progress < len) {
+        const size_t misalign = (progress + offset) % page_size;
+        const size_t page_index = (progress + offset) / page_size;
+        const size_t copy_size = MIN(page_size - misalign, len - progress);
+
+        struct page* p;
+        status = obj->get_page(obj, page_index, &p);
+        if (status != 0)
+            break;
+
+        uintptr_t page_addr = vm_page_addr(p);
+
+        memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
+        progress += copy_size;
+    }
+
+    if (actual_written)
+        *actual_written = progress;
+    return status;
+}
+
+errno_t vmo_copy(
+    struct vmo* target,
+    uintptr_t target_offset,
+    struct vmo* src,
+    uintptr_t src_offset,
+    size_t len,
+    size_t* actual_copied
+) {
+    const size_t page_size = arch_mem_page_size();
+    size_t progress = 0;
+    errno_t status = 0;
+
+    while (progress < len) {
+        const size_t target_misalign = (progress + target_offset) % page_size;
+        const size_t src_misalign = (progress + src_offset) % page_size;
+        const size_t target_page_index = (progress + target_offset) / page_size;
+        const size_t src_page_index = (progress + src_offset) / page_size;
+
+        const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
+
+        struct page* tpage;
+        status = target->get_page(target, target_page_index, &tpage);
+        if (status != 0)
+            break;
+
+        struct page* spage;
+        status = src->get_page(src, src_page_index, &spage);
+        if (status != 0)
+            break;
+
+        uintptr_t taddr = vm_page_addr(tpage);
+        uintptr_t saddr = vm_page_addr(spage);
+
+        memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
+        progress += copy_size;
+    }
+
+    if (actual_copied)
+        *actual_copied = progress;
+    return status;
+}
+
+static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
+    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
+
+    // Try to find the requested page in the cache.
+    struct page_list* iter;
+    spin_lock(&paged->lock);
+    list_foreach(iter, &paged->cache, next) {
+        if (iter->offset != offset_idx)
+            continue;
+
+        // We found a cached page!
+        *out = iter->value;
+        spin_unlock(&paged->lock);
+        return 0;
+    }
+    spin_unlock(&paged->lock);
+
+    // Get page from out pager if we don't have one already.
+    struct page* new_page;
+    errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
+    if (status)
+        return status;
+
+    struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
+    if (!entry) {
+        paged->source.put_page(&paged->source, offset_idx, new_page);
+        return ENOMEM;
+    }
+
+    entry->offset = offset_idx;
+    entry->value = new_page;
+
+    spin_lock(&paged->lock);
+    list_foreach(iter, &paged->cache, next) {
+        if (iter->offset != offset_idx)
+            continue;
+
+        *out = iter->value;
+        spin_unlock(&paged->lock);
+        mem_free(entry);
+        paged->source.put_page(&paged->source, offset_idx, new_page);
+        return 0;
+    }
+
+    // Add page to the object cache.
+    list_insert_head(&paged->cache, entry, next);
+    spin_unlock(&paged->lock);
+
+    *out = new_page;
+    return 0;
+}
+
+static void paged_destroy(struct vmo* vmo) {
+    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
+
+    struct page_list* iter = list_first(&paged->cache);
+    while (iter) {
+        struct page_list* next = list_next(iter, next);
+        paged->source.put_page(&paged->source, iter->offset, iter->value);
+        mem_free(iter);
+        iter = next;
+    }
+
+    mem_free(paged);
+}
+
+errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
+    struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
+    if (!result)
+        return ENOMEM;
+
+    refcount_init(&result->object.rc);
+    result->source = pager;
+    result->object.get_page = paged_get_page;
+    result->object.destroy = paged_destroy;
+
+    *out = result;
+    return 0;
+}
+
+static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
+    phys_t addr;
+    errno_t status = mem_phys_alloc(1, 0, &addr);
+    if (status != 0)
+        return status;
+    size_t idx = addr / arch_mem_page_size();
+    *out = &vm_pfndb[idx];
+    return 0;
+}
+
+static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
+    return mem_phys_free(vm_page_addr(page), 1);
+}
+
+static const struct pager_ops phys_pager = {
+    .get_page = phys_get_page,
+    .put_page = phys_put_page,
+};
+
+errno_t vmo_new_phys(struct paged_vmo** out) {
+    return vmo_new_paged(phys_pager, out);
+}
+
+void vmo_truncate_paged(struct paged_vmo* paged, size_t length) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t keep_pages = ROUND_UP(length, page_size);
+
+    spin_lock(&paged->lock);
+    struct page_list** pp = &list_first(&paged->cache);
+    while (*pp) {
+        struct page_list* entry = *pp;
+        if (entry->offset >= keep_pages) {
+            *pp = list_next(entry, next);
+            paged->source.put_page(&paged->source, entry->offset, entry->value);
+            mem_free(entry);
+        } else {
+            pp = &list_next(entry, next);
+        }
+    }
+    spin_unlock(&paged->lock);
+}
+
+struct phys_range_vmo {
+    struct vmo object;
+    uintptr_t start;
+    size_t len;
+};
+
+static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
+    struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
+    if (offset_idx >= p->len / arch_mem_page_size())
+        return EINVAL;
+    *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
+    return 0;
+}
+
+static void phys_range_destroy(struct vmo* vmo) {
+    mem_free(containerof(vmo, struct phys_range_vmo, object));
+}
+
+errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
+    struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
+    if (!vmo)
+        return ENOMEM;
+
+    refcount_init(&vmo->object.rc);
+    vmo->object.get_page = phys_range_get_page;
+    vmo->object.destroy = phys_range_destroy;
+    vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
+    vmo->len = ALIGN_UP(length, arch_mem_page_size());
+
+    *out = &vmo->object;
+    return 0;
+}