IPI TLB shootdowns

arch/x86_64/Kconfig

@@ -6,6 +6,7 @@ config X86_64
   select ACPI
   select HAVE_MMU
   select HAVE_MMU_GENERIC
+  select HAVE_TLB_SHOOTDOWN_IPI
   help
     64-bit x86 CPU support for AMD and Intel CPUs.
 

arch/x86_64/mem.c

@@ -1,4 +1,3 @@
-#include <zinnia/types.h>
 #include <arch/mem.h>
 #include <stddef.h>
 #include <stdint.h>

arch/x86_64/meson.build

@@ -14,7 +14,6 @@ kernel_sources += files(
     'mem.c',
     'percpu.c',
     'pmap.c',
-    'pmap.c',
     'random.c',
     'sched.c',
     'serial.c',

arch/x86_64/pmap.c

@@ -11,6 +11,10 @@ void pte_clear(pte_t* pte) {
     *pte = 0;
 }
 
+void pte_write(pte_t* pte, pte_t val) {
+    *pte = val;
+}
+
 void pte_flush(uintptr_t vaddr) {
     asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
 }
@@ -67,12 +71,19 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     return result;
 }
 
-bool pte_is_present(pte_t* pte) {
+pte_t pte_protect(pte_t old, enum pte_flags flags) {
-    return *pte & ARCH_FLAG_PRESENT;
+    pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+    if (flags & PTE_WRITE)
+        result |= ARCH_FLAG_READ_WRITE;
+    if (!(flags & PTE_EXEC))
+        result |= ARCH_FLAG_EXECUTE_DISABLE;
+    return result;
 }
 
-bool pte_is_dir(pte_t* pte) {
+bool pte_is_present(pte_t* pte) {
-    return *pte & ARCH_FLAG_SIZE;
+    return *pte & ARCH_FLAG_PRESENT;
 }
 
 phys_t pte_address(pte_t* pte) {

arch/x86_64/sched.c

@@ -11,6 +11,7 @@
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
@@ -93,6 +94,11 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
     return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
 }
 
+constexpr size_t FPU_FCW_OFFSET = 0;
+constexpr size_t FPU_MXCSR_OFFSET = 24;
+constexpr uint16_t FPU_DEFAULT_FCW = 0x37F;
+constexpr uint32_t FPU_DEFAULT_MXCSR = 0x1F80;
+
 // Calls task_entry with the correct parameters.
 [[__naked]]
 void task_entry_thunk() {
@@ -142,6 +148,11 @@ errno_t arch_task_init(
         context->fpu_region = (void*)fpu_base;
         context->fpu_phys = pages;
 
+        const uint16_t fcw = FPU_DEFAULT_FCW;
+        const uint32_t mxcsr = FPU_DEFAULT_MXCSR;
+        memcpy((uint8_t*)fpu_base + FPU_FCW_OFFSET, &fcw, sizeof(fcw));
+        memcpy((uint8_t*)fpu_base + FPU_MXCSR_OFFSET, &mxcsr, sizeof(mxcsr));
+
         context->ds = asm_read_ds();
         context->es = asm_read_es();
         context->fs = asm_read_fs();

arch/x86_64/smp.c

@@ -148,7 +148,7 @@ static void smp_trampoline() {
           [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
           [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
           [cr0] "i"(CR0_PE | CR0_ET),
-          [cr0_pg] "i"(CR0_PG),
+          [cr0_pg] "i"((uint32_t)CR0_PG),
           [cr4_pae] "i"(CR4_PAE),
           [msr_efer] "i"(MSR_EFER),
           [efer_lme] "i"(MSR_EFER_LME),

boot/limine/entry.c

@@ -1,6 +1,7 @@
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/fdt.h>
 #include <zinnia/init.h>
 #include <zinnia/print.h>
 #include <zinnia/utils.h>
@@ -94,6 +95,12 @@ void kernel_entry() {
     info.virt_base = (uintptr_t)exec_res->virtual_base;
     info.hhdm_base = (uintptr_t)hhdm_res->offset;
 
+    const struct limine_dtb_response* dtb_res = dtb_request.response;
+    if (dtb_res && dtb_res->dtb_ptr) {
+        info.dtb = (phys_t)((uintptr_t)dtb_res->dtb_ptr - hhdm_res->offset);
+        info.dtb_size = fdt_read_u32((const uint8_t*)dtb_res->dtb_ptr + 4);
+    }
+
     const struct limine_framebuffer_response* fb_res = framebuffer_request.response;
     if (fb_res) {
         // Only use the first framebuffer.

core/mem/Kconfig

@@ -4,5 +4,9 @@ config HAVE_MMU
   bool
 config HAVE_MMU_GENERIC
   bool
+config HAVE_MMU_SPLIT_ROOT
+  bool
+config HAVE_TLB_SHOOTDOWN_IPI
+  bool
 
 endmenu

core/mem/meson.build

@@ -15,3 +15,7 @@ if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y'
         'pmap.c',
     )
 endif
+
+if kconfig.get('CONFIG_HAVE_TLB_SHOOTDOWN_IPI', 'n') == 'y'
+    kernel_sources += files('shootdown.c')
+endif

core/mem/pmap.c

@@ -1,12 +1,8 @@
 #include <zinnia/alloc.h>
-#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
-#include <zinnia/irq.h>
-#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/spin.h>
 #include <zinnia/vma.h>
-#include <arch/cpu.h>
 #include <config.h>
 
 // TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
@@ -20,6 +16,7 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
     if (status)
         return status;
 
+#if !CONFIG_HAVE_MMU_SPLIT_ROOT
     pte_t* root = HHDM_PTR(addr);
     const size_t num_entries = (size_t)1 << arch_mem_level_bits();
     for (size_t i = num_entries / 2; i < num_entries; i++) {
@@ -27,8 +24,9 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
         status = mem_phys_alloc(1, flags, &dir);
         if (status)
             return status;
-        root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
+        pte_write(&root[i], pte_build(dir, PTE_DIR, CACHE_ANY));
     }
+#endif
 
     struct pmap result = {
         .root = addr,
@@ -42,13 +40,23 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
 
 errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
     phys_t user_l1;
-    errno_t status = mem_phys_alloc(1, 0, &user_l1);
+
+    // If the kernel and userspace PTEs are on separate page tables,
+    // allocate them individually. Otherwise, copy over the higher half PTEs
+    // into the new user page table from the kernel VMA.
+#if CONFIG_HAVE_MMU_SPLIT_ROOT
+    errno_t status = mem_phys_alloc(1, flags & ~ALLOC_NOZERO, &user_l1);
+    if (status)
+        return status;
+#else
+    errno_t status = mem_phys_alloc(1, flags, &user_l1);
     if (status)
         return status;
 
     void* user_l1_ptr = HHDM_PTR(user_l1);
     void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
     memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
+#endif
 
     struct pmap result = {
         .root = user_l1,
@@ -84,7 +92,9 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
 
         if (pte_is_present(pte)) {
             // Get the next level.
-            *pte |= pte_build(0, level_flags, CACHE_ANY);
+            const pte_t upgraded = *pte | pte_build(0, level_flags, CACHE_ANY);
+            if (upgraded != *pte)
+                pte_write(pte, upgraded);
             current_head = HHDM_PTR(pte_address(pte));
         } else {
             // If the current level isn't present, we can skip the rest.
@@ -95,7 +105,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
             if (mem_phys_alloc(1, 0, &addr) != 0)
                 return nullptr;
 
-            *pte = pte_build(addr, level_flags, CACHE_ANY);
+            pte_write(pte, pte_build(addr, level_flags, CACHE_ANY));
             current_head = HHDM_PTR(addr);
         }
     }
@@ -113,7 +123,11 @@ errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags
         goto fail;
     }
 
-    *pte = pte_build(paddr, flags, cache);
+    if (pte_is_present(pte)) {
+        pte_clear(pte);
+        pte_flush(vaddr);
+    }
+    pte_write(pte, pte_build(paddr, flags, cache));
 
 fail:
     spin_unlock(&pt->lock);
@@ -162,7 +176,7 @@ errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
         goto fail;
     }
 
-    *pte = pte_build(pte_address(pte), flags, CACHE_ANY);
+    pte_write(pte, pte_protect(*pte, flags));
     pte_flush(vaddr);
 
 fail:
@@ -188,52 +202,15 @@ void pmap_destroy(struct pmap* pt) {
 
     pte_t* root = HHDM_PTR(pt->root);
     const size_t num_entries = (size_t)1 << arch_mem_level_bits();
-    for (size_t i = 0; i < num_entries / 2; i++) {
+#if CONFIG_HAVE_MMU_SPLIT_ROOT
+    const size_t user_entries = num_entries;
+#else
+    const size_t user_entries = num_entries / 2;
+#endif
+    for (size_t i = 0; i < user_entries; i++) {
         if (pte_is_present(&root[i]))
             free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
     }
     mem_phys_free(pt->root, 1);
     pt->root = 0;
 }
-
-[[__percpu]]
-static atomic_size_t shootdown_req;
-
-[[__percpu]]
-static atomic_size_t shootdown_ack;
-
-void pmap_shootdown_handle() {
-    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
-    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
-        return;
-    pte_flush_all();
-    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
-}
-
-void pmap_shootdown(struct pmap* pt) {
-    const size_t count = percpu_count();
-    if (count < 2)
-        return;
-
-    irq_lock();
-    struct percpu* self = percpu_get();
-    for (size_t i = 0; i < count; i++) {
-        struct percpu* cpu = percpu_cpu(i);
-        if (cpu == self || !cpu->online)
-            continue;
-        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
-        pte_shootdown_kick(cpu);
-    }
-
-    for (size_t i = 0; i < count; i++) {
-        struct percpu* cpu = percpu_cpu(i);
-        if (cpu == self || !cpu->online)
-            continue;
-        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
-        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
-            pmap_shootdown_handle();
-            arch_cpu_pause();
-        }
-    }
-    irq_unlock();
-}

core/mem/shootdown.c

@@ -0,0 +1,47 @@
+#include <zinnia/atomic.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <arch/cpu.h>
+
+[[__percpu]]
+static atomic_size_t shootdown_req;
+
+[[__percpu]]
+static atomic_size_t shootdown_ack;
+
+void pmap_shootdown_handle() {
+    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
+    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
+        return;
+    pte_flush_all();
+    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
+}
+
+void pmap_shootdown(struct pmap* pt) {
+    const size_t count = percpu_count();
+    if (count < 2)
+        return;
+
+    irq_lock();
+    struct percpu* self = percpu_get();
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
+        pte_shootdown_kick(cpu);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
+        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
+            pmap_shootdown_handle();
+            arch_cpu_pause();
+        }
+    }
+    irq_unlock();
+}

core/spin.c

@@ -17,6 +17,16 @@ void spin_lock(struct spinlock* spin) {
     }
 }
 
+bool spin_trylock(struct spinlock* spin) {
+    irq_lock();
+    uint32_t owner = atomic_load(&spin->owner, ATOMIC_RELAXED);
+    uint32_t expected = owner;
+    if (atomic_compare_exchange(&spin->next, &expected, owner + 1, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return true;
+    irq_unlock();
+    return false;
+}
+
 void spin_unlock(struct spinlock* spin) {
     uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED);
     atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE);

dev/dt/driver.c

@@ -0,0 +1,96 @@
+#define pr_fmt(fmt) "fdt: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/fdt.h>
+#include <zinnia/init.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <string.h>
+
+static struct fdt tree;
+static bool tree_valid = false;
+
+static struct spinlock driver_lock = {};
+static vec(struct fdt_driver*) drivers = {};
+
+struct fdt* fdt_get() {
+    return tree_valid ? &tree : nullptr;
+}
+
+static const struct fdt_match* fdt_match_table(
+    const struct fdt_match* table,
+    size_t num_matches,
+    struct fdt_node node
+) {
+    for (size_t i = 0; i < num_matches; i++) {
+        if (fdt_node_is_compatible(node, table[i].compatible))
+            return &table[i];
+    }
+    return nullptr;
+}
+
+errno_t fdt_driver_register(struct fdt_driver* driver) {
+    if (driver->match == nullptr || driver->num_matches == 0) {
+        log_warn("Driver \"%s\" does not define any matches, ignoring\n", driver->name);
+        return 0;
+    }
+
+    if (!tree_valid)
+        return ENODEV;
+
+    spin_lock(&driver_lock);
+    vec_foreach(&drivers, i) {
+        if (strcmp(vec_elem_get(&drivers, i)->name, driver->name) == 0) {
+            spin_unlock(&driver_lock);
+            log_warn("Driver \"%s\" is already registered\n", driver->name);
+            return EEXIST;
+        }
+    }
+    errno_t status = vec_push(&drivers, driver);
+    spin_unlock(&driver_lock);
+    if (status != 0)
+        return status;
+
+    log("Registered platform driver \"%s\" with %zu match%s\n",
+        driver->name,
+        driver->num_matches,
+        driver->num_matches == 1 ? "" : "es");
+    for (size_t i = 0; i < driver->num_matches; i++)
+        log("  [%zu] = \"%s\"\n", i, driver->match[i].compatible);
+
+    for (struct fdt_node node = fdt_root(&tree); node.tree; node = fdt_next_node(node)) {
+        const struct fdt_match* match = fdt_match_table(driver->match, driver->num_matches, node);
+        if (match == nullptr)
+            continue;
+
+        log("Matched driver \"%s\" on rule [%zu]\n", driver->name, (size_t)(match - driver->match));
+
+        status = driver->probe(node, match);
+        if (status != 0)
+            log_warn("%s failed to probe: %i\n", driver->name, status);
+    }
+
+    return 0;
+}
+
+[[__init]]
+static bool fdt_stage_init(struct boot_info* info) {
+    if (!info->dtb)
+        return false;
+
+    const void* blob = HHDM_PTR(info->dtb);
+    if (!fdt_init(&tree, blob, info->dtb_size)) {
+        log_warn("Unable to parse the device tree blob\n");
+        return false;
+    }
+    tree_valid = true;
+
+    struct fdt_prop model = fdt_property(fdt_root(&tree), "model");
+    if (model.tree && model.length)
+        log("Running on \"%.*s\"\n", (int)model.length, (const char*)model.data);
+
+    return true;
+}
+INIT_TASK(fdt_stage, "dev.dt", fdt_stage_init);
+INIT_DEPENDS(fdt_stage, mem_stage);

dev/dt/fdt.c

@@ -212,24 +212,70 @@ bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value) {
     return true;
 }
 
-struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
+struct fdt_node fdt_next_node(struct fdt_node node) {
-    if (!tree || !tree->blob || !compatible)
-        return (struct fdt_node){};
     struct token token;
-    for (size_t offset = 0; read_token(tree, offset, &token); offset = token.next) {
+    if (!node.tree || !read_token(node.tree, node.offset, &token))
-        if (token.kind != FDT_BEGIN_NODE)
+        return (struct fdt_node){};
-            continue;
+    for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
-        struct fdt_node node = {tree, offset};
+        if (token.kind == FDT_BEGIN_NODE)
-        struct fdt_prop property = fdt_property(node, "compatible");
+            return (struct fdt_node){node.tree, offset};
-        const char* data = property.data;
+        if (token.kind == FDT_END)
-        for (size_t i = 0; i < property.length;) {
+            break;
-            size_t length = strnlen(data + i, property.length - i);
+    }
-            if (length == property.length - i)
+    return (struct fdt_node){};
-                break;
+}
-            if (!strcmp(data + i, compatible))
+
-                return node;
+bool fdt_node_is_compatible(struct fdt_node node, const char* compatible) {
-            i += length + 1;
+    struct fdt_prop property = fdt_property(node, "compatible");
-        }
+    const char* data = property.data;
+    for (size_t i = 0; i < property.length;) {
+        size_t length = strnlen(data + i, property.length - i);
+        if (length == property.length - i)
+            break;
+        if (!strcmp(data + i, compatible))
+            return true;
+        i += length + 1;
+    }
+    return false;
+}
+
+static uint64_t read_cells(const uint8_t* data, uint32_t cells) {
+    uint64_t value = 0;
+    for (uint32_t i = 0; i < cells; i++)
+        value = value << 32 | fdt_read_u32(data + i * 4);
+    return value;
+}
+
+bool fdt_node_reg(struct fdt_node node, size_t index, phys_t* addr, size_t* size) {
+    if (!node.tree)
+        return false;
+    struct fdt_node root = fdt_root(node.tree);
+    uint32_t addr_cells = 2;
+    uint32_t size_cells = 2;
+    fdt_property_u32(root, "#address-cells", &addr_cells);
+    fdt_property_u32(root, "#size-cells", &size_cells);
+    if (addr_cells > 2 || size_cells > 2)
+        return false;
+
+    struct fdt_prop reg = fdt_property(node, "reg");
+    const size_t stride = (addr_cells + size_cells) * 4;
+    if (!reg.tree || index >= reg.length / stride)
+        return false;
+
+    const uint8_t* entry = (const uint8_t*)reg.data + index * stride;
+    if (addr)
+        *addr = read_cells(entry, addr_cells);
+    if (size)
+        *size = read_cells(entry + addr_cells * 4, size_cells);
+    return true;
+}
+
+struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
+    if (!tree || !tree->blob || !compatible)
+        return (struct fdt_node){};
+    for (struct fdt_node node = fdt_root(tree); node.tree; node = fdt_next_node(node)) {
+        if (fdt_node_is_compatible(node, compatible))
+            return node;
     }
     return (struct fdt_node){};
 }

dev/dt/meson.build

@@ -1 +1,4 @@
-kernel_sources += files('fdt.c')
+kernel_sources += files(
+    'driver.c',
+    'fdt.c',
+)

fs/vfs/ident.c

@@ -5,3 +5,13 @@ static const struct ident kernel_identity = {0};
 const struct ident* ident_kernel() {
     return &kernel_identity;
 }
+
+bool ident_in_group(const struct ident* identity, gid_t gid, bool use_real) {
+    if (gid == (use_real ? identity->gid : identity->egid))
+        return true;
+    vec_foreach(&identity->groups, i) {
+        if (vec_elem_get(&identity->groups, i) == gid)
+            return true;
+    }
+    return false;
+}

include/zinnia/fdt.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/types.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -28,8 +31,6 @@ struct fdt_match {
     uintptr_t data;
 };
 
-#define FDT_MATCH_COMPATIBLE(compat) .compatible = (compat)
-
 struct fdt {
     const uint8_t* blob;
     size_t total_size;
@@ -65,3 +66,21 @@ bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value);
 
 // Root of the device tree.
 struct fdt_node fdt_root(struct fdt* tree);
+
+struct fdt_node fdt_next_node(struct fdt_node node);
+bool fdt_node_is_compatible(struct fdt_node node, const char* compatible);
+bool fdt_node_reg(struct fdt_node node, size_t index, phys_t* addr, size_t* size);
+
+struct fdt_driver {
+    const char* name;
+
+    const struct fdt_match* match;
+    const size_t num_matches;
+
+    errno_t (*probe)(struct fdt_node node, const struct fdt_match* match);
+};
+
+struct fdt* fdt_get();
+errno_t fdt_driver_register(struct fdt_driver* driver);
+
+INIT_EXTERN(fdt_stage);

include/zinnia/ident.h

@@ -11,3 +11,5 @@ struct ident {
 };
 
 const struct ident* ident_kernel(void);
+
+bool ident_in_group(const struct ident* identity, gid_t gid, bool use_real);

include/zinnia/init.h

@@ -39,7 +39,7 @@ struct boot_info {
     size_t num_files;
     struct boot_framebuffer fb;
     struct boot_file kernel_file;
-    void* dtb;
+    phys_t dtb;
     size_t dtb_size;
 #if CONFIG_ACPI
     phys_t rsdp;

include/zinnia/list.h

@@ -10,7 +10,7 @@
         type _sl_link; \
     }
 
-#define list_is_empty(head)   ((head)->_sl_head == NULL)
+#define list_is_empty(head)   ((head)->_sl_head == nullptr)
 #define list_first(head)      ((head)->_sl_head)
 #define list_next(elm, field) ((elm)->field._sl_link)
 

include/zinnia/pmap.h

@@ -5,6 +5,7 @@
 #include <zinnia/spin.h>
 #include <zinnia/utils.h>
 #include <bits/pmap.h>
+#include <config.h>
 
 struct percpu;
 
@@ -39,13 +40,16 @@ errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out);
 void pmap_destroy(struct pmap* pt);
 void pmap_set(struct pmap* pt);
 void pmap_shootdown(struct pmap* pt);
-void pmap_shootdown_handle();
 
 void pte_clear(pte_t* pte);
+void pte_write(pte_t* pte, pte_t val);
 void pte_flush(uintptr_t vaddr);
 void pte_flush_all();
+#if CONFIG_HAVE_TLB_SHOOTDOWN_IPI
+void pmap_shootdown_handle();
 void pte_shootdown_kick(struct percpu* cpu);
+#endif
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
+pte_t pte_protect(pte_t old, enum pte_flags flags);
 bool pte_is_present(pte_t* pte);
-bool pte_is_dir(pte_t* pte);
 phys_t pte_address(pte_t* pte);

include/zinnia/refcount.h

@@ -16,6 +16,15 @@ static inline void refcount_inc(struct refcount* rc) {
     atomic_fetch_add(&rc->count, 1, ATOMIC_RELAXED);
 }
 
+static inline bool refcount_inc_not_zero(struct refcount* rc) {
+    size_t count = atomic_load(&rc->count, ATOMIC_RELAXED);
+    do {
+        if (count == 0)
+            return false;
+    } while (!atomic_compare_exchange_weak(&rc->count, &count, count + 1, ATOMIC_ACQUIRE, ATOMIC_RELAXED));
+    return true;
+}
+
 static inline bool refcount_dec(struct refcount* rc) {
     if (atomic_fetch_sub(&rc->count, 1, ATOMIC_RELEASE) != 1)
         return false;

include/zinnia/spin.h

@@ -14,5 +14,7 @@ void spin_init(struct spinlock* spin);
 // Attempts to lock a spinlock. If it's already locked, waits until it's freed.
 void spin_lock(struct spinlock* spin);
 
+bool spin_trylock(struct spinlock* spin);
+
 // Unlocks a spinlock.
 void spin_unlock(struct spinlock* spin);