pmap tlb shootdowns

core/mem/pmap.c

@@ -1,5 +1,8 @@
 #include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/spin.h>
 #include <zinnia/vma.h>
@@ -133,3 +136,103 @@ fail:
     spin_unlock(&pt->lock);
     return status;
 }
+
+errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte))
+        status = ENOENT;
+    else
+        *out = pte_address(pte);
+
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte)) {
+        status = ENOENT;
+        goto fail;
+    }
+
+    *pte = pte_build(pte_address(pte), flags, CACHE_ANY);
+    pte_flush(vaddr);
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+static void free_table(phys_t table, int8_t level) {
+    if (level > 0) {
+        pte_t* entries = HHDM_PTR(table);
+        const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+        for (size_t i = 0; i < num_entries; i++) {
+            if (pte_is_present(&entries[i]))
+                free_table(pte_address(&entries[i]), level - 1);
+        }
+    }
+    mem_phys_free(table, 1);
+}
+
+void pmap_destroy(struct pmap* pt) {
+    if (!pt->is_user)
+        return;
+
+    pte_t* root = HHDM_PTR(pt->root);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+    for (size_t i = 0; i < num_entries / 2; i++) {
+        if (pte_is_present(&root[i]))
+            free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
+    }
+    mem_phys_free(pt->root, 1);
+    pt->root = 0;
+}
+
+[[__percpu]]
+static atomic_size_t shootdown_req;
+
+[[__percpu]]
+static atomic_size_t shootdown_ack;
+
+void pmap_shootdown_handle() {
+    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
+    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
+        return;
+    pte_flush_all();
+    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
+}
+
+void pmap_shootdown(struct pmap* pt) {
+    const size_t count = percpu_count();
+    if (count < 2)
+        return;
+
+    irq_lock();
+    struct percpu* self = percpu_get();
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
+        pte_shootdown_kick(cpu);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
+        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
+            pmap_shootdown_handle();
+            asm volatile("pause");
+        }
+    }
+    irq_unlock();
+}

core/mem/stack.c

@@ -52,6 +52,7 @@ void stack_destroy(struct stack* ks) {
     const size_t page_size = arch_mem_page_size();
     for (size_t i = 0; i < ks->size; i += page_size)
         pmap_unmap(&kernel_vma.pmap, ks->base + i);
+    pmap_shootdown(&kernel_vma.pmap);
 
     mem_phys_free(ks->phys, ks->size / page_size);
     vm_free(ks->base - page_size, ks->size + page_size);

core/module.c

@@ -204,6 +204,10 @@ static void module_free(struct module* mod) {
         const struct module_segment* seg = &mod->segments[i];
         for (size_t off = 0; off < seg->size; off += page_size)
             pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+    }
+    pmap_shootdown(&kernel_vma.pmap);
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
         mem_phys_free(seg->phys, seg->size / page_size);
     }
     mem_free(mod->segments);

core/percpu.c

@@ -3,12 +3,11 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/compiler.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/utils.h>
+#include <zinnia/vma.h>
 #include <arch/percpu.h>
 
-extern uint8_t __ld_percpu_start[];
-extern uint8_t __ld_percpu_end[];
-
 static _Atomic(size_t) num_cpus = 1;
 
 static size_t max_cpus = SIZE_MAX;
@@ -25,20 +24,33 @@ struct percpu percpu_bsp = {
 };
 
 struct percpu* percpu_new() {
-    const size_t size = __ld_percpu_end - __ld_percpu_start;
-    const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
-
-    // TODO: Map pages past the end of the kernel.
-    // TODO: Document how allocations work.
-
-    phys_t mem;
-    if (mem_phys_alloc(num_pages, 0, &mem))
+    const size_t pgsz = arch_mem_page_size();
+    const size_t stride = percpu_stride();
+    const size_t id = atomic_load(&num_cpus, ATOMIC_RELAXED);
+    if (id >= (UINTPTR_MAX - (uintptr_t)__ld_percpu_start) / stride)
         return nullptr;
 
-    struct percpu* cpu = HHDM_PTR(mem);
+    phys_t mem;
+    if (mem_phys_alloc(stride / pgsz, 0, &mem))
+        return nullptr;
+
+    const uintptr_t base = (uintptr_t)percpu_cpu(id);
+    for (size_t offset = 0; offset < stride; offset += pgsz) {
+        if (pmap_map(&kernel_vma.pmap, base + offset, mem + offset, PTE_READ | PTE_WRITE, CACHE_ANY)) {
+            while (offset) {
+                offset -= pgsz;
+                pmap_unmap(&kernel_vma.pmap, base + offset);
+            }
+            mem_phys_free(mem, stride / pgsz);
+            return nullptr;
+        }
+    }
+
+    struct percpu* cpu = (struct percpu*)base;
     cpu->self = cpu;
-    cpu->id = atomic_fetch_add(&num_cpus, 1, ATOMIC_RELAXED);
+    cpu->id = id;
     cpu->online = false;
+    atomic_store(&num_cpus, id + 1, ATOMIC_RELEASE);
     return cpu;
 }
 
@@ -46,6 +58,10 @@ size_t percpu_max_cpus() {
     return max_cpus;
 }
 
+size_t percpu_count() {
+    return atomic_load(&num_cpus, ATOMIC_RELAXED);
+}
+
 struct percpu* percpu_get() {
     return arch_percpu_get();
 }