pmap tlb shootdowns

arch/x86_64/idt.c

@@ -3,6 +3,7 @@
 #include <zinnia/irq.h>
 #include <zinnia/panic.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
 #include <zinnia/usercopy.h>
@@ -93,6 +94,10 @@ void interrupt_handler(struct arch_context* ctx) {
         if (arch_sched_preempt_enable())
             sched_reschedule(&cpu->sched);
         break;
+    case IDT_IPI_SHOOTDOWN:
+        lapic_eoi(&arch_cpu->lapic);
+        pmap_shootdown_handle();
+        break;
     default:
         if (isr < 0x20)
             panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);

arch/x86_64/pmap.c

@@ -1,5 +1,10 @@
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/types.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 #include <x86_64/pmap.h>
 
 void pte_clear(pte_t* pte) {
@@ -10,6 +15,26 @@ void pte_flush(uintptr_t vaddr) {
     asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
 }
 
+void pte_flush_all() {
+    uint64_t cr3;
+    asm volatile("mov %0, cr3" : "=r"(cr3));
+    asm volatile("mov cr3, %0" ::"r"(cr3) : "memory");
+}
+
+void pte_shootdown_kick(struct percpu* cpu) {
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_SHOOTDOWN,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
 

arch/x86_64/sched.c

@@ -162,6 +162,7 @@ void arch_task_fini(struct arch_task_context* context) {
     const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
     for (size_t i = 0; i < fpu_size; i += page_size)
         pmap_unmap(&kernel_vma.pmap, fpu_base + i);
+    pmap_shootdown(&kernel_vma.pmap);
     mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
     vm_free(fpu_base, fpu_size);
 

core/mem/pmap.c

@@ -1,5 +1,8 @@
 #include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/spin.h>
 #include <zinnia/vma.h>
@@ -133,3 +136,103 @@ fail:
     spin_unlock(&pt->lock);
     return status;
 }
+
+errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte))
+        status = ENOENT;
+    else
+        *out = pte_address(pte);
+
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte)) {
+        status = ENOENT;
+        goto fail;
+    }
+
+    *pte = pte_build(pte_address(pte), flags, CACHE_ANY);
+    pte_flush(vaddr);
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+static void free_table(phys_t table, int8_t level) {
+    if (level > 0) {
+        pte_t* entries = HHDM_PTR(table);
+        const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+        for (size_t i = 0; i < num_entries; i++) {
+            if (pte_is_present(&entries[i]))
+                free_table(pte_address(&entries[i]), level - 1);
+        }
+    }
+    mem_phys_free(table, 1);
+}
+
+void pmap_destroy(struct pmap* pt) {
+    if (!pt->is_user)
+        return;
+
+    pte_t* root = HHDM_PTR(pt->root);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+    for (size_t i = 0; i < num_entries / 2; i++) {
+        if (pte_is_present(&root[i]))
+            free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
+    }
+    mem_phys_free(pt->root, 1);
+    pt->root = 0;
+}
+
+[[__percpu]]
+static atomic_size_t shootdown_req;
+
+[[__percpu]]
+static atomic_size_t shootdown_ack;
+
+void pmap_shootdown_handle() {
+    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
+    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
+        return;
+    pte_flush_all();
+    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
+}
+
+void pmap_shootdown(struct pmap* pt) {
+    const size_t count = percpu_count();
+    if (count < 2)
+        return;
+
+    irq_lock();
+    struct percpu* self = percpu_get();
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
+        pte_shootdown_kick(cpu);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
+        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
+            pmap_shootdown_handle();
+            asm volatile("pause");
+        }
+    }
+    irq_unlock();
+}

core/mem/stack.c

@@ -52,6 +52,7 @@ void stack_destroy(struct stack* ks) {
     const size_t page_size = arch_mem_page_size();
     for (size_t i = 0; i < ks->size; i += page_size)
         pmap_unmap(&kernel_vma.pmap, ks->base + i);
+    pmap_shootdown(&kernel_vma.pmap);
 
     mem_phys_free(ks->phys, ks->size / page_size);
     vm_free(ks->base - page_size, ks->size + page_size);

core/module.c

@@ -204,6 +204,10 @@ static void module_free(struct module* mod) {
         const struct module_segment* seg = &mod->segments[i];
         for (size_t off = 0; off < seg->size; off += page_size)
             pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+    }
+    pmap_shootdown(&kernel_vma.pmap);
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
         mem_phys_free(seg->phys, seg->size / page_size);
     }
     mem_free(mod->segments);

core/percpu.c

@@ -3,12 +3,11 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/compiler.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/utils.h>
+#include <zinnia/vma.h>
 #include <arch/percpu.h>
 
-extern uint8_t __ld_percpu_start[];
-extern uint8_t __ld_percpu_end[];
-
 static _Atomic(size_t) num_cpus = 1;
 
 static size_t max_cpus = SIZE_MAX;
@@ -25,20 +24,33 @@ struct percpu percpu_bsp = {
 };
 
 struct percpu* percpu_new() {
-    const size_t size = __ld_percpu_end - __ld_percpu_start;
-    const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
-
-    // TODO: Map pages past the end of the kernel.
-    // TODO: Document how allocations work.
-
-    phys_t mem;
-    if (mem_phys_alloc(num_pages, 0, &mem))
+    const size_t pgsz = arch_mem_page_size();
+    const size_t stride = percpu_stride();
+    const size_t id = atomic_load(&num_cpus, ATOMIC_RELAXED);
+    if (id >= (UINTPTR_MAX - (uintptr_t)__ld_percpu_start) / stride)
         return nullptr;
 
-    struct percpu* cpu = HHDM_PTR(mem);
+    phys_t mem;
+    if (mem_phys_alloc(stride / pgsz, 0, &mem))
+        return nullptr;
+
+    const uintptr_t base = (uintptr_t)percpu_cpu(id);
+    for (size_t offset = 0; offset < stride; offset += pgsz) {
+        if (pmap_map(&kernel_vma.pmap, base + offset, mem + offset, PTE_READ | PTE_WRITE, CACHE_ANY)) {
+            while (offset) {
+                offset -= pgsz;
+                pmap_unmap(&kernel_vma.pmap, base + offset);
+            }
+            mem_phys_free(mem, stride / pgsz);
+            return nullptr;
+        }
+    }
+
+    struct percpu* cpu = (struct percpu*)base;
     cpu->self = cpu;
-    cpu->id = atomic_fetch_add(&num_cpus, 1, ATOMIC_RELAXED);
+    cpu->id = id;
     cpu->online = false;
+    atomic_store(&num_cpus, id + 1, ATOMIC_RELEASE);
     return cpu;
 }
 
@@ -46,6 +58,10 @@ size_t percpu_max_cpus() {
     return max_cpus;
 }
 
+size_t percpu_count() {
+    return atomic_load(&num_cpus, ATOMIC_RELAXED);
+}
+
 struct percpu* percpu_get() {
     return arch_percpu_get();
 }

include/zinnia/percpu.h

@@ -5,6 +5,7 @@
 #include <zinnia/sched.h>
 #include <zinnia/usercopy.h>
 #include <stddef.h>
+#include <stdint.h>
 
 // # On using per-CPU data
 //
@@ -37,8 +38,20 @@ struct percpu {
     struct scheduler sched; // Scheduler information.
 };
 
+extern uint8_t __ld_percpu_start[];
+extern uint8_t __ld_percpu_end[];
+
+static inline size_t percpu_stride() {
+    return __ld_percpu_end - __ld_percpu_start;
+}
+
+static inline struct percpu* percpu_cpu(size_t id) {
+    return (struct percpu*)(__ld_percpu_start + id * percpu_stride());
+}
+
 struct percpu* percpu_new();
 size_t percpu_max_cpus();
+size_t percpu_count();
 struct percpu* percpu_get();
 void percpu_bsp_init();
 void percpu_init(struct boot_info* info);

include/zinnia/pmap.h

@@ -6,6 +6,8 @@
 #include <zinnia/utils.h>
 #include <bits/pmap.h>
 
+struct percpu;
+
 enum pte_flags : uint8_t {
     PTE_READ = 1 << 0,  // Can read from this page.
     PTE_WRITE = 1 << 1, // Can write to this page.
@@ -33,10 +35,16 @@ errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags);
 errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache);
 errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags);
 errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr);
+errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out);
+void pmap_destroy(struct pmap* pt);
 void pmap_set(struct pmap* pt);
+void pmap_shootdown(struct pmap* pt);
+void pmap_shootdown_handle();
 
 void pte_clear(pte_t* pte);
 void pte_flush(uintptr_t vaddr);
+void pte_flush_all();
+void pte_shootdown_kick(struct percpu* cpu);
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
 bool pte_is_present(pte_t* pte);
 bool pte_is_dir(pte_t* pte);