IPI TLB shootdowns

arch/x86_64/Kconfig

@@ -6,6 +6,7 @@ config X86_64
   select ACPI
   select HAVE_MMU
   select HAVE_MMU_GENERIC
+  select HAVE_TLB_SHOOTDOWN_IPI
   help
     64-bit x86 CPU support for AMD and Intel CPUs.
 

arch/x86_64/pmap.c

@@ -11,6 +11,10 @@ void pte_clear(pte_t* pte) {
     *pte = 0;
 }
 
+void pte_write(pte_t* pte, pte_t val) {
+    *pte = val;
+}
+
 void pte_flush(uintptr_t vaddr) {
     asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
 }
@@ -67,12 +71,19 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     return result;
 }
 
-bool pte_is_present(pte_t* pte) {
+pte_t pte_protect(pte_t old, enum pte_flags flags) {
-    return *pte & ARCH_FLAG_PRESENT;
+    pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+    if (flags & PTE_WRITE)
+        result |= ARCH_FLAG_READ_WRITE;
+    if (!(flags & PTE_EXEC))
+        result |= ARCH_FLAG_EXECUTE_DISABLE;
+    return result;
 }
 
-bool pte_is_dir(pte_t* pte) {
+bool pte_is_present(pte_t* pte) {
-    return *pte & ARCH_FLAG_SIZE;
+    return *pte & ARCH_FLAG_PRESENT;
 }
 
 phys_t pte_address(pte_t* pte) {

core/mem/Kconfig

@@ -4,5 +4,9 @@ config HAVE_MMU
   bool
 config HAVE_MMU_GENERIC
   bool
+config HAVE_MMU_SPLIT_ROOT
+  bool
+config HAVE_TLB_SHOOTDOWN_IPI
+  bool
 
 endmenu

core/mem/meson.build

@@ -15,3 +15,7 @@ if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y'
         'pmap.c',
     )
 endif
+
+if kconfig.get('CONFIG_HAVE_TLB_SHOOTDOWN_IPI', 'n') == 'y'
+    kernel_sources += files('shootdown.c')
+endif

core/mem/pmap.c

@@ -1,12 +1,8 @@
 #include <zinnia/alloc.h>
-#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
-#include <zinnia/irq.h>
-#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/spin.h>
 #include <zinnia/vma.h>
-#include <arch/cpu.h>
 #include <config.h>
 
 // TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
@@ -20,6 +16,7 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
     if (status)
         return status;
 
+#if !CONFIG_HAVE_MMU_SPLIT_ROOT
     pte_t* root = HHDM_PTR(addr);
     const size_t num_entries = (size_t)1 << arch_mem_level_bits();
     for (size_t i = num_entries / 2; i < num_entries; i++) {
@@ -27,8 +24,9 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
         status = mem_phys_alloc(1, flags, &dir);
         if (status)
             return status;
-        root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
+        pte_write(&root[i], pte_build(dir, PTE_DIR, CACHE_ANY));
     }
+#endif
 
     struct pmap result = {
         .root = addr,
@@ -42,13 +40,23 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
 
 errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
     phys_t user_l1;
-    errno_t status = mem_phys_alloc(1, 0, &user_l1);
+
+    // If the kernel and userspace PTEs are on separate page tables,
+    // allocate them individually. Otherwise, copy over the higher half PTEs
+    // into the new user page table from the kernel VMA.
+#if CONFIG_HAVE_MMU_SPLIT_ROOT
+    errno_t status = mem_phys_alloc(1, flags & ~ALLOC_NOZERO, &user_l1);
+    if (status)
+        return status;
+#else
+    errno_t status = mem_phys_alloc(1, flags, &user_l1);
     if (status)
         return status;
 
     void* user_l1_ptr = HHDM_PTR(user_l1);
     void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
     memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
+#endif
 
     struct pmap result = {
         .root = user_l1,
@@ -84,7 +92,9 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
 
         if (pte_is_present(pte)) {
             // Get the next level.
-            *pte |= pte_build(0, level_flags, CACHE_ANY);
+            const pte_t upgraded = *pte | pte_build(0, level_flags, CACHE_ANY);
+            if (upgraded != *pte)
+                pte_write(pte, upgraded);
             current_head = HHDM_PTR(pte_address(pte));
         } else {
             // If the current level isn't present, we can skip the rest.
@@ -95,7 +105,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
             if (mem_phys_alloc(1, 0, &addr) != 0)
                 return nullptr;
 
-            *pte = pte_build(addr, level_flags, CACHE_ANY);
+            pte_write(pte, pte_build(addr, level_flags, CACHE_ANY));
             current_head = HHDM_PTR(addr);
         }
     }
@@ -113,7 +123,11 @@ errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags
         goto fail;
     }
 
-    *pte = pte_build(paddr, flags, cache);
+    if (pte_is_present(pte)) {
+        pte_clear(pte);
+        pte_flush(vaddr);
+    }
+    pte_write(pte, pte_build(paddr, flags, cache));
 
 fail:
     spin_unlock(&pt->lock);
@@ -162,7 +176,7 @@ errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
         goto fail;
     }
 
-    *pte = pte_build(pte_address(pte), flags, CACHE_ANY);
+    pte_write(pte, pte_protect(*pte, flags));
     pte_flush(vaddr);
 
 fail:
@@ -188,52 +202,15 @@ void pmap_destroy(struct pmap* pt) {
 
     pte_t* root = HHDM_PTR(pt->root);
     const size_t num_entries = (size_t)1 << arch_mem_level_bits();
-    for (size_t i = 0; i < num_entries / 2; i++) {
+#if CONFIG_HAVE_MMU_SPLIT_ROOT
+    const size_t user_entries = num_entries;
+#else
+    const size_t user_entries = num_entries / 2;
+#endif
+    for (size_t i = 0; i < user_entries; i++) {
         if (pte_is_present(&root[i]))
             free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
     }
     mem_phys_free(pt->root, 1);
     pt->root = 0;
 }
-
-[[__percpu]]
-static atomic_size_t shootdown_req;
-
-[[__percpu]]
-static atomic_size_t shootdown_ack;
-
-void pmap_shootdown_handle() {
-    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
-    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
-        return;
-    pte_flush_all();
-    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
-}
-
-void pmap_shootdown(struct pmap* pt) {
-    const size_t count = percpu_count();
-    if (count < 2)
-        return;
-
-    irq_lock();
-    struct percpu* self = percpu_get();
-    for (size_t i = 0; i < count; i++) {
-        struct percpu* cpu = percpu_cpu(i);
-        if (cpu == self || !cpu->online)
-            continue;
-        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
-        pte_shootdown_kick(cpu);
-    }
-
-    for (size_t i = 0; i < count; i++) {
-        struct percpu* cpu = percpu_cpu(i);
-        if (cpu == self || !cpu->online)
-            continue;
-        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
-        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
-            pmap_shootdown_handle();
-            arch_cpu_pause();
-        }
-    }
-    irq_unlock();
-}

core/mem/shootdown.c

@@ -0,0 +1,47 @@
+#include <zinnia/atomic.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <arch/cpu.h>
+
+[[__percpu]]
+static atomic_size_t shootdown_req;
+
+[[__percpu]]
+static atomic_size_t shootdown_ack;
+
+void pmap_shootdown_handle() {
+    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
+    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
+        return;
+    pte_flush_all();
+    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
+}
+
+void pmap_shootdown(struct pmap* pt) {
+    const size_t count = percpu_count();
+    if (count < 2)
+        return;
+
+    irq_lock();
+    struct percpu* self = percpu_get();
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
+        pte_shootdown_kick(cpu);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
+        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
+            pmap_shootdown_handle();
+            arch_cpu_pause();
+        }
+    }
+    irq_unlock();
+}

include/zinnia/pmap.h

@@ -5,6 +5,7 @@
 #include <zinnia/spin.h>
 #include <zinnia/utils.h>
 #include <bits/pmap.h>
+#include <config.h>
 
 struct percpu;
 
@@ -39,13 +40,16 @@ errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out);
 void pmap_destroy(struct pmap* pt);
 void pmap_set(struct pmap* pt);
 void pmap_shootdown(struct pmap* pt);
-void pmap_shootdown_handle();
 
 void pte_clear(pte_t* pte);
+void pte_write(pte_t* pte, pte_t val);
 void pte_flush(uintptr_t vaddr);
 void pte_flush_all();
+#if CONFIG_HAVE_TLB_SHOOTDOWN_IPI
+void pmap_shootdown_handle();
 void pte_shootdown_kick(struct percpu* cpu);
+#endif
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
+pte_t pte_protect(pte_t old, enum pte_flags flags);
 bool pte_is_present(pte_t* pte);
-bool pte_is_dir(pte_t* pte);
 phys_t pte_address(pte_t* pte);