diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig index ca3d411..aabf5bd 100644 --- a/arch/x86_64/Kconfig +++ b/arch/x86_64/Kconfig @@ -6,6 +6,7 @@ config X86_64 select ACPI select HAVE_MMU select HAVE_MMU_GENERIC + select HAVE_TLB_SHOOTDOWN_IPI help 64-bit x86 CPU support for AMD and Intel CPUs. diff --git a/arch/x86_64/pmap.c b/arch/x86_64/pmap.c index aa3b38c..926d5c7 100644 --- a/arch/x86_64/pmap.c +++ b/arch/x86_64/pmap.c @@ -11,6 +11,10 @@ void pte_clear(pte_t* pte) { *pte = 0; } +void pte_write(pte_t* pte, pte_t val) { + *pte = val; +} + void pte_flush(uintptr_t vaddr) { asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory"); } @@ -67,12 +71,19 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) { return result; } -bool pte_is_present(pte_t* pte) { - return *pte & ARCH_FLAG_PRESENT; +pte_t pte_protect(pte_t old, enum pte_flags flags) { + pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE); + if (flags & PTE_USER) + result |= ARCH_FLAG_USER_MODE; + if (flags & PTE_WRITE) + result |= ARCH_FLAG_READ_WRITE; + if (!(flags & PTE_EXEC)) + result |= ARCH_FLAG_EXECUTE_DISABLE; + return result; } -bool pte_is_dir(pte_t* pte) { - return *pte & ARCH_FLAG_SIZE; +bool pte_is_present(pte_t* pte) { + return *pte & ARCH_FLAG_PRESENT; } phys_t pte_address(pte_t* pte) { diff --git a/core/mem/Kconfig b/core/mem/Kconfig index ea463b6..b66cc1f 100644 --- a/core/mem/Kconfig +++ b/core/mem/Kconfig @@ -4,5 +4,9 @@ config HAVE_MMU bool config HAVE_MMU_GENERIC bool +config HAVE_MMU_SPLIT_ROOT + bool +config HAVE_TLB_SHOOTDOWN_IPI + bool endmenu diff --git a/core/mem/meson.build b/core/mem/meson.build index a510624..9cbcf9b 100644 --- a/core/mem/meson.build +++ b/core/mem/meson.build @@ -15,3 +15,7 @@ if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y' 'pmap.c', ) endif + +if kconfig.get('CONFIG_HAVE_TLB_SHOOTDOWN_IPI', 'n') == 'y' + kernel_sources += files('shootdown.c') +endif diff --git a/core/mem/pmap.c b/core/mem/pmap.c index 8bf1524..4727d55 100644 --- a/core/mem/pmap.c +++ b/core/mem/pmap.c @@ -1,12 +1,8 @@ #include -#include #include -#include -#include #include #include #include -#include #include // TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU. @@ -20,6 +16,7 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) { if (status) return status; +#if !CONFIG_HAVE_MMU_SPLIT_ROOT pte_t* root = HHDM_PTR(addr); const size_t num_entries = (size_t)1 << arch_mem_level_bits(); for (size_t i = num_entries / 2; i < num_entries; i++) { @@ -27,8 +24,9 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) { status = mem_phys_alloc(1, flags, &dir); if (status) return status; - root[i] = pte_build(dir, PTE_DIR, CACHE_ANY); + pte_write(&root[i], pte_build(dir, PTE_DIR, CACHE_ANY)); } +#endif struct pmap result = { .root = addr, @@ -42,13 +40,23 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) { errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) { phys_t user_l1; - errno_t status = mem_phys_alloc(1, 0, &user_l1); + + // If the kernel and userspace PTEs are on separate page tables, + // allocate them individually. Otherwise, copy over the higher half PTEs + // into the new user page table from the kernel VMA. +#if CONFIG_HAVE_MMU_SPLIT_ROOT + errno_t status = mem_phys_alloc(1, flags & ~ALLOC_NOZERO, &user_l1); + if (status) + return status; +#else + errno_t status = mem_phys_alloc(1, flags, &user_l1); if (status) return status; void* user_l1_ptr = HHDM_PTR(user_l1); void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root); memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size()); +#endif struct pmap result = { .root = user_l1, @@ -84,7 +92,9 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check if (pte_is_present(pte)) { // Get the next level. - *pte |= pte_build(0, level_flags, CACHE_ANY); + const pte_t upgraded = *pte | pte_build(0, level_flags, CACHE_ANY); + if (upgraded != *pte) + pte_write(pte, upgraded); current_head = HHDM_PTR(pte_address(pte)); } else { // If the current level isn't present, we can skip the rest. @@ -95,7 +105,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check if (mem_phys_alloc(1, 0, &addr) != 0) return nullptr; - *pte = pte_build(addr, level_flags, CACHE_ANY); + pte_write(pte, pte_build(addr, level_flags, CACHE_ANY)); current_head = HHDM_PTR(addr); } } @@ -113,7 +123,11 @@ errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags goto fail; } - *pte = pte_build(paddr, flags, cache); + if (pte_is_present(pte)) { + pte_clear(pte); + pte_flush(vaddr); + } + pte_write(pte, pte_build(paddr, flags, cache)); fail: spin_unlock(&pt->lock); @@ -162,7 +176,7 @@ errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) { goto fail; } - *pte = pte_build(pte_address(pte), flags, CACHE_ANY); + pte_write(pte, pte_protect(*pte, flags)); pte_flush(vaddr); fail: @@ -188,52 +202,15 @@ void pmap_destroy(struct pmap* pt) { pte_t* root = HHDM_PTR(pt->root); const size_t num_entries = (size_t)1 << arch_mem_level_bits(); - for (size_t i = 0; i < num_entries / 2; i++) { +#if CONFIG_HAVE_MMU_SPLIT_ROOT + const size_t user_entries = num_entries; +#else + const size_t user_entries = num_entries / 2; +#endif + for (size_t i = 0; i < user_entries; i++) { if (pte_is_present(&root[i])) free_table(pte_address(&root[i]), arch_mem_num_levels() - 2); } mem_phys_free(pt->root, 1); pt->root = 0; } - -[[__percpu]] -static atomic_size_t shootdown_req; - -[[__percpu]] -static atomic_size_t shootdown_ack; - -void pmap_shootdown_handle() { - const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE); - if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req) - return; - pte_flush_all(); - atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE); -} - -void pmap_shootdown(struct pmap* pt) { - const size_t count = percpu_count(); - if (count < 2) - return; - - irq_lock(); - struct percpu* self = percpu_get(); - for (size_t i = 0; i < count; i++) { - struct percpu* cpu = percpu_cpu(i); - if (cpu == self || !cpu->online) - continue; - atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE); - pte_shootdown_kick(cpu); - } - - for (size_t i = 0; i < count; i++) { - struct percpu* cpu = percpu_cpu(i); - if (cpu == self || !cpu->online) - continue; - const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED); - while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) { - pmap_shootdown_handle(); - arch_cpu_pause(); - } - } - irq_unlock(); -} diff --git a/core/mem/shootdown.c b/core/mem/shootdown.c new file mode 100644 index 0000000..f828ff8 --- /dev/null +++ b/core/mem/shootdown.c @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include + +[[__percpu]] +static atomic_size_t shootdown_req; + +[[__percpu]] +static atomic_size_t shootdown_ack; + +void pmap_shootdown_handle() { + const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE); + if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req) + return; + pte_flush_all(); + atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE); +} + +void pmap_shootdown(struct pmap* pt) { + const size_t count = percpu_count(); + if (count < 2) + return; + + irq_lock(); + struct percpu* self = percpu_get(); + for (size_t i = 0; i < count; i++) { + struct percpu* cpu = percpu_cpu(i); + if (cpu == self || !cpu->online) + continue; + atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE); + pte_shootdown_kick(cpu); + } + + for (size_t i = 0; i < count; i++) { + struct percpu* cpu = percpu_cpu(i); + if (cpu == self || !cpu->online) + continue; + const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED); + while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) { + pmap_shootdown_handle(); + arch_cpu_pause(); + } + } + irq_unlock(); +} diff --git a/include/zinnia/pmap.h b/include/zinnia/pmap.h index 5e2c9e9..679450c 100644 --- a/include/zinnia/pmap.h +++ b/include/zinnia/pmap.h @@ -5,6 +5,7 @@ #include #include #include +#include struct percpu; @@ -39,13 +40,16 @@ errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out); void pmap_destroy(struct pmap* pt); void pmap_set(struct pmap* pt); void pmap_shootdown(struct pmap* pt); -void pmap_shootdown_handle(); void pte_clear(pte_t* pte); +void pte_write(pte_t* pte, pte_t val); void pte_flush(uintptr_t vaddr); void pte_flush_all(); +#if CONFIG_HAVE_TLB_SHOOTDOWN_IPI +void pmap_shootdown_handle(); void pte_shootdown_kick(struct percpu* cpu); +#endif pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache); +pte_t pte_protect(pte_t old, enum pte_flags flags); bool pte_is_present(pte_t* pte); -bool pte_is_dir(pte_t* pte); phys_t pte_address(pte_t* pte);