marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IPI TLB shootdowns

Browse Source
This commit is contained in:
Marvin
2026-10-18 21:29:47 +02:00
parent 7aaf953a1c
commit 70ca5115df
7 changed files with 107 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/x86_64/Kconfig
+1
View File
@@ -6,6 +6,7 @@ config X86_64
select ACPI
select HAVE_MMU
select HAVE_MMU_GENERIC
select HAVE_TLB_SHOOTDOWN_IPI
help
64-bit x86 CPU support for AMD and Intel CPUs.
arch/x86_64/pmap.c
+15 -4
View File
@@ -11,6 +11,10 @@ void pte_clear(pte_t* pte) {
*pte = 0;
}
void pte_write(pte_t* pte, pte_t val) {
*pte = val;
}
void pte_flush(uintptr_t vaddr) {
asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
}
@@ -67,12 +71,19 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
return result;
}
bool pte_is_present(pte_t* pte) {
return *pte & ARCH_FLAG_PRESENT;
pte_t pte_protect(pte_t old, enum pte_flags flags) {
pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
if (flags & PTE_USER)
result |= ARCH_FLAG_USER_MODE;
if (flags & PTE_WRITE)
result |= ARCH_FLAG_READ_WRITE;
if (!(flags & PTE_EXEC))
result |= ARCH_FLAG_EXECUTE_DISABLE;
return result;
}
bool pte_is_dir(pte_t* pte) {
return *pte & ARCH_FLAG_SIZE;
bool pte_is_present(pte_t* pte) {
return *pte & ARCH_FLAG_PRESENT;
}
phys_t pte_address(pte_t* pte) {
core/mem/Kconfig
+4
View File
@@ -4,5 +4,9 @@ config HAVE_MMU
bool
config HAVE_MMU_GENERIC
bool
config HAVE_MMU_SPLIT_ROOT
bool
config HAVE_TLB_SHOOTDOWN_IPI
bool
endmenu
core/mem/meson.build
+4
View File
@@ -15,3 +15,7 @@ if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y'
'pmap.c',
)
endif
if kconfig.get('CONFIG_HAVE_TLB_SHOOTDOWN_IPI', 'n') == 'y'
kernel_sources += files('shootdown.c')
endif
core/mem/pmap.c
+30 -53
View File
@@ -1,12 +1,8 @@
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/spin.h>
#include <zinnia/vma.h>
#include <arch/cpu.h>
#include <config.h>
// TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
@@ -20,6 +16,7 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
if (status)
return status;
#if !CONFIG_HAVE_MMU_SPLIT_ROOT
pte_t* root = HHDM_PTR(addr);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = num_entries / 2; i < num_entries; i++) {
@@ -27,8 +24,9 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
status = mem_phys_alloc(1, flags, &dir);
if (status)
return status;
root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
pte_write(&root[i], pte_build(dir, PTE_DIR, CACHE_ANY));
}
#endif
struct pmap result = {
.root = addr,
@@ -42,13 +40,23 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
phys_t user_l1;
errno_t status = mem_phys_alloc(1, 0, &user_l1);
// If the kernel and userspace PTEs are on separate page tables,
// allocate them individually. Otherwise, copy over the higher half PTEs
// into the new user page table from the kernel VMA.
#if CONFIG_HAVE_MMU_SPLIT_ROOT
errno_t status = mem_phys_alloc(1, flags & ~ALLOC_NOZERO, &user_l1);
if (status)
return status;
#else
errno_t status = mem_phys_alloc(1, flags, &user_l1);
if (status)
return status;
void* user_l1_ptr = HHDM_PTR(user_l1);
void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
#endif
struct pmap result = {
.root = user_l1,
@@ -84,7 +92,9 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
if (pte_is_present(pte)) {
// Get the next level.
*pte |= pte_build(0, level_flags, CACHE_ANY);
const pte_t upgraded = *pte | pte_build(0, level_flags, CACHE_ANY);
if (upgraded != *pte)
pte_write(pte, upgraded);
current_head = HHDM_PTR(pte_address(pte));
} else {
// If the current level isn't present, we can skip the rest.
@@ -95,7 +105,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
if (mem_phys_alloc(1, 0, &addr) != 0)
return nullptr;
*pte = pte_build(addr, level_flags, CACHE_ANY);
pte_write(pte, pte_build(addr, level_flags, CACHE_ANY));
current_head = HHDM_PTR(addr);
}
}
@@ -113,7 +123,11 @@ errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags
goto fail;
}
*pte = pte_build(paddr, flags, cache);
if (pte_is_present(pte)) {
pte_clear(pte);
pte_flush(vaddr);
}
pte_write(pte, pte_build(paddr, flags, cache));
fail:
spin_unlock(&pt->lock);
@@ -162,7 +176,7 @@ errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
goto fail;
}
*pte = pte_build(pte_address(pte), flags, CACHE_ANY);
pte_write(pte, pte_protect(*pte, flags));
pte_flush(vaddr);
fail:
@@ -188,52 +202,15 @@ void pmap_destroy(struct pmap* pt) {
pte_t* root = HHDM_PTR(pt->root);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = 0; i < num_entries / 2; i++) {
#if CONFIG_HAVE_MMU_SPLIT_ROOT
const size_t user_entries = num_entries;
#else
const size_t user_entries = num_entries / 2;
#endif
for (size_t i = 0; i < user_entries; i++) {
if (pte_is_present(&root[i]))
free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
}
mem_phys_free(pt->root, 1);
pt->root = 0;
}
[[__percpu]]
static atomic_size_t shootdown_req;
[[__percpu]]
static atomic_size_t shootdown_ack;
void pmap_shootdown_handle() {
const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
return;
pte_flush_all();
atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
}
void pmap_shootdown(struct pmap* pt) {
const size_t count = percpu_count();
if (count < 2)
return;
irq_lock();
struct percpu* self = percpu_get();
for (size_t i = 0; i < count; i++) {
struct percpu* cpu = percpu_cpu(i);
if (cpu == self || !cpu->online)
continue;
atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
pte_shootdown_kick(cpu);
}
for (size_t i = 0; i < count; i++) {
struct percpu* cpu = percpu_cpu(i);
if (cpu == self || !cpu->online)
continue;
const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
pmap_shootdown_handle();
arch_cpu_pause();
}
}
irq_unlock();
}
core/mem/shootdown.c
+47
View File
@@ -0,0 +1,47 @@
#include <zinnia/atomic.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <arch/cpu.h>
[[__percpu]]
static atomic_size_t shootdown_req;
[[__percpu]]
static atomic_size_t shootdown_ack;
void pmap_shootdown_handle() {
const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
return;
pte_flush_all();
atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
}
void pmap_shootdown(struct pmap* pt) {
const size_t count = percpu_count();
if (count < 2)
return;
irq_lock();
struct percpu* self = percpu_get();
for (size_t i = 0; i < count; i++) {
struct percpu* cpu = percpu_cpu(i);
if (cpu == self || !cpu->online)
continue;
atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
pte_shootdown_kick(cpu);
}
for (size_t i = 0; i < count; i++) {
struct percpu* cpu = percpu_cpu(i);
if (cpu == self || !cpu->online)
continue;
const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
pmap_shootdown_handle();
arch_cpu_pause();
}
}
irq_unlock();
}
include/zinnia/pmap.h
+6 -2
View File
@@ -5,6 +5,7 @@
#include <zinnia/spin.h>
#include <zinnia/utils.h>
#include <bits/pmap.h>
#include <config.h>
struct percpu;
@@ -39,13 +40,16 @@ errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out);
void pmap_destroy(struct pmap* pt);
void pmap_set(struct pmap* pt);
void pmap_shootdown(struct pmap* pt);
void pmap_shootdown_handle();
void pte_clear(pte_t* pte);
void pte_write(pte_t* pte, pte_t val);
void pte_flush(uintptr_t vaddr);
void pte_flush_all();
#if CONFIG_HAVE_TLB_SHOOTDOWN_IPI
void pmap_shootdown_handle();
void pte_shootdown_kick(struct percpu* cpu);
#endif
pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
pte_t pte_protect(pte_t old, enum pte_flags flags);
bool pte_is_present(pte_t* pte);
bool pte_is_dir(pte_t* pte);
phys_t pte_address(pte_t* pte);