From 0a49ca75b53874e1319d802a0e8da147115a4de4 Mon Sep 17 00:00:00 2001 From: Matheus Date: Mon, 16 Sep 2024 17:34:05 -0300 Subject: [PATCH] mmu: arch_mmu_invalidate() -> arch_mmu_invalidate_range() --- kernel-src/arch/x86-64/mmu.c | 67 +++++++++++++++++----------- kernel-src/include/x86-64/arch/mmu.h | 3 +- kernel-src/mm/vmm.c | 1 - 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/kernel-src/arch/x86-64/mmu.c b/kernel-src/arch/x86-64/mmu.c index 3b65d70e..ddbb30a5 100644 --- a/kernel-src/arch/x86-64/mmu.c +++ b/kernel-src/arch/x86-64/mmu.c @@ -136,17 +136,12 @@ bool arch_mmu_map(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t fla return add_page(table, vaddr, entry, 0); } -void arch_mmu_invalidate(void *vaddr) { - arch_mmu_tlbshootdown(vaddr); - asm volatile ("invlpg (%%rax)" : : "a"(vaddr)); -} - void arch_mmu_unmap(pagetableptr_t table, void *vaddr) { uint64_t *entry = get_page(table, vaddr); if (entry == NULL) return; *entry = 0; - arch_mmu_invalidate(vaddr); + arch_mmu_invalidate_range(vaddr, PAGE_SIZE); } void arch_mmu_remap(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t flags) { @@ -155,7 +150,7 @@ void arch_mmu_remap(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t f return; uintptr_t addr = paddr == NULL ? (*entryptr & ADDRMASK) : ((uintptr_t)paddr & ADDRMASK); *entryptr = addr | flags; - arch_mmu_invalidate(vaddr); + arch_mmu_invalidate_range(vaddr, PAGE_SIZE); } void *arch_mmu_getphysical(pagetableptr_t table, void *vaddr) { @@ -209,35 +204,53 @@ pagetableptr_t arch_mmu_newtable() { return table; } -static void *mmupage = NULL; -static spinlock_t shootdownlock; -static int remaining = 0; +static void *shootdown_page; +static size_t shootdown_size; +static spinlock_t shootdown_lock; +static int shootdown_remaining = 0; -void arch_mmu_tlbipi(isr_t *isr, context_t *context) { - asm volatile ("invlpg (%%rax)" : : "a"(mmupage)); - __atomic_sub_fetch(&remaining, 1, __ATOMIC_SEQ_CST); +static inline void do_invalidate(void *page, size_t size) { + for (uintptr_t i = 0; i < size; i += PAGE_SIZE) { + uintptr_t ptr = (uintptr_t)page + i; + asm volatile ("invlpg (%%rax)" : : "a"(ptr) : "memory"); + } } -void arch_mmu_tlbshootdown(void *page) { - // scheduler not up yet, or there are no other cpus, nothing to do - if (current_thread() == NULL || arch_smp_cpusawake == 1) - return; +void arch_mmu_tlbipi(isr_t *isr, context_t *context) { + do_invalidate(shootdown_page, shootdown_size); + __atomic_sub_fetch(&shootdown_remaining, 1, __ATOMIC_SEQ_CST); +} - if (page >= KERNELSPACE_START || - (current_thread()->proc->runningthreadcount > 1 && page >= USERSPACE_START && page < USERSPACE_END)) { - // shoot down if page is in the kernel space or in a multithreaded userland application - int oldipl = interrupt_raiseipl(IPL_DPC); - spinlock_acquire(&shootdownlock); +void arch_mmu_invalidate_range(void *page, size_t size) { + __assert(((uintptr_t)page % PAGE_SIZE) == 0); + thread_t *thread = current_thread(); - mmupage = page; - remaining = arch_smp_cpusawake - 1; + bool do_shootdown = // do shootdown if + current_thread() // scheduler is up + && arch_smp_cpusawake >= 2 // and there are multiple cpus in the system + && (page >= KERNELSPACE_START // and its either in the kernel + || (page >= USERSPACE_START && page < USERSPACE_END // or in an userspace... + && thread->proc && thread->proc->runningthreadcount > 1)); // in a process which has multiple threads running + + int old_ipl; + if (do_shootdown) { + old_ipl = interrupt_raiseipl(IPL_DPC); + spinlock_acquire(&shootdown_lock); + + shootdown_page = page; + shootdown_size = size; + shootdown_remaining = arch_smp_cpusawake - 1; arch_smp_sendipi(NULL, ¤t_cpu()->isr[0xfe], ARCH_SMP_IPI_OTHERCPUS, false); + } - while (__atomic_load_n(&remaining, __ATOMIC_SEQ_CST)) CPU_PAUSE(); + do_invalidate(page, size); - spinlock_release(&shootdownlock); - interrupt_loweripl(oldipl); + if (do_shootdown) { + while (__atomic_load_n(&shootdown_remaining, __ATOMIC_SEQ_CST)) CPU_PAUSE(); + + spinlock_release(&shootdown_lock); + interrupt_loweripl(old_ipl); } } diff --git a/kernel-src/include/x86-64/arch/mmu.h b/kernel-src/include/x86-64/arch/mmu.h index 6c343d95..72599471 100644 --- a/kernel-src/include/x86-64/arch/mmu.h +++ b/kernel-src/include/x86-64/arch/mmu.h @@ -27,7 +27,6 @@ typedef uint64_t * pagetableptr_t; // physical address void arch_mmu_destroytable(pagetableptr_t table); bool arch_mmu_map(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t flags); -void arch_mmu_invalidate(void *vaddr); void arch_mmu_unmap(pagetableptr_t table, void *vaddr); void arch_mmu_remap(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t flags); void arch_mmu_switch(pagetableptr_t table); @@ -38,7 +37,7 @@ bool arch_mmu_isdirty(pagetableptr_t table, void *vaddr); pagetableptr_t arch_mmu_newtable(); void arch_mmu_init(); void arch_mmu_apswitch(); -void arch_mmu_tlbshootdown(void *page); +void arch_mmu_invalidate_range(void *page, size_t size); bool arch_mmu_getflags(pagetableptr_t table, void *vaddr, mmuflags_t *mmuflagsp); #endif diff --git a/kernel-src/mm/vmm.c b/kernel-src/mm/vmm.c index 94faaae3..0ad39e4a 100644 --- a/kernel-src/mm/vmm.c +++ b/kernel-src/mm/vmm.c @@ -292,7 +292,6 @@ static void changemmurange(vmmrange_t *range, void *base, size_t size, mmuflags_ // we will only change the mapping if the permissions decreased if (mask) { arch_mmu_remap(current_vmm_context()->pagetable, physical, address, currentflags & ~mask); - arch_mmu_invalidate(address); } } }