diff --git a/kernel-src/arch/x86-64/mmu.c b/kernel-src/arch/x86-64/mmu.c index ddbb30a5..c8237aeb 100644 --- a/kernel-src/arch/x86-64/mmu.c +++ b/kernel-src/arch/x86-64/mmu.c @@ -141,7 +141,6 @@ void arch_mmu_unmap(pagetableptr_t table, void *vaddr) { if (entry == NULL) return; *entry = 0; - arch_mmu_invalidate_range(vaddr, PAGE_SIZE); } void arch_mmu_remap(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t flags) { @@ -150,7 +149,6 @@ void arch_mmu_remap(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t f return; uintptr_t addr = paddr == NULL ? (*entryptr & ADDRMASK) : ((uintptr_t)paddr & ADDRMASK); *entryptr = addr | flags; - arch_mmu_invalidate_range(vaddr, PAGE_SIZE); } void *arch_mmu_getphysical(pagetableptr_t table, void *vaddr) { @@ -210,9 +208,15 @@ static spinlock_t shootdown_lock; static int shootdown_remaining = 0; static inline void do_invalidate(void *page, size_t size) { - for (uintptr_t i = 0; i < size; i += PAGE_SIZE) { - uintptr_t ptr = (uintptr_t)page + i; - asm volatile ("invlpg (%%rax)" : : "a"(ptr) : "memory"); + // if a full reload was requested or we are doing a big release on + if (page == NULL || size >= 128 * PAGE_SIZE) { + // TODO if global pages are ever supported, we should disable them in CR4, flush, and then reenable them + asm volatile("mov %%cr3, %%rax; mov %%rax, %%cr3;" : : : "rax", "memory"); + } else { + for (uintptr_t i = 0; i < size; i += PAGE_SIZE) { + uintptr_t ptr = (uintptr_t)page + i; + asm volatile ("invlpg (%%rax)" : : "a"(ptr) : "memory"); + } } } @@ -221,6 +225,7 @@ void arch_mmu_tlbipi(isr_t *isr, context_t *context) { __atomic_sub_fetch(&shootdown_remaining, 1, __ATOMIC_SEQ_CST); } +// if page == NULL, this will do a userspace shootdown that flushes the whole tlb void arch_mmu_invalidate_range(void *page, size_t size) { __assert(((uintptr_t)page % PAGE_SIZE) == 0); thread_t *thread = current_thread(); @@ -229,7 +234,7 @@ void arch_mmu_invalidate_range(void *page, size_t size) { current_thread() // scheduler is up && arch_smp_cpusawake >= 2 // and there are multiple cpus in the system && (page >= KERNELSPACE_START // and its either in the kernel - || (page >= USERSPACE_START && page < USERSPACE_END // or in an userspace... + || ((page == NULL || (page >= USERSPACE_START && page < USERSPACE_END)) // or in userspace... && thread->proc && thread->proc->runningthreadcount > 1)); // in a process which has multiple threads running int old_ipl; @@ -241,7 +246,12 @@ void arch_mmu_invalidate_range(void *page, size_t size) { shootdown_size = size; shootdown_remaining = arch_smp_cpusawake - 1; - arch_smp_sendipi(NULL, ¤t_cpu()->isr[0xfe], ARCH_SMP_IPI_OTHERCPUS, false); + for (int i = 0; i < arch_smp_cpusawake; ++i) { + if (smp_cpus[i] == current_cpu()) + continue; + + arch_smp_sendipi(smp_cpus[i], &smp_cpus[i]->isr[0xfe], ARCH_SMP_IPI_TARGET, false); + } } do_invalidate(page, size); diff --git a/kernel-src/io/fb.c b/kernel-src/io/fb.c index 2063de08..a555d32b 100644 --- a/kernel-src/io/fb.c +++ b/kernel-src/io/fb.c @@ -156,10 +156,11 @@ static int mmap(int minor, void *addr, uintmax_t offset, int flags) { static int munmap(int minor, void *addr, uintmax_t offset, int flags) { void *phys = arch_mmu_getphysical(current_vmm_context()->pagetable, addr); + bool release = (flags & V_FFLAGS_SHARED) && arch_mmu_iswritable(current_vmm_context()->pagetable, addr); arch_mmu_unmap(current_vmm_context()->pagetable, addr); - if ((flags & V_FFLAGS_SHARED) == 0) { + arch_mmu_invalidate_range(addr, PAGE_SIZE); + if (release) pmm_release(phys); - } return 0; } diff --git a/kernel-src/mm/vmm.c b/kernel-src/mm/vmm.c index 0ad39e4a..33def4da 100644 --- a/kernel-src/mm/vmm.c +++ b/kernel-src/mm/vmm.c @@ -317,9 +317,13 @@ static int changemap(vmmspace_t *space, void *address, size_t size, bool free, i int error = 0; - while (range) { + while (range && range->start >= top) { __assert(range != space->ranges || range->prev == NULL); void *rangetop = RANGE_TOP(range); + if ((uintptr_t)range->start >= (uintptr_t)address + size) + break; + + vmmrange_t *nextsave = range->next; if (range->start >= address && rangetop <= top) { // completely changed if (free) { @@ -452,7 +456,7 @@ static int changemap(vmmspace_t *space, void *address, size_t size, bool free, i if (free) { destroyrange(range, range->size, difference, 0); } else { - newrange->start = (void *)((uintptr_t)range->start + difference); + newrange->start = (void *)((uintptr_t)range->start + range->size + difference); newrange->size = difference; newrange->flags = range->flags; newrange->mmuflags = newmmuflags; @@ -461,7 +465,7 @@ static int changemap(vmmspace_t *space, void *address, size_t size, bool free, i if (range->flags & VMM_FLAGS_FILE) { newrange->vnode = range->vnode; - newrange->offset = range->offset + difference; + newrange->offset = range->offset + range->size + difference; VOP_HOLD(range->vnode); } @@ -474,7 +478,7 @@ static int changemap(vmmspace_t *space, void *address, size_t size, bool free, i } } - range = range->next; + range = nextsave; } leave: @@ -500,7 +504,10 @@ int vmm_changemmuflags(void *base, size_t size, mmuflags_t mmuflags, int flags) return ENOMEM; MUTEX_ACQUIRE(&space->lock, false); + int error = changemap(space, base, size, false, flags, mmuflags); + arch_mmu_invalidate_range(base, size); + MUTEX_RELEASE(&space->lock); return error; } @@ -634,6 +641,7 @@ bool vmm_pagefault(void *addr, bool user, int actions) { } else { memcpy(MAKE_HHDM(newphys), MAKE_HHDM(oldphys), PAGE_SIZE); arch_mmu_remap(current_vmm_context()->pagetable, newphys, addr, range->mmuflags); + arch_mmu_invalidate_range(addr, PAGE_SIZE); if ((range->flags & VMM_FLAGS_FILE) == 0 || vfs_iscacheable(range->vnode)) pmm_release(oldphys); @@ -704,6 +712,10 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, range->flags = VMM_PERMANENT_FLAGS_MASK & flags; range->mmuflags = mmuflags; __assert((flags & (VMM_FLAGS_ALLOCATE | VMM_FLAGS_PHYSICAL)) == 0); + // make the memory inacessible + changemap(space, addr, size, false, flags, 0); + arch_mmu_invalidate_range(addr, size); + // and then free it changemap(space, addr, size, true, flags, 0); } else { retaddr = start; @@ -729,6 +741,8 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, for (uintmax_t j = 0; j < size; j += PAGE_SIZE) arch_mmu_unmap(current_vmm_context()->pagetable, (void *)((uintptr_t)start + j)); + // invalidate here just to be sure + arch_mmu_invalidate_range(start, size); retaddr = NULL; goto cleanup; } @@ -753,6 +767,8 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, } } + // invalidate here just to be sure + arch_mmu_invalidate_range(start, size); retaddr = NULL; goto cleanup; } @@ -785,7 +801,14 @@ void vmm_unmap(void *addr, size_t size, int flags) { return; MUTEX_ACQUIRE(&space->lock, false); + + // make memory inacessible + changemap(space, addr, size, false, flags, 0); + arch_mmu_invalidate_range(addr, size); + + // and then free it changemap(space, addr, size, true, flags, 0); + MUTEX_RELEASE(&space->lock); } @@ -858,13 +881,17 @@ vmmcontext_t *vmm_fork(vmmcontext_t *oldcontext) { if (arch_mmu_map(newcontext->pagetable, phys, vaddr, newrange->mmuflags & ~ARCH_MMU_FLAGS_WRITE) == false) goto error; + pmm_hold(phys); + arch_mmu_remap(oldcontext->pagetable, phys, vaddr, newrange->mmuflags & ~ARCH_MMU_FLAGS_WRITE); } range = range->next; } + arch_mmu_invalidate_range(NULL, 0); + MUTEX_RELEASE(&oldcontext->space.lock); return newcontext; error: