diff --git a/kernel-src/include/kernel/mm.h b/kernel-src/include/kernel/mm.h index 051dff7d..a55f32f0 100644 --- a/kernel-src/include/kernel/mm.h +++ b/kernel-src/include/kernel/mm.h @@ -3,6 +3,7 @@ #include #include +#include extern size_t mm_cache_cached_pages; @@ -16,4 +17,13 @@ int mm_cache_push_page(vnode_t *vnode, uintmax_t offset, page_t *page); int mm_cache_sync(void); int mm_cache_evict(page_t *page); +void mm_range_init(void); +vmmrange_t *mm_alloc_range(void); +void mm_free_range(vmmrange_t *range); +vmmrange_t *mm_get_range(vmmspace_t *space, void *addr); +void *mm_get_free_range(vmmspace_t *space, void *addr, size_t size); +void mm_insert_range(vmmspace_t *space, vmmrange_t *new_range); +void mm_destroy_range(vmmrange_t *range, uintmax_t offset, size_t size, int flags); +int mm_change_range(vmmspace_t *space, void *address, size_t size, bool free, int flags, mmuflags_t new_mmuflags); + #endif diff --git a/kernel-src/mm/range.c b/kernel-src/mm/range.c new file mode 100644 index 00000000..66a773e2 --- /dev/null +++ b/kernel-src/mm/range.c @@ -0,0 +1,448 @@ +#include +#include +#include +#include +#include +#include +#include + +#define RANGE_TOP(x) (void *)((uintptr_t)x->start + x->size) + +static scache_t *range_cache; + +void mm_range_init(void) { + range_cache = slab_create_new_cache_from_pmm(sizeof(vmmrange_t), 0, NULL, NULL); + __assert(range_cache); +} + +vmmrange_t *mm_alloc_range(void) { + return slab_allocate(range_cache); +} + +void mm_free_range(vmmrange_t *range) { + slab_free(range_cache, range); +} + +static int rbtree_value_compare(void *addr, rbtree_t *node) { + vmmrange_t *range = container_of(node, vmmrange_t, rbtree_node); + void *range_top = RANGE_TOP(range); + + if (addr >= range_top) + return 1; + + if (addr < range->start) + return -1; + + return 0; +} + +// get a range from an address +vmmrange_t *mm_get_range(vmmspace_t *space, void *addr) { + rbtree_t *rbtree = rbtree_lookup(space->ranges, addr, rbtree_value_compare); + return container_of(rbtree, vmmrange_t, rbtree_node); +} + +// get start of range that fits specific size from specific offset +void *mm_get_free_range(vmmspace_t *space, void *addr, size_t size) { + rbtree_t *rbtree = space->ranges; + if (addr == NULL) + addr = space->start; + + // no ranges + if (rbtree == NULL) + return addr; + + rbtree = rbtree_first(rbtree); + vmmrange_t *range = container_of(rbtree, vmmrange_t, rbtree_node); + + // if theres free space before the first range + if (range->start != space->start && addr < range->start && (uintptr_t)range->start - (uintptr_t)addr >= size) + return addr; + + // check for space between two ranges + rbtree_t *next = rbtree_successor(rbtree); + while (next) { + vmmrange_t *next_range = container_of(next, vmmrange_t, rbtree_node); + + void *rangetop = RANGE_TOP(range); + if (addr < rangetop) + addr = rangetop; + + if (addr < next_range->start) { + size_t freesize = (uintptr_t)next_range->start - (uintptr_t)addr; + if (freesize >= size) + return addr; + } + + rbtree = next; + range = container_of(rbtree, vmmrange_t, rbtree_node); + + next = rbtree_successor(next); + } + + // if theres free space after the last range + void *rangetop = RANGE_TOP(range); + if (addr < rangetop) + addr = rangetop; + + if (addr != space->end && (uintptr_t)space->end - (uintptr_t)addr >= size) + return addr; + + return NULL; +} + +static int rbtree_compare(rbtree_t *a, rbtree_t *b) { + vmmrange_t *range_a = container_of(a, vmmrange_t, rbtree_node); + vmmrange_t *range_b = container_of(b, vmmrange_t, rbtree_node); + + if (range_a->start == range_b->start) + return 0; + + return range_a->start > range_b->start ? 1 : -1; +} + +static bool compatible(vmmrange_t *prev, vmmrange_t *next) { + void *prev_range_top = RANGE_TOP(prev); + + return prev_range_top == next->start && prev->flags == next->flags && prev->mmuflags == next->mmuflags && // general compatibility + ((prev->flags & VMM_FLAGS_FILE) == 0 || (prev->vnode == next->vnode && prev->offset + prev->size == next->offset)); // same file mapping, if any +} + +void mm_insert_range(vmmspace_t *space, vmmrange_t *new_range) { + rbtree_insert(&space->ranges, &new_range->rbtree_node, rbtree_compare); + + rbtree_t *successor = rbtree_successor(&new_range->rbtree_node); + rbtree_t *predecessor = rbtree_predecessor(&new_range->rbtree_node); + + vmmrange_t *next_range = successor ? container_of(successor, vmmrange_t, rbtree_node) : NULL; + vmmrange_t *prev_range = predecessor ? container_of(predecessor, vmmrange_t, rbtree_node) : NULL; + + // fragmentation checking + + // check next range + if (next_range && compatible(new_range, next_range)) { + new_range->size += next_range->size; + + rbtree_remove(&space->ranges, &next_range->rbtree_node); + mm_free_range(next_range); + if (new_range->flags & VMM_FLAGS_FILE) { + VOP_RELEASE(new_range->vnode); + } + } + + // check prev range + if (prev_range && compatible(prev_range, new_range)) { + prev_range->size += new_range->size; + + rbtree_remove(&space->ranges, &new_range->rbtree_node); + mm_free_range(new_range); + if (prev_range->flags & VMM_FLAGS_FILE) { + VOP_RELEASE(prev_range->vnode); + } + } +} + +void mm_destroy_range(vmmrange_t *range, uintmax_t _offset, size_t size, int flags) { + // TODO find first + uintmax_t top = _offset + size; + + for (uintmax_t offset = _offset; offset < top; offset += PAGE_SIZE) { + void *vaddr = (void *)((uintptr_t)range->start + offset); + void *physical = arch_mmu_getphysical(current_vmm_context()->pagetable, vaddr); + if (physical == NULL) + continue; + + thread_t *thread = current_thread(); + proc_t *proc = thread ? thread->proc : NULL; + cred_t *cred = proc ? &proc->cred : NULL; + + if ((range->flags & VMM_FLAGS_FILE) && ((range->flags & VMM_FLAGS_SHARED) || vfs_iscacheable(range->vnode) == false)) { + // shared file mapping or non cacheable mapping + if (vfs_iscacheable(range->vnode) == false) { + // non cacheable mapping + VOP_LOCK(range->vnode); + __assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0); + VOP_UNLOCK(range->vnode); + } else if (arch_mmu_iswritable(current_vmm_context()->pagetable, vaddr)) { + // dirty page cache mapping + arch_mmu_unmap(current_vmm_context()->pagetable, vaddr); + VOP_LOCK(range->vnode); + mm_cache_make_dirty(mm_get_page(physical)); + VOP_UNLOCK(range->vnode); + mm_release_page(physical); + } else { + // non dirty page mapping + arch_mmu_unmap(current_vmm_context()->pagetable, vaddr); + mm_release_page(physical); + } + } else { + // anonymous, physical or private non character device mapping + arch_mmu_unmap(current_vmm_context()->pagetable, vaddr); + if ((range->flags & VMM_FLAGS_PHYSICAL) == 0) + mm_release_page(physical); + } + } + + if ((range->flags & VMM_FLAGS_FILE) && range->size == size) + VOP_RELEASE(range->vnode); +} + +#define CHANGE_MASK_CHECK(m, f, c, n) \ + if (((n) & (f)) == 0 && ((c) & (f))) \ + m |= f; + +static void change_mmu_range(vmmrange_t *range, void *base, size_t size, mmuflags_t newflags) { + for (uintmax_t offset = 0; offset < size; offset += PAGE_SIZE) { + void *address = (void *)((uintptr_t)base + offset); + + mmuflags_t currentflags; + // if page is not mapped, do nothing + if (arch_mmu_getflags(current_vmm_context()->pagetable, address, ¤tflags) == false) + continue; + + void *physical = arch_mmu_getphysical(current_vmm_context()->pagetable, address); + + uintmax_t mask = 0; + // check which flags are currently set and will be unset + CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_READ, currentflags, newflags); + CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_WRITE, currentflags, newflags); + CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_USER, currentflags, newflags); + CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_NOEXEC, currentflags, newflags); + + if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED) && (mask & ARCH_MMU_FLAGS_WRITE) && vfs_iscacheable(range->vnode)) { + // removing write permissions from a writeable dirty shared mapped page, mark it as dirty, as + // it won't be marked dirty upon a vmm_unmap after this + page_t *page = mm_get_page(physical); + VOP_LOCK(range->vnode); + mm_cache_make_dirty(page); + VOP_UNLOCK(range->vnode); + } + + // we will only change the mapping if the permissions decreased + if (mask) { + arch_mmu_remap(current_vmm_context()->pagetable, physical, address, currentflags & ~mask); + } + } +} + +static inline bool can_write_vnode(vmmrange_t *range) { + VOP_LOCK(range->vnode); + int error = VOP_ACCESS(range->vnode, V_ACCESS_WRITE, ¤t_thread()->proc->cred); + VOP_UNLOCK(range->vnode); + return error == 0; +} + +int mm_change_range(vmmspace_t *space, void *address, size_t size, bool free, int flags, mmuflags_t new_mmuflags) { + void *top = (void *)((uintptr_t)address + size); + vmmrange_t *new_range = NULL; + + if (space->ranges == NULL) + return 0; + + // allocated here and as soon as its used to make sure that + // even in an allocation failure there will always be a valid mapping + if (free == false) { + new_range = mm_alloc_range(); + if (new_range == NULL) + return ENOMEM; + } + + int error = 0; + + // get first range after address + rbtree_t *rbtree = rbtree_find_first_larger_equal(space->ranges, address, rbtree_value_compare); + vmmrange_t *range = container_of(rbtree, vmmrange_t, rbtree_node); + + if (rbtree == NULL) + goto leave; + + // check if we actually start at the predecessor + rbtree_t *predecessor = rbtree_predecessor(rbtree); + if (predecessor) { + vmmrange_t *predecessor_range = container_of(predecessor, vmmrange_t, rbtree_node); + if (RANGE_TOP(predecessor_range) > address) { + rbtree = predecessor; + range = predecessor_range; + } + } + + // rbtree now has the first vmm range to be unmapped/remapped + + // is the range we want to change in the middle of a mapping? + if (address > range->start && top < RANGE_TOP(range)) { + // split the mapping + + // check for write permission if changing a shared file mapping + if (free == false && (flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && + (range->flags & VMM_FLAGS_FILE) && can_write_vnode(range) == false) { + error = EACCES; + goto leave; + } + + vmmrange_t *new = mm_alloc_range(); + if (range == NULL) { + error = ENOMEM; + goto leave; + } + + *new = *range; // copy most metadata + + if (free) { + // release page data + mm_destroy_range(range, (uintptr_t)address - (uintptr_t)range->start, size, 0); + } + + // set up ranges + new->start = top; + new->size = (uintptr_t)RANGE_TOP(range) - (uintptr_t)new->start; + range->size = (uintptr_t)address - (uintptr_t)range->start; + + if (range->flags & VMM_FLAGS_FILE) { + VOP_HOLD(range->vnode); + new->offset += range->size + size; + } + + mm_insert_range(space, new); + + // if we are not just freeing memory, insert a new range + if (free == false) { + new_range->start = address; + new_range->size = size; + new_range->flags = range->flags; + new_range->mmuflags = new_mmuflags; + + change_mmu_range(range, new_range->start, new_range->size, new_range->mmuflags); + + if (range->flags & VMM_FLAGS_FILE) { + new_range->vnode = range->vnode; + new_range->offset = range->offset + range->size; + VOP_HOLD(range->vnode); + } + + mm_insert_range(space, new_range); + return 0; // dont free the new range + } + + goto leave; + } + + // are we starting from the end of a mapping? + if (address > range->start) { + // shrink the mapping + + // permission checking + if (free == false && (flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && + (range->flags & VMM_FLAGS_FILE) && can_write_vnode(range) == false) { + error = EACCES; + goto leave; + } + + size_t difference = (uintptr_t)RANGE_TOP(range) - (uintptr_t)address; + range->size -= difference; + + if (free) { + mm_destroy_range(range, range->size, difference, 0); + rbtree = rbtree_successor(rbtree); + } else { + // create a new range for the changes + new_range->start = (void *)((uintptr_t)range->start + range->size); + new_range->size = difference; + new_range->flags = range->flags; + new_range->mmuflags = new_mmuflags; + + change_mmu_range(range, new_range->start, new_range->size, new_range->mmuflags); + + if (range->flags & VMM_FLAGS_FILE) { + new_range->vnode = range->vnode; + new_range->offset = range->offset + range->size; + VOP_HOLD(range->vnode); + } + + mm_insert_range(space, new_range); + + rbtree = rbtree_successor(&new_range->rbtree_node); + + new_range = mm_alloc_range(); + if (new_range == NULL) { + error = ENOMEM; + goto leave; + } + } + } + + // mappings that lie entirely in the range + for (;;) { + if (rbtree == NULL) + goto leave; + + range = container_of(rbtree, vmmrange_t, rbtree_node); + if (RANGE_TOP(range) > top) + break; + + rbtree = rbtree_successor(rbtree); + + // remove or change the flags of the range + if (free) { + rbtree_remove(&space->ranges, &range->rbtree_node); + mm_destroy_range(range, 0, range->size, 0); + mm_free_range(range); + } else { + // permission check + if ((flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && + (range->flags & VMM_FLAGS_FILE) && can_write_vnode(range) == false) { + error = EACCES; + goto leave; + } + + change_mmu_range(range, range->start, range->size, new_mmuflags); + range->mmuflags = new_mmuflags; + } + } + + // start of mapping is in the range + if (range->start < top) { + // move the start of the range + + // permission check + if (free == false && (flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && + (range->flags & VMM_FLAGS_FILE) && can_write_vnode(range) == false) { + error = EACCES; + goto leave; + } + + size_t difference = (uintptr_t)top - (uintptr_t)range->start; + if (free) + mm_destroy_range(range, 0, difference, 0); + + range->start = (void *)((uintptr_t)range->start + difference); + range->size -= difference; + + if (range->flags & VMM_FLAGS_FILE) + range->offset += difference; + + if (free == false) { + // insert range at the start + new_range->start = (void *)((uintptr_t)range->start - difference); + new_range->size = difference; + new_range->flags = range->flags; + new_range->mmuflags = new_mmuflags; + + change_mmu_range(range, new_range->start, new_range->size, new_range->mmuflags); + + if (range->flags & VMM_FLAGS_FILE) { + new_range->vnode = range->vnode; + new_range->offset = range->offset - difference; + VOP_HOLD(range->vnode); + } + + mm_insert_range(space, new_range); + return 0; // don't free the range, just leave + } + } + + leave: + if (free == false) + mm_free_range(new_range); + + return error; +} diff --git a/kernel-src/mm/vmm.c b/kernel-src/mm/vmm.c index c8a46f7c..62a409d7 100644 --- a/kernel-src/mm/vmm.c +++ b/kernel-src/mm/vmm.c @@ -9,10 +9,6 @@ #include #include -#define RANGE_TOP(x) (void *)((uintptr_t)x->start + x->size) - -static scache_t *range_cache; - // ranges are separated into kernel and user. the kernel has a temporary user context vmmcontext_t vmm_kernelctx; static vmmspace_t kernelspace = { @@ -30,75 +26,7 @@ static vmmspace_t *getspace(void *vaddr) { return NULL; } -static int rbtree_value_compare(void *addr, rbtree_t *node) { - vmmrange_t *range = container_of(node, vmmrange_t, rbtree_node); - void *range_top = RANGE_TOP(range); - - if (addr >= range_top) - return 1; - - if (addr < range->start) - return -1; - - return 0; -} - -// get a range from an address -static vmmrange_t *getrange(vmmspace_t *space, void *addr) { - rbtree_t *rbtree = rbtree_lookup(space->ranges, addr, rbtree_value_compare); - return container_of(rbtree, vmmrange_t, rbtree_node); -} - -// get start of range that fits specific size from specific offset -static void *getfreerange(vmmspace_t *space, void *addr, size_t size) { - rbtree_t *rbtree = space->ranges; - if (addr == NULL) - addr = space->start; - - // no ranges - if (rbtree == NULL) - return addr; - - rbtree = rbtree_first(rbtree); - vmmrange_t *range = container_of(rbtree, vmmrange_t, rbtree_node); - - // if theres free space before the first range - if (range->start != space->start && addr < range->start && (uintptr_t)range->start - (uintptr_t)addr >= size) - return addr; - - // check for space between two ranges - rbtree_t *next = rbtree_successor(rbtree); - while (next) { - vmmrange_t *next_range = container_of(next, vmmrange_t, rbtree_node); - - void *rangetop = RANGE_TOP(range); - if (addr < rangetop) - addr = rangetop; - - if (addr < next_range->start) { - size_t freesize = (uintptr_t)next_range->start - (uintptr_t)addr; - if (freesize >= size) - return addr; - } - - rbtree = next; - range = container_of(rbtree, vmmrange_t, rbtree_node); - - next = rbtree_successor(next); - } - - // if theres free space after the last range - void *rangetop = RANGE_TOP(range); - if (addr < rangetop) - addr = rangetop; - - if (addr != space->end && (uintptr_t)space->end - (uintptr_t)addr >= size) - return addr; - - return NULL; -} - -static int rbtree_compare(rbtree_t *a, rbtree_t *b) { +static int vmm_range_compare(rbtree_t *a, rbtree_t *b) { vmmrange_t *range_a = container_of(a, vmmrange_t, rbtree_node); vmmrange_t *range_b = container_of(b, vmmrange_t, rbtree_node); @@ -108,353 +36,6 @@ static int rbtree_compare(rbtree_t *a, rbtree_t *b) { return range_a->start > range_b->start ? 1 : -1; } -static bool compatible(vmmrange_t *prev, vmmrange_t *next) { - void *prev_range_top = RANGE_TOP(prev); - - return prev_range_top == next->start && prev->flags == next->flags && prev->mmuflags == next->mmuflags && // general compatibility - ((prev->flags & VMM_FLAGS_FILE) == 0 || (prev->vnode == next->vnode && prev->offset + prev->size == next->offset)); // same file mapping, if any -} - -static void printspace(vmmspace_t *space); -static void insertrange(vmmspace_t *space, vmmrange_t *new_range) { - rbtree_insert(&space->ranges, &new_range->rbtree_node, rbtree_compare); - - rbtree_t *successor = rbtree_successor(&new_range->rbtree_node); - rbtree_t *predecessor = rbtree_predecessor(&new_range->rbtree_node); - - vmmrange_t *next_range = successor ? container_of(successor, vmmrange_t, rbtree_node) : NULL; - vmmrange_t *prev_range = predecessor ? container_of(predecessor, vmmrange_t, rbtree_node) : NULL; - - // fragmentation checking - - // check next range - if (next_range && compatible(new_range, next_range)) { - new_range->size += next_range->size; - - rbtree_remove(&space->ranges, &next_range->rbtree_node); - slab_free(range_cache, next_range); - if (new_range->flags & VMM_FLAGS_FILE) { - VOP_RELEASE(new_range->vnode); - } - } - - // check prev range - if (prev_range && compatible(prev_range, new_range)) { - prev_range->size += new_range->size; - - rbtree_remove(&space->ranges, &new_range->rbtree_node); - slab_free(range_cache, new_range); - if (prev_range->flags & VMM_FLAGS_FILE) { - VOP_RELEASE(prev_range->vnode); - } - } -} - -static void destroyrange(vmmrange_t *range, uintmax_t _offset, size_t size, int flags) { - // TODO find first - uintmax_t top = _offset + size; - - for (uintmax_t offset = _offset; offset < top; offset += PAGE_SIZE) { - void *vaddr = (void *)((uintptr_t)range->start + offset); - void *physical = arch_mmu_getphysical(current_vmm_context()->pagetable, vaddr); - if (physical == NULL) - continue; - - thread_t *thread = current_thread(); - proc_t *proc = thread ? thread->proc : NULL; - cred_t *cred = proc ? &proc->cred : NULL; - - if ((range->flags & VMM_FLAGS_FILE) && ((range->flags & VMM_FLAGS_SHARED) || vfs_iscacheable(range->vnode) == false)) { - // shared file mapping or non cacheable mapping - if (vfs_iscacheable(range->vnode) == false) { - // non cacheable mapping - VOP_LOCK(range->vnode); - __assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0); - VOP_UNLOCK(range->vnode); - } else if (arch_mmu_iswritable(current_vmm_context()->pagetable, vaddr)) { - // dirty page cache mapping - arch_mmu_unmap(current_vmm_context()->pagetable, vaddr); - VOP_LOCK(range->vnode); - mm_cache_make_dirty(mm_get_page(physical)); - VOP_UNLOCK(range->vnode); - mm_release_page(physical); - } else { - // non dirty page mapping - arch_mmu_unmap(current_vmm_context()->pagetable, vaddr); - mm_release_page(physical); - } - } else { - // anonymous, physical or private non character device mapping - arch_mmu_unmap(current_vmm_context()->pagetable, vaddr); - if ((range->flags & VMM_FLAGS_PHYSICAL) == 0) - mm_release_page(physical); - } - } - - if ((range->flags & VMM_FLAGS_FILE) && range->size == size) - VOP_RELEASE(range->vnode); -} - -#define CHANGE_MASK_CHECK(m, f, c, n) \ - if (((n) & (f)) == 0 && ((c) & (f))) \ - m |= f; - -static void changemmurange(vmmrange_t *range, void *base, size_t size, mmuflags_t newflags) { - for (uintmax_t offset = 0; offset < size; offset += PAGE_SIZE) { - void *address = (void *)((uintptr_t)base + offset); - - mmuflags_t currentflags; - // if page is not mapped, do nothing - if (arch_mmu_getflags(current_vmm_context()->pagetable, address, ¤tflags) == false) - continue; - - void *physical = arch_mmu_getphysical(current_vmm_context()->pagetable, address); - - uintmax_t mask = 0; - // check which flags are currently set and will be unset - CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_READ, currentflags, newflags); - CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_WRITE, currentflags, newflags); - CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_USER, currentflags, newflags); - CHANGE_MASK_CHECK(mask, ARCH_MMU_FLAGS_NOEXEC, currentflags, newflags); - - if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED) && (mask & ARCH_MMU_FLAGS_WRITE) && vfs_iscacheable(range->vnode)) { - // removing write permissions from a writeable dirty shared mapped page, mark it as dirty, as - // it won't be marked dirty upon a vmm_unmap after this - page_t *page = mm_get_page(physical); - VOP_LOCK(range->vnode); - mm_cache_make_dirty(page); - VOP_UNLOCK(range->vnode); - } - - // we will only change the mapping if the permissions decreased - if (mask) { - arch_mmu_remap(current_vmm_context()->pagetable, physical, address, currentflags & ~mask); - } - } -} - -static inline bool canwritevnode(vmmrange_t *range) { - VOP_LOCK(range->vnode); - int error = VOP_ACCESS(range->vnode, V_ACCESS_WRITE, ¤t_thread()->proc->cred); - VOP_UNLOCK(range->vnode); - return error == 0; -} - -static int changemap(vmmspace_t *space, void *address, size_t size, bool free, int flags, mmuflags_t new_mmuflags) { - void *top = (void *)((uintptr_t)address + size); - vmmrange_t *new_range = NULL; - - if (space->ranges == NULL) - return 0; - - // allocated here and as soon as its used to make sure that - // even in an allocation failure there will always be a valid mapping - if (free == false) { - new_range = slab_allocate(range_cache); - if (new_range == NULL) - return ENOMEM; - } - - int error = 0; - - // get first range after address - rbtree_t *rbtree = rbtree_find_first_larger_equal(space->ranges, address, rbtree_value_compare); - vmmrange_t *range = container_of(rbtree, vmmrange_t, rbtree_node); - - if (rbtree == NULL) - goto leave; - - // check if we actually start at the predecessor - rbtree_t *predecessor = rbtree_predecessor(rbtree); - if (predecessor) { - vmmrange_t *predecessor_range = container_of(predecessor, vmmrange_t, rbtree_node); - if (RANGE_TOP(predecessor_range) > address) { - rbtree = predecessor; - range = predecessor_range; - } - } - - // rbtree now has the first vmm range to be unmapped/remapped - - // is the range we want to change in the middle of a mapping? - if (address > range->start && top < RANGE_TOP(range)) { - // split the mapping - - // check for write permission if changing a shared file mapping - if (free == false && (flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && - (range->flags & VMM_FLAGS_FILE) && canwritevnode(range) == false) { - error = EACCES; - goto leave; - } - - vmmrange_t *new = slab_allocate(range_cache); - if (range == NULL) { - error = ENOMEM; - goto leave; - } - - *new = *range; // copy most metadata - - if (free) { - // release page data - destroyrange(range, (uintptr_t)address - (uintptr_t)range->start, size, 0); - } - - // set up ranges - new->start = top; - new->size = (uintptr_t)RANGE_TOP(range) - (uintptr_t)new->start; - range->size = (uintptr_t)address - (uintptr_t)range->start; - - if (range->flags & VMM_FLAGS_FILE) { - VOP_HOLD(range->vnode); - new->offset += range->size + size; - } - - insertrange(space, new); - - // if we are not just freeing memory, insert a new range - if (free == false) { - new_range->start = address; - new_range->size = size; - new_range->flags = range->flags; - new_range->mmuflags = new_mmuflags; - - changemmurange(range, new_range->start, new_range->size, new_range->mmuflags); - - if (range->flags & VMM_FLAGS_FILE) { - new_range->vnode = range->vnode; - new_range->offset = range->offset + range->size; - VOP_HOLD(range->vnode); - } - - insertrange(space, new_range); - return 0; // dont free the new range - } - - goto leave; - } - - // are we starting from the end of a mapping? - if (address > range->start) { - // shrink the mapping - - // permission checking - if (free == false && (flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && - (range->flags & VMM_FLAGS_FILE) && canwritevnode(range) == false) { - error = EACCES; - goto leave; - } - - size_t difference = (uintptr_t)RANGE_TOP(range) - (uintptr_t)address; - range->size -= difference; - - if (free) { - destroyrange(range, range->size, difference, 0); - rbtree = rbtree_successor(rbtree); - } else { - // create a new range for the changes - new_range->start = (void *)((uintptr_t)range->start + range->size); - new_range->size = difference; - new_range->flags = range->flags; - new_range->mmuflags = new_mmuflags; - - changemmurange(range, new_range->start, new_range->size, new_range->mmuflags); - - if (range->flags & VMM_FLAGS_FILE) { - new_range->vnode = range->vnode; - new_range->offset = range->offset + range->size; - VOP_HOLD(range->vnode); - } - - insertrange(space, new_range); - - rbtree = rbtree_successor(&new_range->rbtree_node); - - new_range = slab_allocate(range_cache); - if (new_range == NULL) { - error = ENOMEM; - goto leave; - } - } - } - - // mappings that lie entirely in the range - for (;;) { - if (rbtree == NULL) - goto leave; - - range = container_of(rbtree, vmmrange_t, rbtree_node); - if (RANGE_TOP(range) > top) - break; - - rbtree = rbtree_successor(rbtree); - - // remove or change the flags of the range - if (free) { - rbtree_remove(&space->ranges, &range->rbtree_node); - destroyrange(range, 0, range->size, 0); - slab_free(range_cache, range); - } else { - // permission check - if ((flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && - (range->flags & VMM_FLAGS_FILE) && canwritevnode(range) == false) { - error = EACCES; - goto leave; - } - - changemmurange(range, range->start, range->size, new_mmuflags); - range->mmuflags = new_mmuflags; - } - } - - // start of mapping is in the range - if (range->start < top) { - // move the start of the range - - // permission check - if (free == false && (flags & VMM_FLAGS_CREDCHECK) && (range->flags & VMM_FLAGS_SHARED) && - (range->flags & VMM_FLAGS_FILE) && canwritevnode(range) == false) { - error = EACCES; - goto leave; - } - - size_t difference = (uintptr_t)top - (uintptr_t)range->start; - if (free) - destroyrange(range, 0, difference, 0); - - range->start = (void *)((uintptr_t)range->start + difference); - range->size -= difference; - - if (range->flags & VMM_FLAGS_FILE) - range->offset += difference; - - if (free == false) { - // insert range at the start - new_range->start = (void *)((uintptr_t)range->start - difference); - new_range->size = difference; - new_range->flags = range->flags; - new_range->mmuflags = new_mmuflags; - - changemmurange(range, new_range->start, new_range->size, new_range->mmuflags); - - if (range->flags & VMM_FLAGS_FILE) { - new_range->vnode = range->vnode; - new_range->offset = range->offset - difference; - VOP_HOLD(range->vnode); - } - - insertrange(space, new_range); - return 0; // don't free the range, just leave - } - } - - leave: - if (free == false) - slab_free(range_cache, new_range); - - return error; -} - int vmm_changemmuflags(void *base, size_t size, mmuflags_t mmuflags, int flags) { base = (void *)ROUND_DOWN((uintptr_t)base, PAGE_SIZE); @@ -475,7 +56,7 @@ int vmm_changemmuflags(void *base, size_t size, mmuflags_t mmuflags, int flags) MUTEX_ACQUIRE(&space->lock); - int error = changemap(space, base, size, false, flags, mmuflags); + int error = mm_change_range(space, base, size, false, flags, mmuflags); arch_mmu_invalidate_range(base, size); MUTEX_RELEASE(&space->lock); @@ -510,7 +91,7 @@ bool vmm_pagefault(void *addr, bool user, int actions) { } MUTEX_ACQUIRE(&space->lock); - vmmrange_t *range = getrange(space, addr); + vmmrange_t *range = mm_get_range(space, addr); bool status = false; @@ -525,10 +106,10 @@ bool vmm_pagefault(void *addr, bool user, int actions) { if ((range->mmuflags & ARCH_MMU_FLAGS_READ) == 0) invalidactions |= VMM_ACTION_READ; - + if ((range->mmuflags & ARCH_MMU_FLAGS_WRITE) == 0) invalidactions |= VMM_ACTION_WRITE; - + if ((range->mmuflags & ARCH_MMU_FLAGS_NOEXEC)) invalidactions |= VMM_ACTION_EXEC; @@ -651,7 +232,7 @@ bool vmm_pagefault(void *addr, bool user, int actions) { /* static int lock_page(vmmspace_t *space, void *vaddr) { - vmmrange_t *range = getrange(space, vaddr); + vmmrange_t *range = mm_get_range(space, vaddr); if (!range) return ENOENT; @@ -736,12 +317,12 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, MUTEX_ACQUIRE(&space->lock); vmmrange_t *range = NULL; - void *start = getfreerange(space, addr, size); + void *start = mm_get_free_range(space, addr, size); void *retaddr = NULL; if (((flags & VMM_FLAGS_EXACT) && start != addr) || start == NULL) goto cleanup; - range = slab_allocate(range_cache); + range = mm_alloc_range(); if (range == NULL) goto cleanup; @@ -754,10 +335,10 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, range->mmuflags = mmuflags; __assert((flags & (VMM_FLAGS_ALLOCATE | VMM_FLAGS_PHYSICAL)) == 0); // make the memory inacessible - changemap(space, addr, size, false, flags, 0); + mm_change_range(space, addr, size, false, flags, 0); arch_mmu_invalidate_range(addr, size); // and then free it - changemap(space, addr, size, true, flags, 0); + mm_change_range(space, addr, size, true, flags, 0); } else { retaddr = start; range->start = start; @@ -817,10 +398,10 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, } } - insertrange(space, range); + mm_insert_range(space, range); cleanup: if (retaddr == NULL && range) - slab_free(range_cache, range); + mm_free_range(range); MUTEX_RELEASE(&space->lock); return retaddr; @@ -844,11 +425,11 @@ void vmm_unmap(void *addr, size_t size, int flags) { MUTEX_ACQUIRE(&space->lock); // make memory inacessible - changemap(space, addr, size, false, flags, 0); + mm_change_range(space, addr, size, false, flags, 0); arch_mmu_invalidate_range(addr, size); // and then free it - changemap(space, addr, size, true, flags, 0); + mm_change_range(space, addr, size, true, flags, 0); MUTEX_RELEASE(&space->lock); } @@ -904,13 +485,13 @@ vmmcontext_t *vmm_fork(vmmcontext_t *oldcontext) { rbtree_t *rbtree = oldcontext->space.ranges ? rbtree_first(oldcontext->space.ranges) : NULL; while (rbtree) { vmmrange_t *range = container_of(rbtree, vmmrange_t, rbtree_node); - vmmrange_t *newrange = slab_allocate(range_cache); + vmmrange_t *newrange = mm_alloc_range(); if (newrange == NULL) goto error; memcpy(newrange, range, sizeof(vmmrange_t)); - rbtree_insert(&newcontext->space.ranges, &newrange->rbtree_node, rbtree_compare); + rbtree_insert(&newcontext->space.ranges, &newrange->rbtree_node, vmm_range_compare); if (range->flags & VMM_FLAGS_FILE) VOP_HOLD(range->vnode); @@ -963,8 +544,7 @@ extern volatile struct limine_memmap_request mm_page_limine_map; void vmm_init() { // set up initial state - range_cache = slab_create_new_cache_from_pmm(sizeof(vmmrange_t), 0, NULL, NULL); - __assert(range_cache); + mm_range_init(); MUTEX_INIT(&kernelspace.lock); vmm_kernelctx.pagetable = arch_mmu_newtable();