From efe1feb8522c4afb21e522af7b2f188388cf4f48 Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Sun, 25 Oct 2026 17:49:25 +0100 Subject: [PATCH] mem: Refactor vma/vmo --- boot/main.c | 10 +- fs/exec/elf.c | 46 +-- include/zinnia/vma.h | 37 ++- include/zinnia/vmo.h | 78 ++--- mem/vma.c | 730 +++++++++++++++++++++++++++++++++++++++---- mem/vmo.c | 500 ++++++++++++++++++----------- 6 files changed, 1084 insertions(+), 317 deletions(-) diff --git a/boot/main.c b/boot/main.c index 15fd562..99db0b8 100644 --- a/boot/main.c +++ b/boot/main.c @@ -42,11 +42,11 @@ static void kernel_main_task(uintptr_t arg0) { struct boot_file* file_data = &info->files[0]; struct task* init_task = nullptr; - struct paged_vmo* init_file; - status = vmo_new_phys(&init_file); + struct vmo* init_file; + status = vmo_new_anon(&init_file); assert(status == 0, "Failed to allocate a memory object"); - vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr); + vmo_write(init_file, 0, HHDM_PTR(file_data->data), file_data->length, nullptr); // Load init. const char* init_argv[] = {"init", nullptr}; @@ -54,12 +54,12 @@ static void kernel_main_task(uintptr_t arg0) { struct exec_info exec = { .space = init_space, - .file_obj = &init_file->object, + .file_obj = init_file, .argv = init_argv, .envp = init_envp, }; - vmo_unref(&init_file->object); + vmo_unref(init_file); assert(status == 0, "Failed to load init process (%i)", status); sched_add_task(&percpu_get()->sched, init_task); diff --git a/fs/exec/elf.c b/fs/exec/elf.c index 17287f1..4917838 100644 --- a/fs/exec/elf.c +++ b/fs/exec/elf.c @@ -94,20 +94,20 @@ errno_t elf_load(struct exec_info* info, struct task** out) { const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1); // Copy the file data into its own mapping. - struct paged_vmo* phdr_obj; - err = vmo_new_phys(&phdr_obj); + struct vmo* phdr_obj; + err = vmo_new_anon(&phdr_obj); if (err) return err; - err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr); + err = vmo_copy(phdr_obj, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr); if (err) { - vmo_unref(&phdr_obj->object); + vmo_unref(phdr_obj); return err; } // We map more than we copied so the rest is filled with zeroed pages. - err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset); - vmo_unref(&phdr_obj->object); + err = vma_map(info->space, phdr_obj, phdr.p_vaddr, phdr.p_memsz, prot, 0, phdr.p_offset); + vmo_unref(phdr_obj); if (err) return err; } @@ -118,8 +118,8 @@ errno_t elf_load(struct exec_info* info, struct task** out) { const uintptr_t stack_start = highest - stack_size; // Allocate stack. - struct paged_vmo* stack; - err = vmo_new_phys(&stack); + struct vmo* stack; + err = vmo_new_anon(&stack); if (err) return err; @@ -147,18 +147,18 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) { mem_free(envp_offsets); mem_free(argv_offsets); - vmo_unref(&stack->object); + vmo_unref(stack); return ENOMEM; } for (size_t env = 0; env < num_envp; env++) { const char nul = 0; stack_off -= 1; - vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr); + vmo_write(stack, stack_off, &nul, sizeof(nul), nullptr); const size_t len = strlen(info->envp[env]); stack_off -= len; - vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr); + vmo_write(stack, stack_off, info->envp[env], len, nullptr); envp_offsets[env] = stack_start + stack_off; } @@ -166,11 +166,11 @@ errno_t elf_load(struct exec_info* info, struct task** out) { for (size_t arg = 0; arg < num_argv; arg++) { const char nul = 0; stack_off -= 1; - vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr); + vmo_write(stack, stack_off, &nul, sizeof(nul), nullptr); const size_t len = strlen(info->argv[arg]); stack_off -= len; - vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr); + vmo_write(stack, stack_off, info->argv[arg], len, nullptr); argv_offsets[arg] = stack_start + stack_off; } @@ -181,17 +181,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if ((1 + num_argv + num_envp) & 1) { stack_off -= sizeof(uintptr_t); uintptr_t zero = 0; - vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); + vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr); } #define WRITE_AUXV(auxv, value) \ do { \ stack_off -= sizeof(uintptr_t); \ uintptr_t auxv_val = value; \ - vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ + vmo_write(stack, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ stack_off -= sizeof(uintptr_t); \ auxv_val = auxv; \ - vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ + vmo_write(stack, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ } while (0) // Write auxiliary values. @@ -206,18 +206,18 @@ errno_t elf_load(struct exec_info* info, struct task** out) { // envp pointers. stack_off -= sizeof(uintptr_t); const uintptr_t zero = 0; - vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); + vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr); for (size_t env = num_envp; env > 0; env--) { stack_off -= sizeof(uintptr_t); - vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr); + vmo_write(stack, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr); } // argv pointers. stack_off -= sizeof(uintptr_t); - vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); + vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr); for (size_t arg = num_argv; arg > 0; arg--) { stack_off -= sizeof(uintptr_t); - vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr); + vmo_write(stack, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr); } mem_free(envp_offsets); @@ -226,10 +226,10 @@ errno_t elf_load(struct exec_info* info, struct task** out) { // argc stack_off -= sizeof(uintptr_t); uintptr_t argc = num_argv; - vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr); + vmo_write(stack, stack_off, &argc, sizeof(argc), nullptr); - err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0); - vmo_unref(&stack->object); + err = vma_map(info->space, stack, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0); + vmo_unref(stack); if (err) return err; diff --git a/include/zinnia/vma.h b/include/zinnia/vma.h index 79f8a7f..54a98ad 100644 --- a/include/zinnia/vma.h +++ b/include/zinnia/vma.h @@ -2,21 +2,34 @@ #pragma once +#include #include #include #include #include -#include +#include #include #include +enum vma_flags : uint8_t { + VMA_SHARED = 1 << 0, + VMA_NEEDS_COPY = 1 << 1, +}; + +struct vma; + struct vma_mapping { size_t start_page; size_t end_page; size_t offset_page; struct vmo* object; + struct amap* amap; + size_t amap_offset; + struct vma* vma; enum prot_flags prot; + enum vma_flags flags; tailq_link(struct vma_mapping) link; + tailq_link(struct vma_mapping) object_link; }; // Virtual address space. @@ -24,30 +37,44 @@ struct vma { // Architecture-specific virtual memory map implementation. struct pmap pmap; tailq_head(struct vma_mapping) mappings; - struct spinlock lock; + struct rwlock lock; + mutex_t map_lock; + struct refcount rc; }; // Try to create a new, empty virtual address space. errno_t vma_new(struct vma** out); -// Delete a virtual address space. -void vma_free(struct vma* v); +void vma_ref(struct vma* v); +void vma_unref(struct vma* v); +errno_t vma_fork(struct vma* src, struct vma** out); errno_t vma_map( struct vma* vma, struct vmo* vmo, uintptr_t addr, size_t len, - enum prot_flags flags, + enum prot_flags prot, + enum vma_flags flags, uintptr_t vmo_offset ); +errno_t vma_find_free(struct vma* v, uintptr_t hint, size_t len, uintptr_t* out); + +bool vma_range_mapped(struct vma* v, uintptr_t vaddr, size_t size); + // Remove any mappings in the given address range. errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size); // Change the protection of an existing mapping. errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags); +errno_t vma_write(struct vma* v, uintptr_t addr, const void* buf, size_t len); +errno_t vma_zero(struct vma* v, uintptr_t addr, size_t len); +errno_t vma_sync(struct vma* v, uintptr_t addr, size_t len); + +void vma_object_remove_page(struct vmo* object, uint64_t index, phys_t phys, bool protect); + struct vma_fault_info { uintptr_t ip; uintptr_t addr; diff --git a/include/zinnia/vmo.h b/include/zinnia/vmo.h index 33d0510..071b87e 100644 --- a/include/zinnia/vmo.h +++ b/include/zinnia/vmo.h @@ -3,57 +3,63 @@ #pragma once #include -#include +#include +#include +#include +#include #include -#include +#include #include #include #include -struct vmo { - struct refcount rc; - errno_t (*get_page)(struct vmo* vmo, uintptr_t offset_idx, struct page** out); +struct vmo; +struct vma_mapping; + +struct vmo_pager_ops { + errno_t (*get_pages)(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid); + errno_t (*put_pages)(struct vmo* vmo, uint64_t index, struct page** pages, size_t count); + errno_t (*lookup)(struct vmo* vmo, uint64_t index, phys_t* out); + size_t max_cluster; void (*destroy)(struct vmo* vmo); }; +struct vmo { + struct refcount rc; + const struct vmo_pager_ops* pager; + mutex_t lock; + struct radix_tree pages; + tailq_head(struct vma_mapping) mappings; + uint64_t ra_next; + size_t ra_window; + enum cache_mode cache; +}; + +enum vmo_access { + VMO_READ, + VMO_WRITE, +}; + +void vmo_init(struct vmo* vmo, const struct vmo_pager_ops* pager); void vmo_ref(struct vmo* obj); void vmo_unref(struct vmo* obj); -errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read); -errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written); +errno_t vmo_new_anon(struct vmo** out); +errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out); +void vmo_put_page(struct page* page); + +errno_t vmo_read(struct vmo* obj, uint64_t offset, void* buf, size_t len, size_t* actual_read); +errno_t vmo_write(struct vmo* obj, uint64_t offset, const void* buf, size_t len, size_t* actual_written); +errno_t vmo_read_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_read); +errno_t vmo_write_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_written); errno_t vmo_copy( struct vmo* target, - uintptr_t target_offset, + uint64_t target_offset, struct vmo* src, - uintptr_t src_offset, + uint64_t src_offset, size_t len, size_t* actual_copied ); -struct pager_ops { - errno_t (*get_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page** out); - errno_t (*put_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page* page); -}; - -// TODO: change to rbtree -struct page_list { - uintptr_t offset; - struct page* value; - list_link(struct page_list*) next; -}; - -struct paged_vmo { - struct vmo object; - struct pager_ops source; - list_head(struct page_list*) cache; - struct spinlock lock; -}; - -errno_t vmo_new_paged(struct pager_ops ops, struct paged_vmo** out); -errno_t vmo_new_phys(struct paged_vmo** out); - -// Drops all cached pages past `length` bytes. -void vmo_truncate_paged(struct paged_vmo* paged, size_t length); - -// Creates a VMO backed by a specific physical address range (e.g. for MMIO). -errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out); +errno_t vmo_sync(struct vmo* vmo, uint64_t first, uint64_t last); +void vmo_truncate(struct vmo* vmo, uint64_t length); diff --git a/mem/vma.c b/mem/vma.c index af1b0c1..276c247 100644 --- a/mem/vma.c +++ b/mem/vma.c @@ -1,11 +1,20 @@ +// SPDX-License-Identifier: ISC + #include +#include #include #include #include #include #include #include +#include #include +#include + +typedef tailq_head(struct vma_mapping) mapping_list_t; + +constexpr uintptr_t VMA_MMAP_BASE = 0x1'0000'0000; // Try to create a new, empty virtual address space. errno_t vma_new(struct vma** out) { @@ -24,48 +33,252 @@ errno_t vma_new(struct vma** out) { goto err1; tailq_init(&vmspace->mappings); + rwlock_init(&vmspace->lock); + mutex_init(&vmspace->map_lock); + refcount_init(&vmspace->rc); *out = vmspace; return 0; err1: mem_free(vmspace); -err0: return res; } -// Delete a virtual address space. -void vma_free(struct vma* v) { - struct vma_mapping* mapping; - while ((mapping = tailq_first(&v->mappings)) != nullptr) { - tailq_remove(&v->mappings, mapping, link); - vmo_unref(mapping->object); - mem_free(mapping); +static void object_lock(struct vma_mapping* m) { + if (m->object) + mutex_lock(&m->object->lock); +} + +static void object_unlock(struct vma_mapping* m) { + if (m->object) + mutex_unlock(&m->object->lock); +} + +static void object_link(struct vma_mapping* m) { + if (!m->object) + return; + mutex_lock(&m->object->lock); + tailq_insert_tail(&m->object->mappings, m, object_link); + mutex_unlock(&m->object->lock); +} + +static void mapping_free(struct vma_mapping* m) { + if (m->object) { + mutex_lock(&m->object->lock); + tailq_remove(&m->object->mappings, m, object_link); + mutex_unlock(&m->object->lock); + vmo_unref(m->object); } + if (m->amap) + amap_unref(m->amap); + mem_free(m); +} + +static void free_mappings(mapping_list_t* list) { + struct vma_mapping* mapping; + while ((mapping = tailq_first(list)) != nullptr) { + tailq_remove(list, mapping, link); + mapping_free(mapping); + } +} + +void vma_ref(struct vma* v) { + refcount_inc(&v->rc); +} + +void vma_unref(struct vma* v) { + assert(v != &kernel_vma, "Tried to release the kernel address space"); + if (!refcount_dec(&v->rc)) + return; + + free_mappings((mapping_list_t*)&v->mappings); + pmap_destroy(&v->pmap); + mem_free(v); +} + +static struct vma_mapping* mapping_alloc() { + return mem_alloc(sizeof(struct vma_mapping), 0); +} + +static void mapping_clone( + struct vma_mapping* dst, + const struct vma_mapping* src, + struct vma* vma, + size_t start, + size_t end +) { + const size_t delta = start - src->start_page; + dst->start_page = start; + dst->end_page = end; + dst->offset_page = src->offset_page + delta; + dst->object = src->object; + dst->amap = src->amap; + dst->amap_offset = src->amap_offset + delta; + dst->vma = vma; + dst->prot = src->prot; + dst->flags = src->flags; + if (dst->object) + vmo_ref(dst->object); + if (dst->amap) + amap_ref(dst->amap); +} + +static void unmap_pages(struct vma* v, size_t start, size_t end) { + const size_t page_size = arch_mem_page_size(); + for (size_t p = start; p < end; p++) + pmap_unmap(&v->pmap, p * page_size); +} + +static void carve(struct vma* v, size_t start, size_t end, struct vma_mapping** spare, mapping_list_t* dead) { + struct vma_mapping* m = tailq_first(&v->mappings); + while (m) { + struct vma_mapping* next = tailq_next(m, link); + if (start < m->end_page && m->start_page < end) { + unmap_pages(v, MAX(start, m->start_page), MIN(end, m->end_page)); + + if (m->start_page >= start && m->end_page <= end) { + tailq_remove(&v->mappings, m, link); + tailq_insert_tail(dead, m, link); + } else if (m->start_page < start && m->end_page > end) { + struct vma_mapping* tail = *spare; + assert(tail != nullptr, "Missing spare mapping"); + *spare = nullptr; + mapping_clone(tail, m, v, end, m->end_page); + object_lock(m); + m->end_page = start; + object_unlock(m); + object_link(tail); + tailq_insert_tail(&v->mappings, tail, link); + } else if (m->start_page < start) { + object_lock(m); + m->end_page = start; + object_unlock(m); + } else { + const size_t delta = end - m->start_page; + object_lock(m); + m->offset_page += delta; + m->amap_offset += delta; + m->start_page = end; + object_unlock(m); + } + } + m = next; + } +} + +static void split_at(struct vma* v, size_t page, struct vma_mapping** spare) { + struct vma_mapping* m; + for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (m->start_page < page && page < m->end_page) + break; + } + if (!m) + return; + + struct vma_mapping* tail = *spare; + assert(tail != nullptr, "Missing spare mapping"); + *spare = nullptr; + mapping_clone(tail, m, v, page, m->end_page); + object_lock(m); + m->end_page = page; + object_unlock(m); + object_link(tail); + tailq_insert_tail(&v->mappings, tail, link); +} + +static errno_t page_range(uintptr_t addr, size_t len, size_t* start, size_t* end) { + const size_t page_size = arch_mem_page_size(); + uintptr_t last; + if (addr % page_size != 0 || len == 0 || ckd_add(&last, addr, len - 1)) + return EINVAL; + *start = addr / page_size; + *end = last / page_size + 1; + return 0; +} + +static bool range_mapped(struct vma* v, size_t start, size_t end) { + size_t page = start; + while (page < end) { + struct vma_mapping* m; + for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (page >= m->start_page && page < m->end_page) + break; + } + if (!m) + return false; + page = m->end_page; + } + return true; +} + +bool vma_range_mapped(struct vma* v, uintptr_t vaddr, size_t size) { + size_t start, end; + if (page_range(vaddr, size, &start, &end)) + return false; + + rwlock_read_lock(&v->lock); + bool mapped = range_mapped(v, start, end); + rwlock_read_unlock(&v->lock); + return mapped; +} + +errno_t vma_find_free(struct vma* v, uintptr_t hint, size_t len, uintptr_t* out) { + const size_t page_size = arch_mem_page_size(); + if (len == 0) + return EINVAL; + + uintptr_t base = MAX(hint, VMA_MMAP_BASE); + if (ckd_add(&base, base, page_size - 1)) + return ENOMEM; + size_t candidate = base / page_size; + const size_t num_pages = len / page_size + (len % page_size != 0); + + rwlock_read_lock(&v->lock); + while (true) { + size_t candidate_end; + if (ckd_add(&candidate_end, candidate, num_pages) || candidate_end > SIZE_MAX / page_size || + !arch_mem_addr_is_user(candidate_end * page_size - 1)) { + rwlock_read_unlock(&v->lock); + return ENOMEM; + } + + struct vma_mapping* m; + for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (candidate < m->end_page && m->start_page < candidate_end) + break; + } + if (!m) + break; + candidate = m->end_page; + } + rwlock_read_unlock(&v->lock); + + *out = candidate * page_size; + return 0; } errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) { - panic("TODO"); -} + size_t start, end; + errno_t status = page_range(vaddr, size, &start, &end); + if (status) + return status; -errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) { - panic("TODO"); -} + struct vma_mapping* spare = mapping_alloc(); + if (!spare) + return ENOMEM; -static struct vma_mapping* find_mapping(struct vma* v, size_t page) { - for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { - if (page >= m->start_page && page < m->end_page) - return m; - } - return nullptr; -} + mapping_list_t dead; + tailq_init(&dead); -static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) { - if (info->caused_by_write) - return prot & PROT_WRITE; - if (info->caused_by_fetch) - return prot & PROT_EXEC; - return prot & (PROT_READ | PROT_WRITE); + rwlock_write_lock(&v->lock); + carve(v, start, end, &spare, &dead); + pmap_shootdown(&v->pmap); + rwlock_write_unlock(&v->lock); + + free_mappings(&dead); + mem_free(spare); + return 0; } static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) { @@ -81,43 +294,344 @@ static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) { return pf; } -bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) { +static void reprotect_pages(struct vma* v, const struct vma_mapping* m) { const size_t page_size = arch_mem_page_size(); - const size_t faulty_page = info->addr / page_size; + const bool none = !(m->prot & (PROT_READ | PROT_WRITE | PROT_EXEC)); + const enum pte_flags flags = prot_to_pte(v, m->prot & ~PROT_WRITE); + for (size_t p = m->start_page; p < m->end_page; p++) { + if (none) + pmap_unmap(&v->pmap, p * page_size); + else + pmap_protect(&v->pmap, p * page_size, flags); + } +} - spin_lock(&v->lock); - struct vma_mapping* mapping = find_mapping(v, faulty_page); - if (!mapping || !can_access(mapping->prot, info)) { - spin_unlock(&v->lock); - return false; +errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) { + size_t start, end; + errno_t status = page_range(vaddr, size, &start, &end); + if (status) + return status; + + struct vma_mapping* spares[2] = {mapping_alloc(), mapping_alloc()}; + if (!spares[0] || !spares[1]) { + mem_free(spares[0]); + mem_free(spares[1]); + return ENOMEM; } - struct vmo* object = mapping->object; - const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page; - const enum prot_flags prot = mapping->prot; - vmo_ref(object); - spin_unlock(&v->lock); + rwlock_write_lock(&v->lock); + if (!range_mapped(v, start, end)) { + rwlock_write_unlock(&v->lock); + mem_free(spares[0]); + mem_free(spares[1]); + return ENOMEM; + } + split_at(v, start, &spares[0]); + split_at(v, end, &spares[1]); + for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (m->start_page >= start && m->end_page <= end) { + object_lock(m); + m->prot = flags; + object_unlock(m); + reprotect_pages(v, m); + } + } + pmap_shootdown(&v->pmap); + rwlock_write_unlock(&v->lock); - struct page* page; - if (object->get_page(object, object_page, &page)) { - vmo_unref(object); - return false; + mem_free(spares[0]); + mem_free(spares[1]); + return 0; +} + +void vma_object_remove_page(struct vmo* object, uint64_t index, phys_t phys, bool protect) { + const size_t page_size = arch_mem_page_size(); + for (struct vma_mapping* m = tailq_first(&object->mappings); m; m = tailq_next(m, object_link)) { + if (index < m->offset_page || index - m->offset_page >= m->end_page - m->start_page) + continue; + + struct pmap* pmap = &m->vma->pmap; + const uintptr_t va = (m->start_page + (index - m->offset_page)) * page_size; + phys_t current; + if (pmap_extract(pmap, va, ¤t) || current != phys) + continue; + + if (protect) + pmap_protect(pmap, va, prot_to_pte(m->vma, m->prot & ~PROT_WRITE)); + else + pmap_unmap(pmap, va); + pmap_shootdown(pmap); + } +} + +static struct vma_mapping* find_mapping(struct vma* v, size_t page) { + for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (page >= m->start_page && page < m->end_page) + return m; + } + return nullptr; +} + +enum fault_kind { + FAULT_READ, + FAULT_WRITE, + FAULT_EXEC, +}; + +static bool can_access(enum prot_flags prot, enum fault_kind kind) { + switch (kind) { + case FAULT_WRITE: + return prot & PROT_WRITE; + case FAULT_EXEC: + return prot & PROT_EXEC; + default: + return prot & (PROT_READ | PROT_WRITE); + } +} + +static errno_t enter(struct vma* v, size_t page, phys_t phys, enum pte_flags flags, enum cache_mode cache) { + const uintptr_t va = page * arch_mem_page_size(); + phys_t old; + const bool replace = pmap_extract(&v->pmap, va, &old) == 0; + errno_t status = pmap_map(&v->pmap, va, phys, flags, cache); + if (status) + return status; + pte_flush(va); + if (replace) + pmap_shootdown(&v->pmap); + return 0; +} + +static errno_t prepare_amap(struct vma* v, size_t page) { + errno_t status = 0; + rwlock_write_lock(&v->lock); + struct vma_mapping* m = find_mapping(v, page); + if (m && !(m->flags & VMA_SHARED)) { + if (!m->amap) { + status = amap_new(&m->amap); + m->amap_offset = 0; + if (!status) + m->flags &= ~VMA_NEEDS_COPY; + } else if (m->flags & VMA_NEEDS_COPY) { + if (amap_is_shared(m->amap)) { + struct amap* copy; + status = amap_copy(m->amap, m->amap_offset, m->end_page - m->start_page, ©); + if (!status) { + amap_unref(m->amap); + m->amap = copy; + m->amap_offset = 0; + } + } + if (!status) + m->flags &= ~VMA_NEEDS_COPY; + } + } + rwlock_write_unlock(&v->lock); + return status; +} + +static errno_t fault_anon( + struct vma* v, + struct vma_mapping* m, + struct vm_anon* anon, + size_t page, + size_t slot, + bool write, + phys_t* out +) { + bool writable = !(m->flags & VMA_NEEDS_COPY) && atomic_load(&anon->rc, ATOMIC_ACQUIRE) == 1; + if (write && !writable) { + struct vm_anon* copy; + errno_t status = anon_new(anon->page, ©); + if (status) + return status; + *radix_slot(&m->amap->anons, slot) = copy; + anon_unref(anon); + anon = copy; + writable = true; } - spin_lock(&v->lock); - mapping = find_mapping(v, faulty_page); - const bool still_valid = mapping && mapping->object == object && - (faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info); + *out = vm_page_addr(anon->page); + return enter(v, page, *out, prot_to_pte(v, writable ? m->prot : m->prot & ~PROT_WRITE), CACHE_ANY); +} - if (still_valid) { - errno_t status = - pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY); - assert(status == 0, "Failed to map a demand-loaded page"); +static errno_t fault_promote( + struct vma* v, + struct vma_mapping* m, + size_t page, + size_t slot, + size_t index, + phys_t* out +) { + struct vmo* obj = m->object; + struct page* src = nullptr; + errno_t status = 0; + if (obj && obj->pager->lookup) { + phys_t phys; + status = obj->pager->lookup(obj, index, &phys); + if (!status) + src = vm_page_from_addr(phys); + } else if (obj) { + status = vmo_get_page(obj, index, VMO_READ, &src); } - spin_unlock(&v->lock); + if (status) + return status; - vmo_unref(object); - return true; + struct vm_anon* anon; + status = anon_new(src, &anon); + if (src && !obj->pager->lookup) + vmo_put_page(src); + if (status) + return status; + + status = radix_insert(&m->amap->anons, slot, anon); + if (status) { + anon_unref(anon); + return status; + } + + *out = vm_page_addr(anon->page); + return enter(v, page, *out, prot_to_pte(v, m->prot), CACHE_ANY); +} + +static errno_t fault_object(struct vma* v, struct vma_mapping* m, size_t page, size_t index, bool write, phys_t* out) { + struct vmo* obj = m->object; + const bool shared = m->flags & VMA_SHARED; + + if (obj->pager->lookup) { + errno_t status = obj->pager->lookup(obj, index, out); + if (status) + return status; + return enter(v, page, *out, prot_to_pte(v, shared ? m->prot : m->prot & ~PROT_WRITE), obj->cache); + } + + struct page* pg; + errno_t status = vmo_get_page(obj, index, shared && write ? VMO_WRITE : VMO_READ, &pg); + if (status) + return status; + + const bool writable = shared && (write || !obj->pager->put_pages); + mutex_lock(&obj->lock); + if (radix_lookup(&obj->pages, index) == pg) { + *out = vm_page_addr(pg); + status = enter(v, page, *out, prot_to_pte(v, writable ? m->prot : m->prot & ~PROT_WRITE), obj->cache); + } else { + status = EAGAIN; + } + mutex_unlock(&obj->lock); + vmo_put_page(pg); + return status; +} + +static errno_t fault(struct vma* v, size_t page, enum fault_kind kind, bool force, phys_t* out) { + const bool write = kind == FAULT_WRITE; + + while (true) { + rwlock_read_lock(&v->lock); + struct vma_mapping* m = find_mapping(v, page); + if (!m) { + rwlock_read_unlock(&v->lock); + return EFAULT; + } + if (!force && !can_access(m->prot, kind)) { + rwlock_read_unlock(&v->lock); + return EACCES; + } + + const bool shared = m->flags & VMA_SHARED; + if (!shared && (write || !m->object) && (!m->amap || (m->flags & VMA_NEEDS_COPY))) { + rwlock_read_unlock(&v->lock); + errno_t status = prepare_amap(v, page); + if (status) + return status; + continue; + } + + const size_t index = page - m->start_page + m->offset_page; + const size_t slot = page - m->start_page + m->amap_offset; + struct amap* amap = shared ? nullptr : m->amap; + errno_t status; + + if (amap) { + mutex_lock(&amap->lock); + struct vm_anon* anon = radix_lookup(&amap->anons, slot); + if (anon) + status = fault_anon(v, m, anon, page, slot, write, out); + else if (write || !m->object) + status = fault_promote(v, m, page, slot, index, out); + else + status = fault_object(v, m, page, index, false, out); + mutex_unlock(&amap->lock); + } else { + status = fault_object(v, m, page, index, write, out); + } + + rwlock_read_unlock(&v->lock); + if (status != EAGAIN) + return status; + } +} + +bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) { + const enum fault_kind kind = info->caused_by_write ? FAULT_WRITE : info->caused_by_fetch ? FAULT_EXEC : FAULT_READ; + phys_t phys; + return fault(v, info->addr / arch_mem_page_size(), kind, false, &phys) == 0; +} + +static errno_t write_range(struct vma* v, uintptr_t addr, const void* buf, size_t len) { + const size_t page_size = arch_mem_page_size(); + size_t progress = 0; + while (progress < len) { + const uintptr_t cur = addr + progress; + const size_t misalign = cur % page_size; + const size_t chunk = MIN(page_size - misalign, len - progress); + + phys_t phys; + errno_t status = fault(v, cur / page_size, FAULT_WRITE, true, &phys); + if (status) + return status; + + uint8_t* dst = (uint8_t*)HHDM_PTR(phys) + misalign; + if (buf) + memcpy(dst, (const uint8_t*)buf + progress, chunk); + else + memset(dst, 0, chunk); + progress += chunk; + } + return 0; +} + +errno_t vma_write(struct vma* v, uintptr_t addr, const void* buf, size_t len) { + return write_range(v, addr, buf, len); +} + +errno_t vma_zero(struct vma* v, uintptr_t addr, size_t len) { + return write_range(v, addr, nullptr, len); +} + +errno_t vma_sync(struct vma* v, uintptr_t addr, size_t len) { + size_t start, end; + errno_t status = page_range(addr, len, &start, &end); + if (status) + return status; + + errno_t result = 0; + rwlock_read_lock(&v->lock); + if (!range_mapped(v, start, end)) { + rwlock_read_unlock(&v->lock); + return ENOMEM; + } + for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (!m->object || !(m->flags & VMA_SHARED) || m->end_page <= start || m->start_page >= end) + continue; + const size_t first = MAX(start, m->start_page) - m->start_page + m->offset_page; + const size_t last = MIN(end, m->end_page) - m->start_page + m->offset_page; + status = vmo_sync(m->object, first, last); + if (status && !result) + result = status; + } + rwlock_read_unlock(&v->lock); + return result; } errno_t vma_map( @@ -125,34 +639,116 @@ errno_t vma_map( struct vmo* vmo, uintptr_t addr, size_t len, - enum prot_flags flags, + enum prot_flags prot, + enum vma_flags flags, uintptr_t vmo_offset ) { - if (!vma || !vmo) + if (!vma || (!vmo && (flags & VMA_SHARED))) return EINVAL; uintptr_t end; - if (ckd_add(&end, addr, len)) + if (len == 0 || ckd_add(&end, addr, len)) return ENOMEM; const size_t page_size = arch_mem_page_size(); if (addr % page_size != vmo_offset % page_size) return EINVAL; - struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0); - if (!mapping) + struct vma_mapping* mapping = mapping_alloc(); + struct vma_mapping* spare = mapping_alloc(); + if (!mapping || !spare) { + mem_free(mapping); + mem_free(spare); return ENOMEM; + } - vmo_ref(vmo); - mapping->start_page = addr / page_size; - mapping->end_page = ALIGN_UP(end, page_size) / page_size; - mapping->offset_page = vmo_offset / page_size; - mapping->object = vmo; - mapping->prot = flags; + *mapping = (struct vma_mapping){ + .start_page = addr / page_size, + .end_page = ALIGN_UP(end, page_size) / page_size, + .offset_page = vmo_offset / page_size, + .object = vmo, + .vma = vma, + .prot = prot, + .flags = flags & VMA_SHARED, + }; + if (vmo) + vmo_ref(vmo); - spin_lock(&vma->lock); - tailq_insert_head(&vma->mappings, mapping, link); - spin_unlock(&vma->lock); + mapping_list_t dead; + tailq_init(&dead); + rwlock_write_lock(&vma->lock); + carve(vma, mapping->start_page, mapping->end_page, &spare, &dead); + pmap_shootdown(&vma->pmap); + object_link(mapping); + tailq_insert_tail(&vma->mappings, mapping, link); + rwlock_write_unlock(&vma->lock); + + free_mappings(&dead); + mem_free(spare); + return 0; +} + +static bool protect_anons(struct vma* v, struct vma_mapping* m) { + const size_t page_size = arch_mem_page_size(); + const size_t num_slots = m->end_page - m->start_page; + const enum pte_flags flags = prot_to_pte(v, m->prot & ~PROT_WRITE); + struct vm_anon* batch[32]; + uint64_t slots[32]; + uint64_t next = m->amap_offset; + bool changed = false; + + mutex_lock(&m->amap->lock); + while (true) { + const size_t found = radix_gang_lookup(&m->amap->anons, next, ARRAY_SIZE(batch), (void**)batch, slots); + if (found == 0) + break; + for (size_t i = 0; i < found; i++) { + if (slots[i] - m->amap_offset >= num_slots) + goto done; + const uintptr_t va = (m->start_page + (slots[i] - m->amap_offset)) * page_size; + if (pmap_protect(&v->pmap, va, flags) == 0) + changed = true; + } + next = slots[found - 1] + 1; + } +done: + mutex_unlock(&m->amap->lock); + return changed; +} + +errno_t vma_fork(struct vma* src, struct vma** out) { + struct vma* dst; + errno_t status = vma_new(&dst); + if (status) + return status; + + bool changed = false; + rwlock_write_lock(&src->lock); + for (struct vma_mapping* m = tailq_first(&src->mappings); m; m = tailq_next(m, link)) { + struct vma_mapping* copy = mapping_alloc(); + if (!copy) { + if (changed) + pmap_shootdown(&src->pmap); + rwlock_write_unlock(&src->lock); + vma_unref(dst); + return ENOMEM; + } + + if (!(m->flags & VMA_SHARED) && m->amap) { + m->flags |= VMA_NEEDS_COPY; + if ((m->prot & PROT_WRITE) && protect_anons(src, m)) + changed = true; + } + + mapping_clone(copy, m, dst, m->start_page, m->end_page); + object_link(copy); + tailq_insert_tail(&dst->mappings, copy, link); + } + if (changed) + pmap_shootdown(&src->pmap); + rwlock_write_unlock(&src->lock); + + *out = dst; return 0; } diff --git a/mem/vmo.c b/mem/vmo.c index 899015a..15573aa 100644 --- a/mem/vmo.c +++ b/mem/vmo.c @@ -1,75 +1,250 @@ +// SPDX-License-Identifier: ISC + #include #include -#include +#include #include +#include #include +#include + +constexpr size_t RA_MIN_PAGES = 8; +constexpr size_t RA_MAX_PAGES = 64; + +void vmo_init(struct vmo* vmo, const struct vmo_pager_ops* pager) { + refcount_init(&vmo->rc); + vmo->pager = pager; + mutex_init(&vmo->lock); + radix_init(&vmo->pages); + tailq_init(&vmo->mappings); + vmo->ra_next = UINT64_MAX; + vmo->ra_window = RA_MIN_PAGES; + vmo->cache = CACHE_ANY; +} void vmo_ref(struct vmo* obj) { refcount_inc(&obj->rc); } +static void release_pages(struct vmo* vmo) { + struct page* batch[32]; + uint64_t indices[32]; + size_t found; + while ((found = radix_gang_lookup(&vmo->pages, 0, ARRAY_SIZE(batch), (void**)batch, indices)) != 0) { + for (size_t i = 0; i < found; i++) { + radix_remove(&vmo->pages, indices[i]); + page_unhold(batch[i]); + } + } + radix_destroy(&vmo->pages); +} + void vmo_unref(struct vmo* obj) { - if (refcount_dec(&obj->rc)) - obj->destroy(obj); + if (!refcount_dec(&obj->rc)) + return; + + if (obj->pager->put_pages) { + errno_t status = vmo_sync(obj, 0, UINT64_MAX); + if (status) + log_warn("vmo: Dropping dirty pages after failed writeback (%i)\n", status); + } + release_pages(obj); + obj->pager->destroy(obj); } -errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) { +static void mark_dirty(struct vmo* vmo, struct page* page, uint64_t index) { + if (!vmo->pager->put_pages) + return; + atomic_fetch_or(&page->flags, PAGE_DIRTY, ATOMIC_RELAXED); + radix_tag_set(&vmo->pages, index, RADIX_TAG_DIRTY); +} + +static void wait_busy(struct vmo* vmo, struct page* page) { + page_hold(page); + mutex_unlock(&vmo->lock); + page_wait(page); + page_unhold(page); + mutex_lock(&vmo->lock); +} + +static size_t cluster_size(struct vmo* vmo, uint64_t index) { + const size_t max = MIN(vmo->pager->max_cluster, RA_MAX_PAGES); + if (!vmo->pager->get_pages || max <= 1) + return 1; + + vmo->ra_window = index == vmo->ra_next ? MIN(vmo->ra_window * 2, max) : MIN(RA_MIN_PAGES, max); + size_t count = vmo->ra_window; + if (index > UINT64_MAX - count) + count = UINT64_MAX - index; + vmo->ra_next = index + count; + + for (size_t i = 1; i < count; i++) { + if (radix_lookup(&vmo->pages, index + i)) + return i; + } + return MAX(count, 1); +} + +static errno_t fill(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) { + struct page* pages[RA_MAX_PAGES]; + size_t count = cluster_size(vmo, index); + const enum alloc_flags flags = vmo->pager->get_pages ? ALLOC_NOZERO : 0; + + for (size_t i = 0; i < count; i++) { + struct page* page; + errno_t status = page_alloc(flags, &page); + if (!status) { + page->obj.owner = vmo; + page->obj.index = index + i; + atomic_store(&page->flags, PAGE_BUSY, ATOMIC_RELAXED); + status = radix_insert(&vmo->pages, index + i, page); + if (status) + page_unhold(page); + } + if (status) { + if (i == 0) { + mutex_unlock(&vmo->lock); + return status; + } + count = i; + break; + } + pages[i] = page; + } + + page_hold(pages[0]); + mutex_unlock(&vmo->lock); + + size_t valid = count; + errno_t status = 0; + if (vmo->pager->get_pages) + status = vmo->pager->get_pages(vmo, index, pages, count, &valid); + + mutex_lock(&vmo->lock); + for (size_t i = 0; i < count; i++) { + if (status || i >= valid) { + radix_remove(&vmo->pages, index + i); + page_unbusy(pages[i]); + page_unhold(pages[i]); + continue; + } + if (i == 0 && access == VMO_WRITE) + mark_dirty(vmo, pages[i], index); + page_unbusy(pages[i]); + } + mutex_unlock(&vmo->lock); + + if (status || valid == 0) { + page_unhold(pages[0]); + return status ? status : ENXIO; + } + + *out = pages[0]; + return 0; +} + +errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) { + if (vmo->pager->lookup) + return ENOTSUP; + + mutex_lock(&vmo->lock); + while (true) { + struct page* page = radix_lookup(&vmo->pages, index); + if (!page) + return fill(vmo, index, access, out); + + if (atomic_load(&page->flags, ATOMIC_ACQUIRE) & PAGE_BUSY) { + wait_busy(vmo, page); + continue; + } + + page_hold(page); + if (access == VMO_WRITE) + mark_dirty(vmo, page, index); + mutex_unlock(&vmo->lock); + *out = page; + return 0; + } +} + +void vmo_put_page(struct page* page) { + page_unhold(page); +} + +static errno_t copy_data( + struct vmo* obj, + uint64_t offset, + size_t len, + enum vmo_access access, + void* buf, + iovec_iter_t* iter, + size_t* out +) { const size_t page_size = arch_mem_page_size(); size_t progress = 0; errno_t status = 0; while (progress < len) { - const size_t misalign = (progress + offset) % page_size; - const size_t page_index = (progress + offset) / page_size; - const size_t copy_size = MIN(page_size - misalign, len - progress); + const size_t misalign = (offset + progress) % page_size; + const uint64_t index = (offset + progress) / page_size; + const size_t chunk = MIN(page_size - misalign, len - progress); - struct page* p; - status = obj->get_page(obj, page_index, &p); - if (status != 0) + struct page* page = nullptr; + phys_t phys; + if (obj->pager->lookup) { + status = obj->pager->lookup(obj, index, &phys); + } else { + status = vmo_get_page(obj, index, access, &page); + if (!status) + phys = vm_page_addr(page); + } + if (status) break; - uintptr_t page_addr = vm_page_addr(p); + uint8_t* ptr = (uint8_t*)HHDM_PTR(phys) + misalign; + size_t done = chunk; + if (iter && access == VMO_WRITE) + status = iovec_iter_copy_from(iter, ptr, chunk, &done); + else if (iter) + status = iovec_iter_copy_to(iter, ptr, chunk, &done); + else if (access == VMO_WRITE) + memcpy(ptr, (uint8_t*)buf + progress, chunk); + else + memcpy((uint8_t*)buf + progress, ptr, chunk); - memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size); - progress += copy_size; + if (page) + vmo_put_page(page); + progress += done; + if (status || done < chunk) + break; } - if (actual_read) - *actual_read = progress; + if (out) + *out = progress; return status; } -errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) { - const size_t page_size = arch_mem_page_size(); - size_t progress = 0; - errno_t status = 0; +errno_t vmo_read(struct vmo* obj, uint64_t offset, void* buf, size_t len, size_t* actual_read) { + return copy_data(obj, offset, len, VMO_READ, buf, nullptr, actual_read); +} - while (progress < len) { - const size_t misalign = (progress + offset) % page_size; - const size_t page_index = (progress + offset) / page_size; - const size_t copy_size = MIN(page_size - misalign, len - progress); +errno_t vmo_write(struct vmo* obj, uint64_t offset, const void* buf, size_t len, size_t* actual_written) { + return copy_data(obj, offset, len, VMO_WRITE, (void*)buf, nullptr, actual_written); +} - struct page* p; - status = obj->get_page(obj, page_index, &p); - if (status != 0) - break; +errno_t vmo_read_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_read) { + return copy_data(obj, offset, len, VMO_READ, nullptr, iter, actual_read); +} - uintptr_t page_addr = vm_page_addr(p); - - memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size); - progress += copy_size; - } - - if (actual_written) - *actual_written = progress; - return status; +errno_t vmo_write_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_written) { + return copy_data(obj, offset, len, VMO_WRITE, nullptr, iter, actual_written); } errno_t vmo_copy( struct vmo* target, - uintptr_t target_offset, + uint64_t target_offset, struct vmo* src, - uintptr_t src_offset, + uint64_t src_offset, size_t len, size_t* actual_copied ) { @@ -80,25 +255,27 @@ errno_t vmo_copy( while (progress < len) { const size_t target_misalign = (progress + target_offset) % page_size; const size_t src_misalign = (progress + src_offset) % page_size; - const size_t target_page_index = (progress + target_offset) / page_size; - const size_t src_page_index = (progress + src_offset) / page_size; - const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress); struct page* tpage; - status = target->get_page(target, target_page_index, &tpage); - if (status != 0) + status = vmo_get_page(target, (progress + target_offset) / page_size, VMO_WRITE, &tpage); + if (status) break; struct page* spage; - status = src->get_page(src, src_page_index, &spage); - if (status != 0) + status = vmo_get_page(src, (progress + src_offset) / page_size, VMO_READ, &spage); + if (status) { + vmo_put_page(tpage); break; + } - uintptr_t taddr = vm_page_addr(tpage); - uintptr_t saddr = vm_page_addr(spage); - - memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size); + memcpy( + (uint8_t*)HHDM_PTR(vm_page_addr(tpage)) + target_misalign, + (uint8_t*)HHDM_PTR(vm_page_addr(spage)) + src_misalign, + copy_size + ); + vmo_put_page(spage); + vmo_put_page(tpage); progress += copy_size; } @@ -107,157 +284,118 @@ errno_t vmo_copy( return status; } -static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) { - struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object); - - // Try to find the requested page in the cache. - struct page_list* iter; - spin_lock(&paged->lock); - list_foreach(iter, &paged->cache, next) { - if (iter->offset != offset_idx) - continue; - - // We found a cached page! - *out = iter->value; - spin_unlock(&paged->lock); +errno_t vmo_sync(struct vmo* vmo, uint64_t first, uint64_t end) { + if (!vmo->pager->put_pages) return 0; - } - spin_unlock(&paged->lock); - // Get page from out pager if we don't have one already. - struct page* new_page; - errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page); - if (status) - return status; + struct page* run[RA_MAX_PAGES]; + const size_t max = MAX(MIN(vmo->pager->max_cluster, RA_MAX_PAGES), 1); + errno_t result = 0; - struct page_list* entry = mem_alloc(sizeof(struct page_list), 0); - if (!entry) { - paged->source.put_page(&paged->source, offset_idx, new_page); - return ENOMEM; - } - - entry->offset = offset_idx; - entry->value = new_page; - - spin_lock(&paged->lock); - list_foreach(iter, &paged->cache, next) { - if (iter->offset != offset_idx) + mutex_lock(&vmo->lock); + while (first < end) { + struct page* found; + uint64_t index; + if (!radix_gang_lookup_tag(&vmo->pages, first, RADIX_TAG_DIRTY, 1, (void**)&found, &index) || index >= end) + break; + if (!page_try_busy(found)) { + wait_busy(vmo, found); continue; + } - *out = iter->value; - spin_unlock(&paged->lock); - mem_free(entry); - paged->source.put_page(&paged->source, offset_idx, new_page); - return 0; + size_t count = 1; + run[0] = found; + while (count < max && index + count < end && index + count != 0) { + struct page* next = radix_lookup(&vmo->pages, index + count); + if (!next || !radix_tag_get(&vmo->pages, index + count, RADIX_TAG_DIRTY) || !page_try_busy(next)) + break; + run[count++] = next; + } + + for (size_t i = 0; i < count; i++) { + radix_tag_clear(&vmo->pages, index + i, RADIX_TAG_DIRTY); + radix_tag_set(&vmo->pages, index + i, RADIX_TAG_WRITEBACK); + atomic_fetch_and(&run[i]->flags, ~PAGE_DIRTY, ATOMIC_RELAXED); + page_hold(run[i]); + vma_object_remove_page(vmo, index + i, vm_page_addr(run[i]), true); + } + mutex_unlock(&vmo->lock); + + errno_t status = vmo->pager->put_pages(vmo, index, run, count); + + mutex_lock(&vmo->lock); + for (size_t i = 0; i < count; i++) { + radix_tag_clear(&vmo->pages, index + i, RADIX_TAG_WRITEBACK); + if (status) + mark_dirty(vmo, run[i], index + i); + page_unbusy(run[i]); + page_unhold(run[i]); + } + if (status && !result) + result = status; + first = index + count; + if (first == 0) + break; } - - // Add page to the object cache. - list_insert_head(&paged->cache, entry, next); - spin_unlock(&paged->lock); - - *out = new_page; - return 0; + mutex_unlock(&vmo->lock); + return result; } -static void paged_destroy(struct vmo* vmo) { - struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object); - - struct page_list* iter = list_first(&paged->cache); - while (iter) { - struct page_list* next = list_next(iter, next); - paged->source.put_page(&paged->source, iter->offset, iter->value); - mem_free(iter); - iter = next; - } - - mem_free(paged); -} - -errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) { - struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0); - if (!result) - return ENOMEM; - - refcount_init(&result->object.rc); - result->source = pager; - result->object.get_page = paged_get_page; - result->object.destroy = paged_destroy; - - *out = result; - return 0; -} - -static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) { - phys_t addr; - errno_t status = mem_phys_alloc(1, 0, &addr); - if (status != 0) - return status; - size_t idx = addr / arch_mem_page_size(); - *out = &vm_pfndb[idx]; - return 0; -} - -static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) { - return mem_phys_free(vm_page_addr(page), 1); -} - -static const struct pager_ops phys_pager = { - .get_page = phys_get_page, - .put_page = phys_put_page, -}; - -errno_t vmo_new_phys(struct paged_vmo** out) { - return vmo_new_paged(phys_pager, out); -} - -void vmo_truncate_paged(struct paged_vmo* paged, size_t length) { +void vmo_truncate(struct vmo* vmo, uint64_t length) { const size_t page_size = arch_mem_page_size(); - const size_t keep_pages = ROUND_UP(length, page_size); + uint64_t first = length / page_size + (length % page_size != 0); - spin_lock(&paged->lock); - struct page_list** pp = &list_first(&paged->cache); - while (*pp) { - struct page_list* entry = *pp; - if (entry->offset >= keep_pages) { - *pp = list_next(entry, next); - paged->source.put_page(&paged->source, entry->offset, entry->value); - mem_free(entry); - } else { - pp = &list_next(entry, next); + mutex_lock(&vmo->lock); + while (true) { + struct page* page; + uint64_t index; + if (!radix_gang_lookup(&vmo->pages, first, 1, (void**)&page, &index)) + break; + if (!page_try_busy(page)) { + wait_busy(vmo, page); + continue; + } + vma_object_remove_page(vmo, index, vm_page_addr(page), false); + radix_remove(&vmo->pages, index); + page_unbusy(page); + page_unhold(page); + first = index + 1; + } + + if (length % page_size != 0) { + while (true) { + struct page* tail = radix_lookup(&vmo->pages, length / page_size); + if (!tail) + break; + if (atomic_load(&tail->flags, ATOMIC_ACQUIRE) & PAGE_BUSY) { + wait_busy(vmo, tail); + continue; + } + memset((uint8_t*)HHDM_PTR(vm_page_addr(tail)) + length % page_size, 0, page_size - length % page_size); + mark_dirty(vmo, tail, length / page_size); + break; } } - spin_unlock(&paged->lock); + mutex_unlock(&vmo->lock); } -struct phys_range_vmo { - struct vmo object; - uintptr_t start; - size_t len; +struct anon_vmo { + struct vmo vmo; }; -static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) { - struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object); - if (offset_idx >= p->len / arch_mem_page_size()) - return EINVAL; - *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx]; - return 0; +static void anon_destroy(struct vmo* vmo) { + mem_free(containerof(vmo, struct anon_vmo, vmo)); } -static void phys_range_destroy(struct vmo* vmo) { - mem_free(containerof(vmo, struct phys_range_vmo, object)); -} +static const struct vmo_pager_ops anon_pager = { + .destroy = anon_destroy, +}; -errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) { - struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0); - if (!vmo) +errno_t vmo_new_anon(struct vmo** out) { + struct anon_vmo* anon = mem_alloc(sizeof(struct anon_vmo), 0); + if (!anon) return ENOMEM; - - refcount_init(&vmo->object.rc); - vmo->object.get_page = phys_range_get_page; - vmo->object.destroy = phys_range_destroy; - vmo->start = ALIGN_DOWN(addr, arch_mem_page_size()); - vmo->len = ALIGN_UP(length, arch_mem_page_size()); - - *out = &vmo->object; + vmo_init(&anon->vmo, &anon_pager); + *out = &anon->vmo; return 0; }