marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mem: Refactor vma/vmo

Browse Source
This commit is contained in:
Marvin
2026-10-25 17:49:25 +01:00
parent ef08e2e5d7
commit efe1feb852
6 changed files with 1084 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mem/vma.c
+663 -67
View File
@@ -1,11 +1,20 @@
// SPDX-License-Identifier: ISC
#include <zinnia/alloc.h>
#include <zinnia/amap.h>
#include <zinnia/assert.h>
#include <zinnia/errno.h>
#include <zinnia/mman.h>
#include <zinnia/pmap.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/mem.h>
#include <stdckdint.h>
#include <string.h>
typedef tailq_head(struct vma_mapping) mapping_list_t;
constexpr uintptr_t VMA_MMAP_BASE = 0x1'0000'0000;
// Try to create a new, empty virtual address space.
errno_t vma_new(struct vma** out) {
@@ -24,48 +33,252 @@ errno_t vma_new(struct vma** out) {
goto err1;
tailq_init(&vmspace->mappings);
rwlock_init(&vmspace->lock);
mutex_init(&vmspace->map_lock);
refcount_init(&vmspace->rc);
*out = vmspace;
return 0;
err1:
mem_free(vmspace);
err0:
return res;
}
// Delete a virtual address space.
void vma_free(struct vma* v) {
struct vma_mapping* mapping;
while ((mapping = tailq_first(&v->mappings)) != nullptr) {
tailq_remove(&v->mappings, mapping, link);
vmo_unref(mapping->object);
mem_free(mapping);
static void object_lock(struct vma_mapping* m) {
if (m->object)
mutex_lock(&m->object->lock);
}
static void object_unlock(struct vma_mapping* m) {
if (m->object)
mutex_unlock(&m->object->lock);
}
static void object_link(struct vma_mapping* m) {
if (!m->object)
return;
mutex_lock(&m->object->lock);
tailq_insert_tail(&m->object->mappings, m, object_link);
mutex_unlock(&m->object->lock);
}
static void mapping_free(struct vma_mapping* m) {
if (m->object) {
mutex_lock(&m->object->lock);
tailq_remove(&m->object->mappings, m, object_link);
mutex_unlock(&m->object->lock);
vmo_unref(m->object);
}
if (m->amap)
amap_unref(m->amap);
mem_free(m);
}
static void free_mappings(mapping_list_t* list) {
struct vma_mapping* mapping;
while ((mapping = tailq_first(list)) != nullptr) {
tailq_remove(list, mapping, link);
mapping_free(mapping);
}
}
void vma_ref(struct vma* v) {
refcount_inc(&v->rc);
}
void vma_unref(struct vma* v) {
assert(v != &kernel_vma, "Tried to release the kernel address space");
if (!refcount_dec(&v->rc))
return;
free_mappings((mapping_list_t*)&v->mappings);
pmap_destroy(&v->pmap);
mem_free(v);
}
static struct vma_mapping* mapping_alloc() {
return mem_alloc(sizeof(struct vma_mapping), 0);
}
static void mapping_clone(
struct vma_mapping* dst,
const struct vma_mapping* src,
struct vma* vma,
size_t start,
size_t end
) {
const size_t delta = start - src->start_page;
dst->start_page = start;
dst->end_page = end;
dst->offset_page = src->offset_page + delta;
dst->object = src->object;
dst->amap = src->amap;
dst->amap_offset = src->amap_offset + delta;
dst->vma = vma;
dst->prot = src->prot;
dst->flags = src->flags;
if (dst->object)
vmo_ref(dst->object);
if (dst->amap)
amap_ref(dst->amap);
}
static void unmap_pages(struct vma* v, size_t start, size_t end) {
const size_t page_size = arch_mem_page_size();
for (size_t p = start; p < end; p++)
pmap_unmap(&v->pmap, p * page_size);
}
static void carve(struct vma* v, size_t start, size_t end, struct vma_mapping** spare, mapping_list_t* dead) {
struct vma_mapping* m = tailq_first(&v->mappings);
while (m) {
struct vma_mapping* next = tailq_next(m, link);
if (start < m->end_page && m->start_page < end) {
unmap_pages(v, MAX(start, m->start_page), MIN(end, m->end_page));
if (m->start_page >= start && m->end_page <= end) {
tailq_remove(&v->mappings, m, link);
tailq_insert_tail(dead, m, link);
} else if (m->start_page < start && m->end_page > end) {
struct vma_mapping* tail = *spare;
assert(tail != nullptr, "Missing spare mapping");
*spare = nullptr;
mapping_clone(tail, m, v, end, m->end_page);
object_lock(m);
m->end_page = start;
object_unlock(m);
object_link(tail);
tailq_insert_tail(&v->mappings, tail, link);
} else if (m->start_page < start) {
object_lock(m);
m->end_page = start;
object_unlock(m);
} else {
const size_t delta = end - m->start_page;
object_lock(m);
m->offset_page += delta;
m->amap_offset += delta;
m->start_page = end;
object_unlock(m);
}
}
m = next;
}
}
static void split_at(struct vma* v, size_t page, struct vma_mapping** spare) {
struct vma_mapping* m;
for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (m->start_page < page && page < m->end_page)
break;
}
if (!m)
return;
struct vma_mapping* tail = *spare;
assert(tail != nullptr, "Missing spare mapping");
*spare = nullptr;
mapping_clone(tail, m, v, page, m->end_page);
object_lock(m);
m->end_page = page;
object_unlock(m);
object_link(tail);
tailq_insert_tail(&v->mappings, tail, link);
}
static errno_t page_range(uintptr_t addr, size_t len, size_t* start, size_t* end) {
const size_t page_size = arch_mem_page_size();
uintptr_t last;
if (addr % page_size != 0 || len == 0 || ckd_add(&last, addr, len - 1))
return EINVAL;
*start = addr / page_size;
*end = last / page_size + 1;
return 0;
}
static bool range_mapped(struct vma* v, size_t start, size_t end) {
size_t page = start;
while (page < end) {
struct vma_mapping* m;
for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (page >= m->start_page && page < m->end_page)
break;
}
if (!m)
return false;
page = m->end_page;
}
return true;
}
bool vma_range_mapped(struct vma* v, uintptr_t vaddr, size_t size) {
size_t start, end;
if (page_range(vaddr, size, &start, &end))
return false;
rwlock_read_lock(&v->lock);
bool mapped = range_mapped(v, start, end);
rwlock_read_unlock(&v->lock);
return mapped;
}
errno_t vma_find_free(struct vma* v, uintptr_t hint, size_t len, uintptr_t* out) {
const size_t page_size = arch_mem_page_size();
if (len == 0)
return EINVAL;
uintptr_t base = MAX(hint, VMA_MMAP_BASE);
if (ckd_add(&base, base, page_size - 1))
return ENOMEM;
size_t candidate = base / page_size;
const size_t num_pages = len / page_size + (len % page_size != 0);
rwlock_read_lock(&v->lock);
while (true) {
size_t candidate_end;
if (ckd_add(&candidate_end, candidate, num_pages) || candidate_end > SIZE_MAX / page_size ||
!arch_mem_addr_is_user(candidate_end * page_size - 1)) {
rwlock_read_unlock(&v->lock);
return ENOMEM;
}
struct vma_mapping* m;
for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (candidate < m->end_page && m->start_page < candidate_end)
break;
}
if (!m)
break;
candidate = m->end_page;
}
rwlock_read_unlock(&v->lock);
*out = candidate * page_size;
return 0;
}
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
panic("TODO");
}
size_t start, end;
errno_t status = page_range(vaddr, size, &start, &end);
if (status)
return status;
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
panic("TODO");
}
struct vma_mapping* spare = mapping_alloc();
if (!spare)
return ENOMEM;
static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (page >= m->start_page && page < m->end_page)
return m;
}
return nullptr;
}
mapping_list_t dead;
tailq_init(&dead);
static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) {
if (info->caused_by_write)
return prot & PROT_WRITE;
if (info->caused_by_fetch)
return prot & PROT_EXEC;
return prot & (PROT_READ | PROT_WRITE);
rwlock_write_lock(&v->lock);
carve(v, start, end, &spare, &dead);
pmap_shootdown(&v->pmap);
rwlock_write_unlock(&v->lock);
free_mappings(&dead);
mem_free(spare);
return 0;
}
static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
@@ -81,43 +294,344 @@ static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
return pf;
}
bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
static void reprotect_pages(struct vma* v, const struct vma_mapping* m) {
const size_t page_size = arch_mem_page_size();
const size_t faulty_page = info->addr / page_size;
const bool none = !(m->prot & (PROT_READ | PROT_WRITE | PROT_EXEC));
const enum pte_flags flags = prot_to_pte(v, m->prot & ~PROT_WRITE);
for (size_t p = m->start_page; p < m->end_page; p++) {
if (none)
pmap_unmap(&v->pmap, p * page_size);
else
pmap_protect(&v->pmap, p * page_size, flags);
}
}
spin_lock(&v->lock);
struct vma_mapping* mapping = find_mapping(v, faulty_page);
if (!mapping || !can_access(mapping->prot, info)) {
spin_unlock(&v->lock);
return false;
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
size_t start, end;
errno_t status = page_range(vaddr, size, &start, &end);
if (status)
return status;
struct vma_mapping* spares[2] = {mapping_alloc(), mapping_alloc()};
if (!spares[0] || !spares[1]) {
mem_free(spares[0]);
mem_free(spares[1]);
return ENOMEM;
}
struct vmo* object = mapping->object;
const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
const enum prot_flags prot = mapping->prot;
vmo_ref(object);
spin_unlock(&v->lock);
rwlock_write_lock(&v->lock);
if (!range_mapped(v, start, end)) {
rwlock_write_unlock(&v->lock);
mem_free(spares[0]);
mem_free(spares[1]);
return ENOMEM;
}
split_at(v, start, &spares[0]);
split_at(v, end, &spares[1]);
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (m->start_page >= start && m->end_page <= end) {
object_lock(m);
m->prot = flags;
object_unlock(m);
reprotect_pages(v, m);
}
}
pmap_shootdown(&v->pmap);
rwlock_write_unlock(&v->lock);
struct page* page;
if (object->get_page(object, object_page, &page)) {
vmo_unref(object);
return false;
mem_free(spares[0]);
mem_free(spares[1]);
return 0;
}
void vma_object_remove_page(struct vmo* object, uint64_t index, phys_t phys, bool protect) {
const size_t page_size = arch_mem_page_size();
for (struct vma_mapping* m = tailq_first(&object->mappings); m; m = tailq_next(m, object_link)) {
if (index < m->offset_page || index - m->offset_page >= m->end_page - m->start_page)
continue;
struct pmap* pmap = &m->vma->pmap;
const uintptr_t va = (m->start_page + (index - m->offset_page)) * page_size;
phys_t current;
if (pmap_extract(pmap, va, &current) || current != phys)
continue;
if (protect)
pmap_protect(pmap, va, prot_to_pte(m->vma, m->prot & ~PROT_WRITE));
else
pmap_unmap(pmap, va);
pmap_shootdown(pmap);
}
}
static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (page >= m->start_page && page < m->end_page)
return m;
}
return nullptr;
}
enum fault_kind {
FAULT_READ,
FAULT_WRITE,
FAULT_EXEC,
};
static bool can_access(enum prot_flags prot, enum fault_kind kind) {
switch (kind) {
case FAULT_WRITE:
return prot & PROT_WRITE;
case FAULT_EXEC:
return prot & PROT_EXEC;
default:
return prot & (PROT_READ | PROT_WRITE);
}
}
static errno_t enter(struct vma* v, size_t page, phys_t phys, enum pte_flags flags, enum cache_mode cache) {
const uintptr_t va = page * arch_mem_page_size();
phys_t old;
const bool replace = pmap_extract(&v->pmap, va, &old) == 0;
errno_t status = pmap_map(&v->pmap, va, phys, flags, cache);
if (status)
return status;
pte_flush(va);
if (replace)
pmap_shootdown(&v->pmap);
return 0;
}
static errno_t prepare_amap(struct vma* v, size_t page) {
errno_t status = 0;
rwlock_write_lock(&v->lock);
struct vma_mapping* m = find_mapping(v, page);
if (m && !(m->flags & VMA_SHARED)) {
if (!m->amap) {
status = amap_new(&m->amap);
m->amap_offset = 0;
if (!status)
m->flags &= ~VMA_NEEDS_COPY;
} else if (m->flags & VMA_NEEDS_COPY) {
if (amap_is_shared(m->amap)) {
struct amap* copy;
status = amap_copy(m->amap, m->amap_offset, m->end_page - m->start_page, &copy);
if (!status) {
amap_unref(m->amap);
m->amap = copy;
m->amap_offset = 0;
}
}
if (!status)
m->flags &= ~VMA_NEEDS_COPY;
}
}
rwlock_write_unlock(&v->lock);
return status;
}
static errno_t fault_anon(
struct vma* v,
struct vma_mapping* m,
struct vm_anon* anon,
size_t page,
size_t slot,
bool write,
phys_t* out
) {
bool writable = !(m->flags & VMA_NEEDS_COPY) && atomic_load(&anon->rc, ATOMIC_ACQUIRE) == 1;
if (write && !writable) {
struct vm_anon* copy;
errno_t status = anon_new(anon->page, &copy);
if (status)
return status;
*radix_slot(&m->amap->anons, slot) = copy;
anon_unref(anon);
anon = copy;
writable = true;
}
spin_lock(&v->lock);
mapping = find_mapping(v, faulty_page);
const bool still_valid = mapping && mapping->object == object &&
(faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info);
*out = vm_page_addr(anon->page);
return enter(v, page, *out, prot_to_pte(v, writable ? m->prot : m->prot & ~PROT_WRITE), CACHE_ANY);
}
if (still_valid) {
errno_t status =
pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
assert(status == 0, "Failed to map a demand-loaded page");
static errno_t fault_promote(
struct vma* v,
struct vma_mapping* m,
size_t page,
size_t slot,
size_t index,
phys_t* out
) {
struct vmo* obj = m->object;
struct page* src = nullptr;
errno_t status = 0;
if (obj && obj->pager->lookup) {
phys_t phys;
status = obj->pager->lookup(obj, index, &phys);
if (!status)
src = vm_page_from_addr(phys);
} else if (obj) {
status = vmo_get_page(obj, index, VMO_READ, &src);
}
spin_unlock(&v->lock);
if (status)
return status;
vmo_unref(object);
return true;
struct vm_anon* anon;
status = anon_new(src, &anon);
if (src && !obj->pager->lookup)
vmo_put_page(src);
if (status)
return status;
status = radix_insert(&m->amap->anons, slot, anon);
if (status) {
anon_unref(anon);
return status;
}
*out = vm_page_addr(anon->page);
return enter(v, page, *out, prot_to_pte(v, m->prot), CACHE_ANY);
}
static errno_t fault_object(struct vma* v, struct vma_mapping* m, size_t page, size_t index, bool write, phys_t* out) {
struct vmo* obj = m->object;
const bool shared = m->flags & VMA_SHARED;
if (obj->pager->lookup) {
errno_t status = obj->pager->lookup(obj, index, out);
if (status)
return status;
return enter(v, page, *out, prot_to_pte(v, shared ? m->prot : m->prot & ~PROT_WRITE), obj->cache);
}
struct page* pg;
errno_t status = vmo_get_page(obj, index, shared && write ? VMO_WRITE : VMO_READ, &pg);
if (status)
return status;
const bool writable = shared && (write || !obj->pager->put_pages);
mutex_lock(&obj->lock);
if (radix_lookup(&obj->pages, index) == pg) {
*out = vm_page_addr(pg);
status = enter(v, page, *out, prot_to_pte(v, writable ? m->prot : m->prot & ~PROT_WRITE), obj->cache);
} else {
status = EAGAIN;
}
mutex_unlock(&obj->lock);
vmo_put_page(pg);
return status;
}
static errno_t fault(struct vma* v, size_t page, enum fault_kind kind, bool force, phys_t* out) {
const bool write = kind == FAULT_WRITE;
while (true) {
rwlock_read_lock(&v->lock);
struct vma_mapping* m = find_mapping(v, page);
if (!m) {
rwlock_read_unlock(&v->lock);
return EFAULT;
}
if (!force && !can_access(m->prot, kind)) {
rwlock_read_unlock(&v->lock);
return EACCES;
}
const bool shared = m->flags & VMA_SHARED;
if (!shared && (write || !m->object) && (!m->amap || (m->flags & VMA_NEEDS_COPY))) {
rwlock_read_unlock(&v->lock);
errno_t status = prepare_amap(v, page);
if (status)
return status;
continue;
}
const size_t index = page - m->start_page + m->offset_page;
const size_t slot = page - m->start_page + m->amap_offset;
struct amap* amap = shared ? nullptr : m->amap;
errno_t status;
if (amap) {
mutex_lock(&amap->lock);
struct vm_anon* anon = radix_lookup(&amap->anons, slot);
if (anon)
status = fault_anon(v, m, anon, page, slot, write, out);
else if (write || !m->object)
status = fault_promote(v, m, page, slot, index, out);
else
status = fault_object(v, m, page, index, false, out);
mutex_unlock(&amap->lock);
} else {
status = fault_object(v, m, page, index, write, out);
}
rwlock_read_unlock(&v->lock);
if (status != EAGAIN)
return status;
}
}
bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
const enum fault_kind kind = info->caused_by_write ? FAULT_WRITE : info->caused_by_fetch ? FAULT_EXEC : FAULT_READ;
phys_t phys;
return fault(v, info->addr / arch_mem_page_size(), kind, false, &phys) == 0;
}
static errno_t write_range(struct vma* v, uintptr_t addr, const void* buf, size_t len) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
while (progress < len) {
const uintptr_t cur = addr + progress;
const size_t misalign = cur % page_size;
const size_t chunk = MIN(page_size - misalign, len - progress);
phys_t phys;
errno_t status = fault(v, cur / page_size, FAULT_WRITE, true, &phys);
if (status)
return status;
uint8_t* dst = (uint8_t*)HHDM_PTR(phys) + misalign;
if (buf)
memcpy(dst, (const uint8_t*)buf + progress, chunk);
else
memset(dst, 0, chunk);
progress += chunk;
}
return 0;
}
errno_t vma_write(struct vma* v, uintptr_t addr, const void* buf, size_t len) {
return write_range(v, addr, buf, len);
}
errno_t vma_zero(struct vma* v, uintptr_t addr, size_t len) {
return write_range(v, addr, nullptr, len);
}
errno_t vma_sync(struct vma* v, uintptr_t addr, size_t len) {
size_t start, end;
errno_t status = page_range(addr, len, &start, &end);
if (status)
return status;
errno_t result = 0;
rwlock_read_lock(&v->lock);
if (!range_mapped(v, start, end)) {
rwlock_read_unlock(&v->lock);
return ENOMEM;
}
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (!m->object || !(m->flags & VMA_SHARED) || m->end_page <= start || m->start_page >= end)
continue;
const size_t first = MAX(start, m->start_page) - m->start_page + m->offset_page;
const size_t last = MIN(end, m->end_page) - m->start_page + m->offset_page;
status = vmo_sync(m->object, first, last);
if (status && !result)
result = status;
}
rwlock_read_unlock(&v->lock);
return result;
}
errno_t vma_map(
@@ -125,34 +639,116 @@ errno_t vma_map(
struct vmo* vmo,
uintptr_t addr,
size_t len,
enum prot_flags flags,
enum prot_flags prot,
enum vma_flags flags,
uintptr_t vmo_offset
) {
if (!vma || !vmo)
if (!vma || (!vmo && (flags & VMA_SHARED)))
return EINVAL;
uintptr_t end;
if (ckd_add(&end, addr, len))
if (len == 0 || ckd_add(&end, addr, len))
return ENOMEM;
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
if (!mapping)
struct vma_mapping* mapping = mapping_alloc();
struct vma_mapping* spare = mapping_alloc();
if (!mapping || !spare) {
mem_free(mapping);
mem_free(spare);
return ENOMEM;
}
vmo_ref(vmo);
mapping->start_page = addr / page_size;
mapping->end_page = ALIGN_UP(end, page_size) / page_size;
mapping->offset_page = vmo_offset / page_size;
mapping->object = vmo;
mapping->prot = flags;
*mapping = (struct vma_mapping){
.start_page = addr / page_size,
.end_page = ALIGN_UP(end, page_size) / page_size,
.offset_page = vmo_offset / page_size,
.object = vmo,
.vma = vma,
.prot = prot,
.flags = flags & VMA_SHARED,
};
if (vmo)
vmo_ref(vmo);
spin_lock(&vma->lock);
tailq_insert_head(&vma->mappings, mapping, link);
spin_unlock(&vma->lock);
mapping_list_t dead;
tailq_init(&dead);
rwlock_write_lock(&vma->lock);
carve(vma, mapping->start_page, mapping->end_page, &spare, &dead);
pmap_shootdown(&vma->pmap);
object_link(mapping);
tailq_insert_tail(&vma->mappings, mapping, link);
rwlock_write_unlock(&vma->lock);
free_mappings(&dead);
mem_free(spare);
return 0;
}
static bool protect_anons(struct vma* v, struct vma_mapping* m) {
const size_t page_size = arch_mem_page_size();
const size_t num_slots = m->end_page - m->start_page;
const enum pte_flags flags = prot_to_pte(v, m->prot & ~PROT_WRITE);
struct vm_anon* batch[32];
uint64_t slots[32];
uint64_t next = m->amap_offset;
bool changed = false;
mutex_lock(&m->amap->lock);
while (true) {
const size_t found = radix_gang_lookup(&m->amap->anons, next, ARRAY_SIZE(batch), (void**)batch, slots);
if (found == 0)
break;
for (size_t i = 0; i < found; i++) {
if (slots[i] - m->amap_offset >= num_slots)
goto done;
const uintptr_t va = (m->start_page + (slots[i] - m->amap_offset)) * page_size;
if (pmap_protect(&v->pmap, va, flags) == 0)
changed = true;
}
next = slots[found - 1] + 1;
}
done:
mutex_unlock(&m->amap->lock);
return changed;
}
errno_t vma_fork(struct vma* src, struct vma** out) {
struct vma* dst;
errno_t status = vma_new(&dst);
if (status)
return status;
bool changed = false;
rwlock_write_lock(&src->lock);
for (struct vma_mapping* m = tailq_first(&src->mappings); m; m = tailq_next(m, link)) {
struct vma_mapping* copy = mapping_alloc();
if (!copy) {
if (changed)
pmap_shootdown(&src->pmap);
rwlock_write_unlock(&src->lock);
vma_unref(dst);
return ENOMEM;
}
if (!(m->flags & VMA_SHARED) && m->amap) {
m->flags |= VMA_NEEDS_COPY;
if ((m->prot & PROT_WRITE) && protect_anons(src, m))
changed = true;
}
mapping_clone(copy, m, dst, m->start_page, m->end_page);
object_link(copy);
tailq_insert_tail(&dst->mappings, copy, link);
}
if (changed)
pmap_shootdown(&src->pmap);
rwlock_write_unlock(&src->lock);
*out = dst;
return 0;
}
mem/vmo.c
+319 -181
View File
@@ -1,75 +1,250 @@
// SPDX-License-Identifier: ISC
#include <zinnia/alloc.h>
#include <zinnia/errno.h>
#include <zinnia/list.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <zinnia/vmo.h>
#include <string.h>
constexpr size_t RA_MIN_PAGES = 8;
constexpr size_t RA_MAX_PAGES = 64;
void vmo_init(struct vmo* vmo, const struct vmo_pager_ops* pager) {
refcount_init(&vmo->rc);
vmo->pager = pager;
mutex_init(&vmo->lock);
radix_init(&vmo->pages);
tailq_init(&vmo->mappings);
vmo->ra_next = UINT64_MAX;
vmo->ra_window = RA_MIN_PAGES;
vmo->cache = CACHE_ANY;
}
void vmo_ref(struct vmo* obj) {
refcount_inc(&obj->rc);
}
static void release_pages(struct vmo* vmo) {
struct page* batch[32];
uint64_t indices[32];
size_t found;
while ((found = radix_gang_lookup(&vmo->pages, 0, ARRAY_SIZE(batch), (void**)batch, indices)) != 0) {
for (size_t i = 0; i < found; i++) {
radix_remove(&vmo->pages, indices[i]);
page_unhold(batch[i]);
}
}
radix_destroy(&vmo->pages);
}
void vmo_unref(struct vmo* obj) {
if (refcount_dec(&obj->rc))
obj->destroy(obj);
if (!refcount_dec(&obj->rc))
return;
if (obj->pager->put_pages) {
errno_t status = vmo_sync(obj, 0, UINT64_MAX);
if (status)
log_warn("vmo: Dropping dirty pages after failed writeback (%i)\n", status);
}
release_pages(obj);
obj->pager->destroy(obj);
}
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
static void mark_dirty(struct vmo* vmo, struct page* page, uint64_t index) {
if (!vmo->pager->put_pages)
return;
atomic_fetch_or(&page->flags, PAGE_DIRTY, ATOMIC_RELAXED);
radix_tag_set(&vmo->pages, index, RADIX_TAG_DIRTY);
}
static void wait_busy(struct vmo* vmo, struct page* page) {
page_hold(page);
mutex_unlock(&vmo->lock);
page_wait(page);
page_unhold(page);
mutex_lock(&vmo->lock);
}
static size_t cluster_size(struct vmo* vmo, uint64_t index) {
const size_t max = MIN(vmo->pager->max_cluster, RA_MAX_PAGES);
if (!vmo->pager->get_pages || max <= 1)
return 1;
vmo->ra_window = index == vmo->ra_next ? MIN(vmo->ra_window * 2, max) : MIN(RA_MIN_PAGES, max);
size_t count = vmo->ra_window;
if (index > UINT64_MAX - count)
count = UINT64_MAX - index;
vmo->ra_next = index + count;
for (size_t i = 1; i < count; i++) {
if (radix_lookup(&vmo->pages, index + i))
return i;
}
return MAX(count, 1);
}
static errno_t fill(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) {
struct page* pages[RA_MAX_PAGES];
size_t count = cluster_size(vmo, index);
const enum alloc_flags flags = vmo->pager->get_pages ? ALLOC_NOZERO : 0;
for (size_t i = 0; i < count; i++) {
struct page* page;
errno_t status = page_alloc(flags, &page);
if (!status) {
page->obj.owner = vmo;
page->obj.index = index + i;
atomic_store(&page->flags, PAGE_BUSY, ATOMIC_RELAXED);
status = radix_insert(&vmo->pages, index + i, page);
if (status)
page_unhold(page);
}
if (status) {
if (i == 0) {
mutex_unlock(&vmo->lock);
return status;
}
count = i;
break;
}
pages[i] = page;
}
page_hold(pages[0]);
mutex_unlock(&vmo->lock);
size_t valid = count;
errno_t status = 0;
if (vmo->pager->get_pages)
status = vmo->pager->get_pages(vmo, index, pages, count, &valid);
mutex_lock(&vmo->lock);
for (size_t i = 0; i < count; i++) {
if (status || i >= valid) {
radix_remove(&vmo->pages, index + i);
page_unbusy(pages[i]);
page_unhold(pages[i]);
continue;
}
if (i == 0 && access == VMO_WRITE)
mark_dirty(vmo, pages[i], index);
page_unbusy(pages[i]);
}
mutex_unlock(&vmo->lock);
if (status || valid == 0) {
page_unhold(pages[0]);
return status ? status : ENXIO;
}
*out = pages[0];
return 0;
}
errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) {
if (vmo->pager->lookup)
return ENOTSUP;
mutex_lock(&vmo->lock);
while (true) {
struct page* page = radix_lookup(&vmo->pages, index);
if (!page)
return fill(vmo, index, access, out);
if (atomic_load(&page->flags, ATOMIC_ACQUIRE) & PAGE_BUSY) {
wait_busy(vmo, page);
continue;
}
page_hold(page);
if (access == VMO_WRITE)
mark_dirty(vmo, page, index);
mutex_unlock(&vmo->lock);
*out = page;
return 0;
}
}
void vmo_put_page(struct page* page) {
page_unhold(page);
}
static errno_t copy_data(
struct vmo* obj,
uint64_t offset,
size_t len,
enum vmo_access access,
void* buf,
iovec_iter_t* iter,
size_t* out
) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t misalign = (progress + offset) % page_size;
const size_t page_index = (progress + offset) / page_size;
const size_t copy_size = MIN(page_size - misalign, len - progress);
const size_t misalign = (offset + progress) % page_size;
const uint64_t index = (offset + progress) / page_size;
const size_t chunk = MIN(page_size - misalign, len - progress);
struct page* p;
status = obj->get_page(obj, page_index, &p);
if (status != 0)
struct page* page = nullptr;
phys_t phys;
if (obj->pager->lookup) {
status = obj->pager->lookup(obj, index, &phys);
} else {
status = vmo_get_page(obj, index, access, &page);
if (!status)
phys = vm_page_addr(page);
}
if (status)
break;
uintptr_t page_addr = vm_page_addr(p);
uint8_t* ptr = (uint8_t*)HHDM_PTR(phys) + misalign;
size_t done = chunk;
if (iter && access == VMO_WRITE)
status = iovec_iter_copy_from(iter, ptr, chunk, &done);
else if (iter)
status = iovec_iter_copy_to(iter, ptr, chunk, &done);
else if (access == VMO_WRITE)
memcpy(ptr, (uint8_t*)buf + progress, chunk);
else
memcpy((uint8_t*)buf + progress, ptr, chunk);
memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
progress += copy_size;
if (page)
vmo_put_page(page);
progress += done;
if (status || done < chunk)
break;
}
if (actual_read)
*actual_read = progress;
if (out)
*out = progress;
return status;
}
errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
errno_t vmo_read(struct vmo* obj, uint64_t offset, void* buf, size_t len, size_t* actual_read) {
return copy_data(obj, offset, len, VMO_READ, buf, nullptr, actual_read);
}
while (progress < len) {
const size_t misalign = (progress + offset) % page_size;
const size_t page_index = (progress + offset) / page_size;
const size_t copy_size = MIN(page_size - misalign, len - progress);
errno_t vmo_write(struct vmo* obj, uint64_t offset, const void* buf, size_t len, size_t* actual_written) {
return copy_data(obj, offset, len, VMO_WRITE, (void*)buf, nullptr, actual_written);
}
struct page* p;
status = obj->get_page(obj, page_index, &p);
if (status != 0)
break;
errno_t vmo_read_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_read) {
return copy_data(obj, offset, len, VMO_READ, nullptr, iter, actual_read);
}
uintptr_t page_addr = vm_page_addr(p);
memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
progress += copy_size;
}
if (actual_written)
*actual_written = progress;
return status;
errno_t vmo_write_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_written) {
return copy_data(obj, offset, len, VMO_WRITE, nullptr, iter, actual_written);
}
errno_t vmo_copy(
struct vmo* target,
uintptr_t target_offset,
uint64_t target_offset,
struct vmo* src,
uintptr_t src_offset,
uint64_t src_offset,
size_t len,
size_t* actual_copied
) {
@@ -80,25 +255,27 @@ errno_t vmo_copy(
while (progress < len) {
const size_t target_misalign = (progress + target_offset) % page_size;
const size_t src_misalign = (progress + src_offset) % page_size;
const size_t target_page_index = (progress + target_offset) / page_size;
const size_t src_page_index = (progress + src_offset) / page_size;
const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
struct page* tpage;
status = target->get_page(target, target_page_index, &tpage);
if (status != 0)
status = vmo_get_page(target, (progress + target_offset) / page_size, VMO_WRITE, &tpage);
if (status)
break;
struct page* spage;
status = src->get_page(src, src_page_index, &spage);
if (status != 0)
status = vmo_get_page(src, (progress + src_offset) / page_size, VMO_READ, &spage);
if (status) {
vmo_put_page(tpage);
break;
}
uintptr_t taddr = vm_page_addr(tpage);
uintptr_t saddr = vm_page_addr(spage);
memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
memcpy(
(uint8_t*)HHDM_PTR(vm_page_addr(tpage)) + target_misalign,
(uint8_t*)HHDM_PTR(vm_page_addr(spage)) + src_misalign,
copy_size
);
vmo_put_page(spage);
vmo_put_page(tpage);
progress += copy_size;
}
@@ -107,157 +284,118 @@ errno_t vmo_copy(
return status;
}
static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
// Try to find the requested page in the cache.
struct page_list* iter;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
// We found a cached page!
*out = iter->value;
spin_unlock(&paged->lock);
errno_t vmo_sync(struct vmo* vmo, uint64_t first, uint64_t end) {
if (!vmo->pager->put_pages)
return 0;
}
spin_unlock(&paged->lock);
// Get page from out pager if we don't have one already.
struct page* new_page;
errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
if (status)
return status;
struct page* run[RA_MAX_PAGES];
const size_t max = MAX(MIN(vmo->pager->max_cluster, RA_MAX_PAGES), 1);
errno_t result = 0;
struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
if (!entry) {
paged->source.put_page(&paged->source, offset_idx, new_page);
return ENOMEM;
}
entry->offset = offset_idx;
entry->value = new_page;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
mutex_lock(&vmo->lock);
while (first < end) {
struct page* found;
uint64_t index;
if (!radix_gang_lookup_tag(&vmo->pages, first, RADIX_TAG_DIRTY, 1, (void**)&found, &index) || index >= end)
break;
if (!page_try_busy(found)) {
wait_busy(vmo, found);
continue;
}
*out = iter->value;
spin_unlock(&paged->lock);
mem_free(entry);
paged->source.put_page(&paged->source, offset_idx, new_page);
return 0;
size_t count = 1;
run[0] = found;
while (count < max && index + count < end && index + count != 0) {
struct page* next = radix_lookup(&vmo->pages, index + count);
if (!next || !radix_tag_get(&vmo->pages, index + count, RADIX_TAG_DIRTY) || !page_try_busy(next))
break;
run[count++] = next;
}
for (size_t i = 0; i < count; i++) {
radix_tag_clear(&vmo->pages, index + i, RADIX_TAG_DIRTY);
radix_tag_set(&vmo->pages, index + i, RADIX_TAG_WRITEBACK);
atomic_fetch_and(&run[i]->flags, ~PAGE_DIRTY, ATOMIC_RELAXED);
page_hold(run[i]);
vma_object_remove_page(vmo, index + i, vm_page_addr(run[i]), true);
}
mutex_unlock(&vmo->lock);
errno_t status = vmo->pager->put_pages(vmo, index, run, count);
mutex_lock(&vmo->lock);
for (size_t i = 0; i < count; i++) {
radix_tag_clear(&vmo->pages, index + i, RADIX_TAG_WRITEBACK);
if (status)
mark_dirty(vmo, run[i], index + i);
page_unbusy(run[i]);
page_unhold(run[i]);
}
if (status && !result)
result = status;
first = index + count;
if (first == 0)
break;
}
// Add page to the object cache.
list_insert_head(&paged->cache, entry, next);
spin_unlock(&paged->lock);
*out = new_page;
return 0;
mutex_unlock(&vmo->lock);
return result;
}
static void paged_destroy(struct vmo* vmo) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
struct page_list* iter = list_first(&paged->cache);
while (iter) {
struct page_list* next = list_next(iter, next);
paged->source.put_page(&paged->source, iter->offset, iter->value);
mem_free(iter);
iter = next;
}
mem_free(paged);
}
errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
if (!result)
return ENOMEM;
refcount_init(&result->object.rc);
result->source = pager;
result->object.get_page = paged_get_page;
result->object.destroy = paged_destroy;
*out = result;
return 0;
}
static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
if (status != 0)
return status;
size_t idx = addr / arch_mem_page_size();
*out = &vm_pfndb[idx];
return 0;
}
static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
return mem_phys_free(vm_page_addr(page), 1);
}
static const struct pager_ops phys_pager = {
.get_page = phys_get_page,
.put_page = phys_put_page,
};
errno_t vmo_new_phys(struct paged_vmo** out) {
return vmo_new_paged(phys_pager, out);
}
void vmo_truncate_paged(struct paged_vmo* paged, size_t length) {
void vmo_truncate(struct vmo* vmo, uint64_t length) {
const size_t page_size = arch_mem_page_size();
const size_t keep_pages = ROUND_UP(length, page_size);
uint64_t first = length / page_size + (length % page_size != 0);
spin_lock(&paged->lock);
struct page_list** pp = &list_first(&paged->cache);
while (*pp) {
struct page_list* entry = *pp;
if (entry->offset >= keep_pages) {
*pp = list_next(entry, next);
paged->source.put_page(&paged->source, entry->offset, entry->value);
mem_free(entry);
} else {
pp = &list_next(entry, next);
mutex_lock(&vmo->lock);
while (true) {
struct page* page;
uint64_t index;
if (!radix_gang_lookup(&vmo->pages, first, 1, (void**)&page, &index))
break;
if (!page_try_busy(page)) {
wait_busy(vmo, page);
continue;
}
vma_object_remove_page(vmo, index, vm_page_addr(page), false);
radix_remove(&vmo->pages, index);
page_unbusy(page);
page_unhold(page);
first = index + 1;
}
if (length % page_size != 0) {
while (true) {
struct page* tail = radix_lookup(&vmo->pages, length / page_size);
if (!tail)
break;
if (atomic_load(&tail->flags, ATOMIC_ACQUIRE) & PAGE_BUSY) {
wait_busy(vmo, tail);
continue;
}
memset((uint8_t*)HHDM_PTR(vm_page_addr(tail)) + length % page_size, 0, page_size - length % page_size);
mark_dirty(vmo, tail, length / page_size);
break;
}
}
spin_unlock(&paged->lock);
mutex_unlock(&vmo->lock);
}
struct phys_range_vmo {
struct vmo object;
uintptr_t start;
size_t len;
struct anon_vmo {
struct vmo vmo;
};
static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
if (offset_idx >= p->len / arch_mem_page_size())
return EINVAL;
*out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
return 0;
static void anon_destroy(struct vmo* vmo) {
mem_free(containerof(vmo, struct anon_vmo, vmo));
}
static void phys_range_destroy(struct vmo* vmo) {
mem_free(containerof(vmo, struct phys_range_vmo, object));
}
static const struct vmo_pager_ops anon_pager = {
.destroy = anon_destroy,
};
errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
if (!vmo)
errno_t vmo_new_anon(struct vmo** out) {
struct anon_vmo* anon = mem_alloc(sizeof(struct anon_vmo), 0);
if (!anon)
return ENOMEM;
refcount_init(&vmo->object.rc);
vmo->object.get_page = phys_range_get_page;
vmo->object.destroy = phys_range_destroy;
vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
vmo->len = ALIGN_UP(length, arch_mem_page_size());
*out = &vmo->object;
vmo_init(&anon->vmo, &anon_pager);
*out = &anon->vmo;
return 0;
}