marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compiles

Browse Source
This commit is contained in:
Marvin
2026-09-17 03:19:43 +02:00
parent 6719239036
commit 27fe15f83d
85 changed files with 945 additions and 901 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mem/init.c
-166
View File
@@ -1,166 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/init.h>
#include <zinnia/panic.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <zinnia/vas.h>
#include <zinnia/virt.h>
#include <string.h>
extern uint8_t __ld_text_start[];
extern uint8_t __ld_text_end[];
extern uint8_t __ld_rodata_start[];
extern uint8_t __ld_rodata_end[];
extern uint8_t __ld_data_start[];
extern uint8_t __ld_data_end[];
extern uint8_t __ld_kernel_start[];
struct vas kernel_vas = {0};
[[__init]]
void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) {
// This function creates a kernel page table and initializes all memory managers.
const size_t pgsz = arch_mem_page_size();
kprintf("Memory map:\n");
for (size_t i = 0; i < map_len; i++) {
if (map[i].length == 0)
continue;
const char* label = nullptr;
switch (map[i].usage) {
case PHYS_RESERVED:
label = "Reserved";
break;
case PHYS_USABLE:
label = "Usable";
break;
case PHYS_STATIC:
label = "Static";
break;
case PHYS_RECLAIMABLE:
label = "Reclaimable";
break;
}
kprintf("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
}
// Set up the bootstrap allocator.
struct phys_mem* largest = map;
for (size_t i = 0; i < map_len; i++) {
if (map[i].length > largest->length && map[i].usage == PHYS_USABLE)
largest = &map[i];
}
mem_phys_bootstrap(largest);
kprintf(
"Using region [%p - %p] for bootstrap allocator\n",
(void*)largest->address,
(void*)(largest->address + largest->length - 1)
);
// Set the HHDM base address to the address given by the loader.
// We must not keep any virtual addresses to this region,
// since we're likely going to map it at a different base address.
mem_hhdm_base = tmp_hhdm;
kassert(pmap_new_kernel(&kernel_vas.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
// text
kprintf("Mapping text segment at %p\n", __ld_text_start);
for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vas.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ | PTE_EXEC,
CACHE_ANY
);
assert(!status, "Failed to map %p with error %i\n", p, status);
}
// rodata
kprintf("Mapping rodata segment at %p\n", __ld_rodata_start);
for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vas.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ,
CACHE_ANY
);
assert(!status, "Failed to map %p with error %i\n", p, status);
}
// data
kprintf("Mapping data segment at %p\n", __ld_data_start);
for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vas.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ | PTE_WRITE,
CACHE_ANY
);
assert(!status, "Failed to map %p with error %i\n", p, status);
}
// The kernel address space is divided into 4 segments.
// For example, they could live at:
// HHDM FFFF'8000'0000'0000
// PFNDB FFFF'A000'0000'0000
// Mappings FFFF'C000'0000'0000
// Kernel FFFF'FFFF'8000'0000
// Map all physical memory to the HHDM address.
tmp_hhdm = arch_mem_hhdm_addr();
for (size_t i = 0; i < map_len; i++) {
if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
continue;
for (size_t p = 0; p <= map[i].length; p += pgsz) {
uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
phys_t paddr = (phys_t)(map[i].address + p);
errno_t status = pmap_map(&kernel_vas.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
assert(!status, "Failed to map HHDM page %p to %p with error %i\n", (void*)vaddr, (void*)paddr, status);
}
}
mem_hhdm_base = tmp_hhdm;
// Switch to our own page table.
pmap_set(&kernel_vas.pmap);
// We record metadata for every single page of available memory in a large array.
// This array is contiguous in virtual memory, but is sparsely populated.
// Only those array entries which represent usable memory are mapped.
for (size_t i = 0; i < map_len; i++) {
if (map[i].length == 0 || map[i].usage != PHYS_USABLE)
continue;
const size_t num_pages = (map[i].length + pgsz - 1) / pgsz;
const size_t metadata_size = num_pages * sizeof(struct page);
const uintptr_t metadata_start_vaddr = arch_vm_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size;
// Find the page-aligned boundaries that contain this metadata.
const uintptr_t aligned_start = ALIGN_DOWN(metadata_start_vaddr, pgsz);
const uintptr_t aligned_end = ALIGN_UP(metadata_end_vaddr, pgsz);
const size_t length = aligned_end - aligned_start;
const uintptr_t vaddr = aligned_start;
for (size_t page = 0; page < length; page += pgsz) {
phys_t paddr;
assert(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!\n");
pmap_map(&kernel_vas.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
}
}
vm_pfndb = (struct page*)arch_vm_pfndb_addr();
// We don't need the bootstrap allocator from this point on.
// Initialize the real page allocator.
mem_phys_init(map, map_len);
slab_init();
kprintf("Memory initialization complete\n");
}
mem/meson.build
-11
View File
@@ -1,11 +0,0 @@
kernel_sources += files(
'init.c',
'mmio.c',
'pmap.c',
'pmm.c',
'slab.c',
'usercopy.c',
'virt.c',
'vas.c',
'vmo.c',
)
mem/mmio.c
-22
View File
@@ -1,22 +0,0 @@
#include <bits/mem.h>
#include <zinnia/mmio.h>
#include <zinnia/pmap.h>
#include <zinnia/vas.h>
#include <zinnia/virt.h>
void* mmio_new(phys_t addr, size_t length) {
void* ptr = vm_alloc(length);
for (size_t i = 0; i < length; i += arch_mem_page_size()) {
if (pmap_map(&kernel_vas.pmap, (uintptr_t)ptr + i, addr + i, PTE_READ | PTE_WRITE, CACHE_MMIO)) {
vm_free(ptr, length);
return nullptr;
}
}
return ptr;
}
void mmio_free(void* ptr, size_t length) {
// TODO
}
mem/pmap.c
-104
View File
@@ -1,104 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/errno.h>
#include <zinnia/pmap.h>
#include <zinnia/spin.h>
#include <zinnia/vas.h>
#include <string.h>
errno_t pmap_kernel(struct pmap* pt, enum alloc_flags flags) {
flags &= ~ALLOC_NOZERO;
phys_t addr;
errno_t status = mem_phys_alloc(1, flags, &addr);
if (status)
return status;
struct pmap result = {
.root = addr,
.lock = (spinlock_t){0},
.is_user = false,
};
*pt = result;
return 0;
}
errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
phys_t user_l1;
errno_t status = mem_phys_alloc(1, 0, &user_l1);
if (status)
return status;
void* user_l1_ptr = HHDM_PTR(user_l1);
void* kernel_l1_ptr = HHDM_PTR(kernel_vas.pmap.root);
memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
struct pmap result = {
.root = user_l1,
.lock = (spinlock_t){0},
.is_user = true,
};
*pt = result;
return 0;
}
// Gets a reference to the PTE at the given virtual address.
// If `check_only` is set, only checks if the PTE exists,
// and doesn't allocate new levels if they don't already exist.
// If it can't allocate a page if it has to, returns `nullptr`.
static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) {
pte_t* current_head = HHDM_PTR(pt->root);
size_t index = 0;
for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) {
const size_t addr_mask = (1 << arch_mem_level_bits()) - 1;
const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level);
const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0);
index = (vaddr >> addr_shift) & addr_mask;
// The last level is used to access the actual PTE, so break the loop then.
// We still need to update the index beforehand, that's why we can't just end early.
if (level == 0)
break;
pte_t* pte = &current_head[index];
if (pte_is_present(pte)) {
// Get the next level.
*pte = pte_build(pte_address(pte), level_flags, CACHE_ANY);
current_head = HHDM_PTR(pte_address(pte));
} else {
// If the current level isn't present, we can skip the rest.
if (check_only)
return nullptr;
phys_t addr;
if (mem_phys_alloc(1, 0, &addr) != 0)
return nullptr;
*pte = pte_build(addr, level_flags, CACHE_ANY);
current_head = HHDM_PTR(addr);
}
}
return &current_head[index];
}
errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
spin_lock(&pt->lock);
errno_t status = 0;
pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
if (pte == nullptr) {
status = ENOMEM;
goto fail;
}
*pte = pte_build(paddr, flags, cache);
fail:
spin_unlock(&pt->lock);
return status;
}
mem/pmm.c
-154
View File
@@ -1,154 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/compiler.h>
#include <zinnia/page.h>
#include <zinnia/panic.h>
#include <zinnia/print.h>
#include <zinnia/spin.h>
#include <stdint.h>
#include <string.h>
uintptr_t mem_hhdm_base = 0;
static errno_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
static errno_t (*free_fn)(phys_t start, size_t num_pages);
errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
return alloc_fn(num_pages, flags, out);
}
errno_t mem_phys_free(phys_t start, size_t num_pages) {
return free_fn(start, num_pages);
}
static struct phys_mem bump_mem = {0};
static struct phys_mem* bump_region = nullptr;
static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
const size_t bytes = num_pages * arch_mem_page_size();
if (((intptr_t)bump_mem.length - bytes) <= 0)
return ENOMEM;
*out = bump_mem.address;
bump_mem.length -= bytes;
bump_mem.address += bytes;
if (!(flags & ALLOC_NOZERO)) {
memset(HHDM_PTR(*out), 0, bytes);
}
return 0;
}
static errno_t bump_free(phys_t addr, size_t num_pages) {
// We don't free bootstrap memory.
panic("Attempted to free bootstrap memory!");
}
void mem_phys_bootstrap(struct phys_mem* mem) {
alloc_fn = bump_alloc;
free_fn = bump_free;
// Save the region by reference and value.
// We can't directly modify the region by reference, because that
// interferes with e.g. HHDM mapping logic.
bump_mem = *mem;
bump_region = mem;
}
static struct page* pmm_head = nullptr;
static spinlock_t pmm_lock = {0};
static size_t pmm_total_free = 0;
static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
spin_lock(&pmm_lock);
const size_t bytes = num_pages * arch_mem_page_size();
phys_t limit = UINTPTR_MAX;
if (flags & ALLOC_MEM20) {
limit = (phys_t)1 << 20;
} else if (flags & ALLOC_MEM32) {
limit = (phys_t)1 << 32;
}
struct page* it = pmm_head;
struct page* prev_it = nullptr;
while (it) {
const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
// If the region is too high up or doesn't have enough pages left, continue searching.
if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
prev_it = it;
it = it->freelist.next;
continue;
}
// If the current block has exactly enough pages left, consume it entirely.
if (__unlikely(it->freelist.count == num_pages)) {
if (prev_it) {
prev_it->freelist.next = it->freelist.next;
} else {
pmm_head = it->freelist.next;
}
it->freelist.next = nullptr;
it->freelist.count = 0;
if (__likely(out))
*out = page_addr;
}
// If not, we can just shrink the block.
else {
it->freelist.count -= num_pages;
if (__likely(out))
*out = page_addr + (it->freelist.count * arch_mem_page_size());
}
goto success;
}
spin_unlock(&pmm_lock);
return ENOMEM;
success:
// If the NOZERO flag is *not* specified, zero memory.
if (__likely(out) && __likely(!(flags & ALLOC_NOZERO))) {
memset(HHDM_PTR(*out), 0, bytes);
}
pmm_total_free -= num_pages;
spin_unlock(&pmm_lock);
return 0;
}
static errno_t freelist_free(phys_t addr, size_t num_pages) {
spin_lock(&pmm_lock);
struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
page->freelist.count = num_pages;
page->freelist.next = pmm_head;
pmm_head = page;
spin_unlock(&pmm_lock);
return 0;
}
void mem_phys_init(struct phys_mem* map, size_t length) {
alloc_fn = freelist_alloc;
free_fn = freelist_free;
bump_region->address = bump_mem.address;
bump_region->length = bump_mem.length;
for (size_t i = 0; i < length; i++) {
// Regions smaller than a page are useless.
if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
continue;
struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
page->freelist.count = map[i].length / arch_mem_page_size();
page->freelist.next = pmm_head;
pmm_head = page;
pmm_total_free += page->freelist.count;
}
const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
kprintf("Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
}
mem/slab.c
-143
View File
@@ -1,143 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/spin.h>
#include <string.h>
typedef struct {
size_t num_pages; // Amount of pages connected to this slab.
size_t size; // Size of this slab.
} slab_info;
struct slab {
spinlock_t lock; // Access lock.
size_t ent_size; // Size of one entry.
void** head;
};
typedef struct {
struct slab* slab;
} slab_header;
// Initializes the SLAB structures.
static struct slab slabs[8] = {0};
static errno_t slab_new(struct slab* slab, size_t size) {
slab->lock = (spinlock_t){0};
// Allocate a new page for the head.
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
if (status)
return status;
slab->head = (void**)(addr + arch_mem_hhdm_addr());
slab->ent_size = size;
const size_t offset = ALIGN_UP(sizeof(slab_header), size);
const size_t available_size = arch_mem_page_size() - offset;
slab_header* ptr = (slab_header*)slab->head;
ptr->slab = slab;
slab->head = (void**)((void*)slab->head + offset);
void** arr = slab->head;
const size_t max = available_size / size - 1;
const size_t fact = size / sizeof(void*);
for (size_t i = 0; i < max; i++) {
arr[i * fact] = &arr[(i + 1) * fact];
}
arr[max * fact] = nullptr;
return 0;
}
void slab_init(void) {
slab_new(&slabs[0], 16);
slab_new(&slabs[1], 32);
slab_new(&slabs[2], 64);
slab_new(&slabs[3], 128);
slab_new(&slabs[4], 256);
slab_new(&slabs[5], 512);
slab_new(&slabs[6], 1024);
slab_new(&slabs[7], 2048);
}
static void* slab_do_alloc(struct slab* slab) {
spin_lock(&slab->lock);
if (__unlikely(slab->head == nullptr))
slab_new(slab, slab->ent_size);
void** old_free = slab->head;
slab->head = *old_free;
memset(old_free, 0, slab->ent_size);
spin_unlock(&slab->lock);
return old_free;
}
static void slab_do_free(struct slab* slab, void* addr) {
spin_lock(&slab->lock);
if (__unlikely(addr == nullptr))
goto cleanup;
void** new_head = addr;
*new_head = slab->head;
slab->head = new_head;
cleanup:
spin_unlock(&slab->lock);
}
static inline struct slab* slab_find_size(size_t size) {
for (size_t i = 0; i < ARRAY_SIZE(slabs); i++) {
if (slabs[i].ent_size >= size)
return &slabs[i];
}
return nullptr;
}
void* mem_alloc(size_t size, enum alloc_flags flags) {
if (__unlikely(size == 0))
return nullptr;
struct slab* slab = slab_find_size(size);
if (slab != nullptr) {
return slab_do_alloc(slab);
}
size_t num_pages = ROUND_UP(size, arch_mem_page_size());
// Allocate the pages plus an additional page for metadata.
phys_t ret;
errno_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
if (__unlikely(status != 0))
return nullptr;
ret = ret + (phys_t)arch_mem_hhdm_addr();
// Write metadata into the first page.
slab_info* info = (slab_info*)ret;
info->num_pages = num_pages;
info->size = size;
// Skip the first page and return the next one.
return (void*)(ret + arch_mem_page_size());
}
void mem_free(void* addr) {
if (__unlikely(addr == nullptr))
return;
// If the address is page aligned.
if ((size_t)addr == ALIGN_DOWN((size_t)addr, arch_mem_page_size())) {
slab_info* info = (slab_info*)(addr - arch_mem_page_size());
mem_phys_free(((phys_t)info - (phys_t)arch_mem_hhdm_addr()), info->num_pages + 1);
return;
}
slab_header* header = (slab_header*)(ALIGN_DOWN((size_t)addr, arch_mem_page_size()));
slab_do_free(header->slab, addr);
}
mem/usercopy.c
-15
View File
@@ -1,15 +0,0 @@
#include <zinnia/bits/usercopy.h>
#include <zinnia/percpu.h>
#include <zinnia/usercopy.h>
bool usercopy_read(void* dst, const __user void* src, size_t len) {
return arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
}
bool usercopy_write(__user void* dst, const void* src, size_t len) {
return arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
}
bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
return arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
}
mem/vas.c
-98
View File
@@ -1,98 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/errno.h>
#include <zinnia/mman.h>
#include <zinnia/utils.h>
#include <zinnia/vas.h>
#include <assert.h>
// Try to create a new, empty virtual address space.
errno_t vma_new(struct vas** out) {
debug_assert(out != nullptr, "Missing out");
errno_t res;
if (!out)
return EINVAL;
struct vas* vmspace = mem_alloc(sizeof(struct vas), 0);
if (!vmspace)
return ENOMEM;
res = pmap_new_user(&vmspace->pmap, 0);
if (res)
goto err1;
*out = vmspace;
return 0;
err1:
mem_free(vmspace);
err0:
return res;
}
// Delete a virtual address space.
void vma_delete(struct vas* vm) {}
// Remove any mappings in the given address range.
// May fail if there is no memory and a mapping needs to be split.
errno_t vma_unmap(struct vas* vm, uintptr_t vaddr, size_t size);
// Change the protection of an existing mapping.
// May fail if there is no memory and a mapping needs to be split.
errno_t vma_protect(struct vas* vm, uintptr_t vaddr, size_t size, enum zn_vm_flags flags);
// Try to handle a page fault.
bool vma_page_fault(struct vas* vm, uintptr_t vaddr, enum zn_vm_flags access_type) {
// TODO
return false;
}
errno_t vma_map_vmo(
struct vas* vas,
struct vmo* vmo,
uintptr_t addr,
size_t len,
enum prot_flags flags,
uintptr_t vmo_offset
) {
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
size_t start_page = addr / page_size;
const size_t addr_offset = addr % page_size;
const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size;
size_t end_page = start_page + pages;
if (!vas || !vmo)
return EINVAL;
const size_t first_vmo_page = vmo_offset / page_size;
for (size_t p = start_page; p < end_page; ++p) {
const size_t page_index = p - start_page;
struct page* page_obj;
errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj);
if (status != 0)
return status;
phys_t paddr = (phys_t)vm_page_addr(page_obj);
enum pte_flags pf = 0;
if (flags & PROT_READ)
pf |= PTE_READ;
if (flags & PROT_WRITE)
pf |= PTE_WRITE;
if (flags & PROT_EXEC)
pf |= PTE_EXEC;
if (vas->pmap.is_user)
pf |= PTE_USER;
status = pmap_map(&vas->pmap, p * page_size, paddr, pf, CACHE_ANY);
if (status != 0)
return status;
}
return 0;
}
mem/virt.c
-26
View File
@@ -1,26 +0,0 @@
#include <stdatomic.h>
#include <stddef.h>
#include <zinnia/alloc.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
struct page* vm_pfndb = nullptr;
// TODO: Replace with actual allocator.
static size_t mapping_offset = 0;
void* vm_alloc(size_t length) {
uintptr_t base = arch_mem_mapping_addr();
size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()));
return (void*)(base + offset);
}
void vm_free(void* ptr, size_t length) {
// TODO
}
uintptr_t vm_page_addr(struct page* page) {
return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
}
mem/vmo.c
-196
View File
@@ -1,196 +0,0 @@
#include <string.h>
#include <zinnia/alloc.h>
#include <zinnia/errno.h>
#include <zinnia/list.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <zinnia/vmo.h>
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t misalign = (progress + offset) % page_size;
const size_t page_index = (progress + offset) / page_size;
const size_t copy_size = MIN(page_size - misalign, len - progress);
struct page* p;
status = obj->get_page(obj, page_index, &p);
if (status != 0)
break;
uintptr_t page_addr = vm_page_addr(p);
memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
progress += copy_size;
}
if (actual_read)
*actual_read = progress;
return status;
}
errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t misalign = (progress + offset) % page_size;
const size_t page_index = (progress + offset) / page_size;
const size_t copy_size = MIN(page_size - misalign, len - progress);
struct page* p;
status = obj->get_page(obj, page_index, &p);
if (status != 0)
break;
uintptr_t page_addr = vm_page_addr(p);
memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
progress += copy_size;
}
if (actual_written)
*actual_written = progress;
return status;
}
errno_t vmo_copy(
struct vmo* target,
uintptr_t target_offset,
struct vmo* src,
uintptr_t src_offset,
size_t len,
size_t* actual_copied
) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t target_misalign = (progress + target_offset) % page_size;
const size_t src_misalign = (progress + src_offset) % page_size;
const size_t target_page_index = (progress + target_offset) / page_size;
const size_t src_page_index = (progress + src_offset) / page_size;
const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
struct page* tpage;
status = target->get_page(target, target_page_index, &tpage);
if (status != 0)
break;
struct page* spage;
status = src->get_page(src, src_page_index, &spage);
if (status != 0)
break;
uintptr_t taddr = vm_page_addr(tpage);
uintptr_t saddr = vm_page_addr(spage);
memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
progress += copy_size;
}
if (actual_copied)
*actual_copied = progress;
return status;
}
static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
// Try to find the requested page in the cache.
struct page_list* iter;
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
// We found a cached page!
*out = iter->value;
return 0;
}
// Get page from out pager if we don't have one already.
struct page* new_page;
errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
if (status)
return status;
struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
if (!entry)
return ENOMEM;
entry->offset = offset_idx;
entry->value = new_page;
// Add page to the object cache.
list_insert_head(&paged->cache, entry, next);
*out = new_page;
return 0;
}
errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
if (!result)
return ENOMEM;
result->source = pager;
result->object.get_page = paged_get_page;
*out = result;
return 0;
}
static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
if (status != 0)
return status;
size_t idx = addr / arch_mem_page_size();
*out = &vm_pfndb[idx];
return 0;
}
static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
// We do nothing here.
return 0;
}
static const struct pager_ops phys_pager = {
.get_page = phys_get_page,
.put_page = phys_put_page,
};
errno_t vmo_new_phys(struct paged_vmo** out) {
return vmo_new_paged(phys_pager, out);
}
struct phys_range_vmo {
struct vmo object;
uintptr_t start;
size_t len;
};
static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
*out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
return 0;
}
errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
if (!vmo)
return ENOMEM;
vmo->object.get_page = phys_range_get_page;
vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
vmo->len = ALIGN_UP(length, arch_mem_page_size());
*out = &vmo->object;
return 0;
}