foo

core/mem/init.c

@@ -64,10 +64,11 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
     mem_hhdm_base = tmp_hhdm;
 
     assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
+    tailq_init(&kernel_vma.mappings);
 
     // text
     log("mem: Mapping text segment at %p\n", __ld_text_start);
-    for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
+    for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
         errno_t status = pmap_map(
             &kernel_vma.pmap,
             (uintptr_t)p,
@@ -116,7 +117,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
     for (size_t i = 0; i < map_len; i++) {
         if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
             continue;
-        for (size_t p = 0; p <= map[i].length; p += pgsz) {
+        for (size_t p = 0; p < map[i].length; p += pgsz) {
             uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
             phys_t paddr = (phys_t)(map[i].address + p);
             errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);

core/mem/pmap.c

@@ -17,6 +17,16 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
     if (status)
         return status;
 
+    pte_t* root = HHDM_PTR(addr);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+    for (size_t i = num_entries / 2; i < num_entries; i++) {
+        phys_t dir;
+        status = mem_phys_alloc(1, flags, &dir);
+        if (status)
+            return status;
+        root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
+    }
+
     struct pmap result = {
         .root = addr,
         .lock = (struct spinlock){0},
@@ -71,7 +81,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
 
         if (pte_is_present(pte)) {
             // Get the next level.
-            *pte = pte_build(pte_address(pte), level_flags, CACHE_ANY);
+            *pte |= pte_build(0, level_flags, CACHE_ANY);
             current_head = HHDM_PTR(pte_address(pte));
         } else {
             // If the current level isn't present, we can skip the rest.

core/mem/pmm.c

@@ -25,7 +25,7 @@ static struct phys_mem* bump_region = nullptr;
 
 static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
     const size_t bytes = num_pages * arch_mem_page_size();
-    if (((intptr_t)bump_mem.length - bytes) <= 0)
+    if (bytes > bump_mem.length)
         return ENOMEM;
 
     *out = bump_mem.address;
@@ -76,7 +76,7 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
     while (it) {
         const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
         // If the region is too high up or doesn't have enough pages left, continue searching.
-        if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
+        if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) {
             prev_it = it;
             it = it->freelist.next;
             continue;
@@ -96,11 +96,26 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
                 *out = page_addr;
         }
         // If not, we can just shrink the block.
-        else {
+        else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) {
             it->freelist.count -= num_pages;
 
             if (__likely(out))
                 *out = page_addr + (it->freelist.count * arch_mem_page_size());
+        } else {
+            struct page* rest = it + num_pages;
+            rest->freelist.count = it->freelist.count - num_pages;
+            rest->freelist.next = it->freelist.next;
+
+            if (prev_it)
+                prev_it->freelist.next = rest;
+            else
+                pmm_head = rest;
+
+            it->freelist.next = nullptr;
+            it->freelist.count = 0;
+
+            if (__likely(out))
+                *out = page_addr;
         }
         goto success;
     }
@@ -118,13 +133,41 @@ success:
     return 0;
 }
 
+static void freelist_insert(struct page* page, size_t num_pages) {
+    struct page* prev = nullptr;
+    struct page* next = pmm_head;
+    while (next && next < page) {
+        prev = next;
+        next = next->freelist.next;
+    }
+
+    page->freelist.count = num_pages;
+    page->freelist.next = next;
+
+    if (next && page + page->freelist.count == next) {
+        page->freelist.count += next->freelist.count;
+        page->freelist.next = next->freelist.next;
+        next->freelist.next = nullptr;
+        next->freelist.count = 0;
+    }
+
+    if (prev && prev + prev->freelist.count == page) {
+        prev->freelist.count += page->freelist.count;
+        prev->freelist.next = page->freelist.next;
+        page->freelist.next = nullptr;
+        page->freelist.count = 0;
+    } else if (prev) {
+        prev->freelist.next = page;
+    } else {
+        pmm_head = page;
+    }
+}
+
 static errno_t freelist_free(phys_t addr, size_t num_pages) {
     spin_lock(&pmm_lock);
 
-    struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
-    page->freelist.count = num_pages;
-    page->freelist.next = pmm_head;
-    pmm_head = page;
+    freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages);
+    pmm_total_free += num_pages;
 
     spin_unlock(&pmm_lock);
     return 0;
@@ -141,11 +184,9 @@ void mem_phys_init(struct phys_mem* map, size_t length) {
         if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
             continue;
 
-        struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
-        page->freelist.count = map[i].length / arch_mem_page_size();
-        page->freelist.next = pmm_head;
-        pmm_head = page;
-        pmm_total_free += page->freelist.count;
+        const size_t num_pages = map[i].length / arch_mem_page_size();
+        freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages);
+        pmm_total_free += num_pages;
     }
 
     const size_t free_bytes = (pmm_total_free * arch_mem_page_size());

core/mem/slab.c

@@ -24,8 +24,6 @@ typedef struct {
 static struct slab slabs[8] = {0};
 
 static errno_t slab_new(struct slab* slab, size_t size) {
-    slab->lock = (struct spinlock){0};
-
     // Allocate a new page for the head.
     phys_t addr;
     errno_t status = mem_phys_alloc(1, 0, &addr);
@@ -68,8 +66,10 @@ void slab_init(void) {
 static void* slab_do_alloc(struct slab* slab) {
     spin_lock(&slab->lock);
 
-    if (__unlikely(slab->head == nullptr))
-        slab_new(slab, slab->ent_size);
+    if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) {
+        spin_unlock(&slab->lock);
+        return nullptr;
+    }
 
     void** old_free = slab->head;
     slab->head = *old_free;

core/mem/usercopy.c

@@ -1,8 +1,18 @@
 #include <zinnia/percpu.h>
 #include <zinnia/usercopy.h>
+#include <arch/mem.h>
 #include <arch/usercopy.h>
 
+static bool is_user_range(uintptr_t addr, size_t len) {
+    if (len == 0)
+        return arch_mem_addr_is_user(addr);
+    return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1);
+}
+
 bool usercopy_read(void* dst, const __user void* src, size_t len) {
+    if (!is_user_range((uintptr_t)src, len))
+        return false;
+
     arch_usercopy_allow(true);
     bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
     arch_usercopy_allow(false);
@@ -10,6 +20,9 @@ bool usercopy_read(void* dst, const __user void* src, size_t len) {
 }
 
 bool usercopy_write(__user void* dst, const void* src, size_t len) {
+    if (!is_user_range((uintptr_t)dst, len))
+        return false;
+
     arch_usercopy_allow(true);
     bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
     arch_usercopy_allow(false);
@@ -17,6 +30,13 @@ bool usercopy_write(__user void* dst, const void* src, size_t len) {
 }
 
 bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
+    if (!arch_mem_addr_is_user((uintptr_t)str))
+        return false;
+
+    size_t unused;
+    if (len == nullptr)
+        len = &unused;
+
     arch_usercopy_allow(true);
     bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
     arch_usercopy_allow(false);

core/mem/virt.c

@@ -1,18 +1,19 @@
 #include <zinnia/alloc.h>
-#include <zinnia/atomic.h>
 #include <zinnia/utils.h>
 #include <zinnia/virt.h>
+#include <stdatomic.h>
 #include <stddef.h>
 
 struct page* vm_pfndb = nullptr;
 
 // TODO: Replace with actual allocator.
 
-static size_t mapping_offset = 0;
+static _Atomic(size_t) mapping_offset = 0;
 
 uintptr_t vm_alloc(size_t length) {
     uintptr_t base = arch_mem_mapping_addr();
-    size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE);
+    size_t offset =
+        atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire);
 
     return base + offset;
 }

core/mem/vma.c

@@ -5,6 +5,7 @@
 #include <zinnia/pmap.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
+#include <stdckdint.h>
 
 // Try to create a new, empty virtual address space.
 errno_t vma_new(struct vma** out) {
@@ -22,6 +23,8 @@ errno_t vma_new(struct vma** out) {
     if (res)
         goto err1;
 
+    tailq_init(&vmspace->mappings);
+
     *out = vmspace;
     return 0;
 
@@ -32,20 +35,90 @@ err0:
 }
 
 // Delete a virtual address space.
-void vma_free(struct vma* v) {}
+void vma_free(struct vma* v) {
+    struct vma_mapping* mapping;
+    while ((mapping = tailq_first(&v->mappings)) != nullptr) {
+        tailq_remove(&v->mappings, mapping, link);
+        vmo_unref(mapping->object);
+        mem_free(mapping);
+    }
+}
 
-// Remove any mappings in the given address range.
-// May fail if there is no memory and a mapping needs to be split.
-errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size);
+errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
+    panic("TODO");
+}
 
-// Change the protection of an existing mapping.
-// May fail if there is no memory and a mapping needs to be split.
-errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags);
+errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
+    panic("TODO");
+}
+
+static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
+    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (page >= m->start_page && page < m->end_page)
+            return m;
+    }
+    return nullptr;
+}
+
+static bool access_allowed(enum prot_flags prot, const struct vma_fault_info* info) {
+    if (info->caused_by_write)
+        return prot & PROT_WRITE;
+    if (info->caused_by_fetch)
+        return prot & PROT_EXEC;
+    return prot & (PROT_READ | PROT_WRITE);
+}
+
+static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
+    enum pte_flags pf = 0;
+    if (prot & PROT_READ)
+        pf |= PTE_READ;
+    if (prot & PROT_WRITE)
+        pf |= PTE_WRITE;
+    if (prot & PROT_EXEC)
+        pf |= PTE_EXEC;
+    if (v->pmap.is_user)
+        pf |= PTE_USER;
+    return pf;
+}
 
-// Try to handle a page fault.
 bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
-    // TODO
-    return false;
+    const size_t page_size = arch_mem_page_size();
+    const size_t faulty_page = info->addr / page_size;
+
+    spin_lock(&v->lock);
+    struct vma_mapping* mapping = find_mapping(v, faulty_page);
+    if (!mapping || !access_allowed(mapping->prot, info)) {
+        spin_unlock(&v->lock);
+        return false;
+    }
+
+    struct vmo* object = mapping->object;
+    const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
+    const enum prot_flags prot = mapping->prot;
+    vmo_ref(object);
+    spin_unlock(&v->lock);
+
+    struct page* page;
+    if (object->get_page(object, object_page, &page)) {
+        vmo_unref(object);
+        return false;
+    }
+
+    spin_lock(&v->lock);
+    mapping = find_mapping(v, faulty_page);
+    const bool still_valid = mapping && mapping->object == object &&
+        (faulty_page - mapping->start_page) + mapping->offset_page == object_page &&
+        access_allowed(mapping->prot, info);
+
+    if (still_valid) {
+        errno_t status =
+            pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
+        assert(status == 0, "Failed to map a demand-loaded page\n");
+    }
+    spin_unlock(&v->lock);
+
+    vmo_unref(object);
+    return true;
 }
 
 errno_t vma_map(
@@ -56,44 +129,31 @@ errno_t vma_map(
     enum prot_flags flags,
     uintptr_t vmo_offset
 ) {
-    const size_t page_size = arch_mem_page_size();
-
-    if (addr % page_size != vmo_offset % page_size)
-        return EINVAL;
-
-    size_t start_page = addr / page_size;
-    const size_t addr_offset = addr % page_size;
-    const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size;
-    size_t end_page = start_page + pages;
-
     if (!vma || !vmo)
         return EINVAL;
 
-    const size_t first_vmo_page = vmo_offset / page_size;
+    uintptr_t end;
+    if (ckd_add(&end, addr, len))
+        return ENOMEM;
 
-    for (size_t p = start_page; p < end_page; ++p) {
-        const size_t page_index = p - start_page;
-        struct page* page_obj;
-        errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj);
-        if (status != 0)
-            return status;
+    const size_t page_size = arch_mem_page_size();
+    if (addr % page_size != vmo_offset % page_size)
+        return EINVAL;
 
-        phys_t paddr = (phys_t)vm_page_addr(page_obj);
+    struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
+    if (!mapping)
+        return ENOMEM;
 
-        enum pte_flags pf = 0;
-        if (flags & PROT_READ)
-            pf |= PTE_READ;
-        if (flags & PROT_WRITE)
-            pf |= PTE_WRITE;
-        if (flags & PROT_EXEC)
-            pf |= PTE_EXEC;
-        if (vma->pmap.is_user)
-            pf |= PTE_USER;
+    vmo_ref(vmo);
+    mapping->start_page = addr / page_size;
+    mapping->end_page = ALIGN_UP(end, page_size) / page_size;
+    mapping->offset_page = vmo_offset / page_size;
+    mapping->object = vmo;
+    mapping->prot = flags;
 
-        status = pmap_map(&vma->pmap, p * page_size, paddr, pf, CACHE_ANY);
-        if (status != 0)
-            return status;
-    }
+    spin_lock(&vma->lock);
+    tailq_insert_head(&vma->mappings, mapping, link);
+    spin_unlock(&vma->lock);
 
     return 0;
 }

core/mem/vmo.c

@@ -5,6 +5,15 @@
 #include <zinnia/vmo.h>
 #include <zinnia/string.h>
 
+void vmo_ref(struct vmo* obj) {
+    refcount_inc(&obj->rc);
+}
+
+void vmo_unref(struct vmo* obj) {
+    if (refcount_dec(&obj->rc))
+        obj->destroy(obj);
+}
+
 errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
     const size_t page_size = arch_mem_page_size();
     size_t progress = 0;
@@ -104,14 +113,17 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page
 
     // Try to find the requested page in the cache.
     struct page_list* iter;
+    spin_lock(&paged->lock);
     list_foreach(iter, &paged->cache, next) {
         if (iter->offset != offset_idx)
             continue;
 
         // We found a cached page!
         *out = iter->value;
+        spin_unlock(&paged->lock);
         return 0;
     }
+    spin_unlock(&paged->lock);
 
     // Get page from out pager if we don't have one already.
     struct page* new_page;
@@ -120,26 +132,57 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page
         return status;
 
     struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
-    if (!entry)
+    if (!entry) {
+        paged->source.put_page(&paged->source, offset_idx, new_page);
         return ENOMEM;
+    }
 
     entry->offset = offset_idx;
     entry->value = new_page;
 
+    spin_lock(&paged->lock);
+    list_foreach(iter, &paged->cache, next) {
+        if (iter->offset != offset_idx)
+            continue;
+
+        *out = iter->value;
+        spin_unlock(&paged->lock);
+        mem_free(entry);
+        paged->source.put_page(&paged->source, offset_idx, new_page);
+        return 0;
+    }
+
     // Add page to the object cache.
     list_insert_head(&paged->cache, entry, next);
+    spin_unlock(&paged->lock);
 
     *out = new_page;
     return 0;
 }
 
+static void paged_destroy(struct vmo* vmo) {
+    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
+
+    struct page_list* iter = list_first(&paged->cache);
+    while (iter) {
+        struct page_list* next = list_next(iter, next);
+        paged->source.put_page(&paged->source, iter->offset, iter->value);
+        mem_free(iter);
+        iter = next;
+    }
+
+    mem_free(paged);
+}
+
 errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
     struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
     if (!result)
         return ENOMEM;
 
+    refcount_init(&result->object.rc);
     result->source = pager;
     result->object.get_page = paged_get_page;
+    result->object.destroy = paged_destroy;
 
     *out = result;
     return 0;
@@ -156,8 +199,7 @@ static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct p
 }
 
 static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
-    // We do nothing here.
-    return 0;
+    return mem_phys_free(vm_page_addr(page), 1);
 }
 
 static const struct pager_ops phys_pager = {
@@ -177,16 +219,24 @@ struct phys_range_vmo {
 
 static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
     struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
+    if (offset_idx >= p->len / arch_mem_page_size())
+        return EINVAL;
     *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
     return 0;
 }
 
+static void phys_range_destroy(struct vmo* vmo) {
+    mem_free(containerof(vmo, struct phys_range_vmo, object));
+}
+
 errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
     struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
     if (!vmo)
         return ENOMEM;
 
+    refcount_init(&vmo->object.rc);
     vmo->object.get_page = phys_range_get_page;
+    vmo->object.destroy = phys_range_destroy;
     vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
     vmo->len = ALIGN_UP(length, arch_mem_page_size());