amaps

include/zinnia/amap.h

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/mutex.h>
+#include <zinnia/radix.h>
+#include <zinnia/refcount.h>
+#include <zinnia/virt.h>
+#include <stddef.h>
+
+struct vm_anon {
+    _Atomic(size_t) rc;
+    struct page* page;
+};
+
+struct amap {
+    struct refcount rc;
+    mutex_t lock;
+    struct radix_tree anons;
+};
+
+errno_t anon_new(struct page* src, struct vm_anon** out);
+void anon_ref(struct vm_anon* anon);
+void anon_unref(struct vm_anon* anon);
+
+errno_t amap_new(struct amap** out);
+void amap_ref(struct amap* amap);
+void amap_unref(struct amap* amap);
+bool amap_is_shared(struct amap* amap);
+errno_t amap_copy(struct amap* src, size_t offset, size_t num_slots, struct amap** out);

include/zinnia/virt.h

@@ -2,6 +2,9 @@
 
 #pragma once
 
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
 #include <zinnia/hashmap.h>
 #include <zinnia/types.h>
 #include <stddef.h>
@@ -11,15 +14,26 @@ enum page_type : uint8_t {
     PAGE_PHYS = 0, // Regular physical memory.
 };
 
+enum page_flags : uint32_t {
+    PAGE_BUSY = 1 << 0,
+    PAGE_WANTED = 1 << 1,
+    PAGE_DIRTY = 1 << 2,
+    PAGE_ANON = 1 << 3,
+};
+
 struct page {
     enum page_type type;
-    uint32_t flags;
-    size_t refcount;
+    _Atomic(uint32_t) flags;
+    _Atomic(size_t) holds;
     union {
         struct {
             struct page* next; // Pointer to the next chunk.
             size_t count;      // Amount of free pages.
         } freelist;
+        struct {
+            void* owner;
+            uint64_t index;
+        } obj;
     };
 };
 static_assert(0x1000 % sizeof(struct page) == 0, "struct must fit cleanly inside a page!");
@@ -29,6 +43,14 @@ static_assert(sizeof(struct page) <= 64, "struct must be smaller than 64 bytes!"
 extern struct page* vm_pfndb;
 
 uintptr_t vm_page_addr(struct page* page);
+struct page* vm_page_from_addr(phys_t addr);
+
+errno_t page_alloc(enum alloc_flags flags, struct page** out);
+void page_hold(struct page* page);
+void page_unhold(struct page* page);
+bool page_try_busy(struct page* page);
+void page_unbusy(struct page* page);
+void page_wait(struct page* page);
 
 // Allocator for virtual memory.
 struct vm_alloc {

mem/amap.c

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/amap.h>
+#include <zinnia/utils.h>
+#include <string.h>
+
+errno_t anon_new(struct page* src, struct vm_anon** out) {
+    struct vm_anon* anon = mem_alloc(sizeof(struct vm_anon), 0);
+    if (!anon)
+        return ENOMEM;
+
+    struct page* page;
+    errno_t status = page_alloc(src ? ALLOC_NOZERO : 0, &page);
+    if (status) {
+        mem_free(anon);
+        return status;
+    }
+
+    if (src)
+        memcpy(HHDM_PTR(vm_page_addr(page)), HHDM_PTR(vm_page_addr(src)), arch_mem_page_size());
+
+    atomic_fetch_or(&page->flags, PAGE_ANON, ATOMIC_RELAXED);
+    page->obj.owner = anon;
+    atomic_store(&anon->rc, 1, ATOMIC_RELAXED);
+    anon->page = page;
+    *out = anon;
+    return 0;
+}
+
+void anon_ref(struct vm_anon* anon) {
+    atomic_fetch_add(&anon->rc, 1, ATOMIC_RELAXED);
+}
+
+void anon_unref(struct vm_anon* anon) {
+    if (atomic_fetch_sub(&anon->rc, 1, ATOMIC_RELEASE) != 1)
+        return;
+    atomic_fence(ATOMIC_ACQUIRE);
+    page_unhold(anon->page);
+    mem_free(anon);
+}
+
+errno_t amap_new(struct amap** out) {
+    struct amap* amap = mem_alloc(sizeof(struct amap), 0);
+    if (!amap)
+        return ENOMEM;
+    refcount_init(&amap->rc);
+    mutex_init(&amap->lock);
+    radix_init(&amap->anons);
+    *out = amap;
+    return 0;
+}
+
+void amap_ref(struct amap* amap) {
+    refcount_inc(&amap->rc);
+}
+
+void amap_unref(struct amap* amap) {
+    if (!refcount_dec(&amap->rc))
+        return;
+
+    struct vm_anon* batch[32];
+    uint64_t slots[32];
+    size_t found;
+    while ((found = radix_gang_lookup(&amap->anons, 0, ARRAY_SIZE(batch), (void**)batch, slots)) != 0) {
+        for (size_t i = 0; i < found; i++) {
+            radix_remove(&amap->anons, slots[i]);
+            anon_unref(batch[i]);
+        }
+    }
+    radix_destroy(&amap->anons);
+    mem_free(amap);
+}
+
+bool amap_is_shared(struct amap* amap) {
+    return atomic_load(&amap->rc.count, ATOMIC_ACQUIRE) != 1;
+}
+
+errno_t amap_copy(struct amap* src, size_t offset, size_t num_slots, struct amap** out) {
+    struct amap* copy;
+    errno_t status = amap_new(&copy);
+    if (status)
+        return status;
+
+    struct vm_anon* batch[32];
+    uint64_t slots[32];
+    uint64_t next = offset;
+
+    mutex_lock(&src->lock);
+    while (status == 0) {
+        const size_t found = radix_gang_lookup(&src->anons, next, ARRAY_SIZE(batch), (void**)batch, slots);
+        if (found == 0)
+            break;
+        for (size_t i = 0; i < found && status == 0; i++) {
+            if (slots[i] >= offset + num_slots)
+                goto done;
+            status = radix_insert(&copy->anons, slots[i] - offset, batch[i]);
+            if (!status)
+                anon_ref(batch[i]);
+        }
+        next = slots[found - 1] + 1;
+    }
+done:
+    mutex_unlock(&src->lock);
+
+    if (status) {
+        amap_unref(copy);
+        return status;
+    }
+    *out = copy;
+    return 0;
+}

mem/meson.build

@@ -1,6 +1,8 @@
 kernel_sources += files(
+    'amap.c',
     'fault.c',
     'init.c',
+    'page.c',
     'pmm.c',
     'slab.c',
     'stack.c',

mem/page.c

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/park.h>
+#include <zinnia/virt.h>
+
+struct page* vm_page_from_addr(phys_t addr) {
+    return &vm_pfndb[addr / arch_mem_page_size()];
+}
+
+errno_t page_alloc(enum alloc_flags flags, struct page** out) {
+    phys_t addr;
+    errno_t status = mem_phys_alloc(1, flags, &addr);
+    if (status)
+        return status;
+
+    struct page* page = vm_page_from_addr(addr);
+    page->type = PAGE_PHYS;
+    page->obj.owner = nullptr;
+    page->obj.index = 0;
+    atomic_store(&page->flags, 0, ATOMIC_RELAXED);
+    atomic_store(&page->holds, 1, ATOMIC_RELEASE);
+    *out = page;
+    return 0;
+}
+
+void page_hold(struct page* page) {
+    atomic_fetch_add(&page->holds, 1, ATOMIC_RELAXED);
+}
+
+void page_unhold(struct page* page) {
+    if (atomic_fetch_sub(&page->holds, 1, ATOMIC_RELEASE) != 1)
+        return;
+    atomic_fence(ATOMIC_ACQUIRE);
+    atomic_store(&page->flags, 0, ATOMIC_RELAXED);
+    page->obj.owner = nullptr;
+    mem_phys_free(vm_page_addr(page), 1);
+}
+
+bool page_try_busy(struct page* page) {
+    return !(atomic_fetch_or(&page->flags, PAGE_BUSY, ATOMIC_ACQUIRE) & PAGE_BUSY);
+}
+
+void page_unbusy(struct page* page) {
+    const uint32_t old = atomic_fetch_and(&page->flags, ~(PAGE_BUSY | PAGE_WANTED), ATOMIC_RELEASE);
+    if (old & PAGE_WANTED)
+        unpark_all(&page->flags, 0);
+}
+
+static bool still_busy(void* ctx) {
+    struct page* page = ctx;
+    uint32_t flags = atomic_load(&page->flags, ATOMIC_RELAXED);
+    while (flags & PAGE_BUSY) {
+        if (atomic_compare_exchange_weak(&page->flags, &flags, flags | PAGE_WANTED, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            return true;
+    }
+    return false;
+}
+
+void page_wait(struct page* page) {
+    park(&page->flags, still_busy, page, 0);
+}