kernel: initial vmm implementation

kernel-src/arch/x86-64/main.c

@@ -6,6 +6,7 @@
 #include <kernel/term.h>
 #include <kernel/pmm.h>
 #include <arch/mmu.h>
+#include <kernel/vmm.h>
 
 static cpu_t bsp_cpu;
 
@@ -19,5 +20,6 @@ void kernel_entry() {
 	term_init();
 	logging_sethook(term_putchar);
 	arch_mmu_init();
+	vmm_init();
 	__assert(!"kernel entry end");
 }

kernel-src/arch/x86-64/mmu.c

@@ -69,7 +69,7 @@ static bool add_page(pagetableptr_t top, void *vaddr, uint64_t entry, int depth)
 		pdpt = pmm_alloc(1, PMM_SECTION_DEFAULT);
 		if (pdpt == NULL)
 			return false;
-		pml4[pml4offset] = (uint64_t)FROM_HHDM(pdpt) | INTERMEDIATE_FLAGS;
+		pml4[pml4offset] = (uint64_t)pdpt | INTERMEDIATE_FLAGS;
 		pdpt = MAKE_HHDM(pdpt);
 		memset(pdpt, 0, PAGE_SIZE);
 	}

kernel-src/include/kernel/vmm.h

@@ -0,0 +1,57 @@
+#ifndef _VMM_H
+#define _VMM_H
+
+#include <arch/mmu.h>
+
+#define VMM_FLAGS_PAGESIZE 1
+#define VMM_FLAGS_ALLOCATE 2
+#define VMM_FLAGS_PHYSICAL 4
+#define VMM_FLAGS_FILE     8
+#define VMM_FLAGS_EXACT   16
+#define VMM_FLAGS_SHARED  32
+#define VMM_FLAGS_COPY    64
+
+#define VMM_PERMANENT_FLAGS_MASK (VMM_FLAGS_FILE | VMM_FLAGS_SHARED | VMM_FLAGS_COPY)
+
+struct vmmcache_t;
+typedef struct vmmrange_t{
+	struct vmmrange_t *next;
+	struct vmmrange_t *prev;
+	void *start;
+	size_t size;
+	int flags;
+	mmuflags_t mmuflags;
+	void *private;
+} vmmrange_t;
+
+typedef struct {
+	// TODO lock
+	struct vmmcache_t *next;
+	size_t freecount;
+	uintmax_t firstfree;
+} vmmcacheheader_t;
+
+#define VMM_RANGES_PER_CACHE (PAGE_SIZE - sizeof(vmmcacheheader_t)) / sizeof(vmmrange_t)
+
+typedef struct vmmcache_t {
+	vmmcacheheader_t header;
+	vmmrange_t ranges[VMM_RANGES_PER_CACHE];
+} vmmcache_t;
+
+typedef struct {
+	vmmrange_t *ranges;
+	void *start;
+	void *end;
+} vmmspace_t;
+
+typedef struct {
+	vmmspace_t space;
+	pagetableptr_t pagetable;
+} vmmcontext_t;
+
+void *vmm_map(void *addr, size_t size, int flags, mmuflags_t mmuflags, void *private);
+void vmm_unmap(void *addr, size_t size, int flags);
+vmmcontext_t *vmm_newcontext();
+void vmm_init();
+
+#endif

kernel-src/include/x86-64/arch/cpu.h

@@ -7,12 +7,14 @@
 #include <arch/msr.h>
 #include <arch/ist.h>
 #include <kernel/interrupt.h>
+#include <kernel/vmm.h>
 
 typedef struct {
 	uint64_t gdt[7];
 	ist_t ist;
 	long id;
 	isr_t isr[MAX_ISR_COUNT];
+	vmmcontext_t *vmmctx;
 } cpu_t;
 
 static inline cpu_t *_cpu() {

kernel-src/include/x86-64/arch/mmu.h

@@ -5,6 +5,11 @@
 #include <stdint.h>
 #include <stddef.h>
 
+#define KERNELSPACE_START (void *)0xffff800000000000
+#define KERNELSPACE_END   (void *)0xffffffffffffffff
+#define USERSPACE_START   (void *)0x0000000000001000
+#define USERSPACE_END     (void *)0x00007fffffffffff
+
 #define PAGE_SIZE 4096
 #define ARCH_MMU_FLAG_READ (uint64_t)1
 #define ARCH_MMU_FLAG_WRITE (uint64_t)2
@@ -20,6 +25,7 @@ void arch_mmu_unmap(pagetableptr_t table, void *vaddr);
 void arch_mmu_remap(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t flags);
 void arch_mmu_switch(pagetableptr_t table);
 void *arch_mmu_getphysical(pagetableptr_t table, void *vaddr);
+pagetableptr_t arch_mmu_newtable();
 void arch_mmu_init();
 
 #endif

kernel-src/mm/vmm.c

@@ -0,0 +1,289 @@
+#include <kernel/vmm.h>
+#include <logging.h>
+#include <kernel/pmm.h>
+#include <util.h>
+#include <arch/cpu.h>
+#include <limine.h>
+
+#define RANGE_TOP(x) (void *)((uintptr_t)x->start + x->size)
+
+static vmmcache_t *newcache() {
+	// TODO initialize lock
+	vmmcache_t *ptr = pmm_alloc(1, PMM_SECTION_DEFAULT);
+	if (ptr == NULL)
+		return NULL;
+	ptr = MAKE_HHDM(ptr);
+	ptr->header.freecount = VMM_RANGES_PER_CACHE; 
+	ptr->header.firstfree = 0;
+	ptr->header.next = NULL;
+
+	for (uintmax_t i = 0; i < VMM_RANGES_PER_CACHE; ++i) {
+		ptr->ranges[i].size = 0;
+		ptr->ranges[i].next = NULL;
+	}
+
+	return ptr;
+}
+
+// entries are allocated directly from the pmm into these caches
+static vmmcache_t *cachelist;
+
+// only supposed to be called on locked, not fully used caches
+static uintmax_t getentrynumber(vmmcache_t *cache) {
+	for (uintmax_t i = cache->header.firstfree; i < VMM_RANGES_PER_CACHE; ++i) {
+		if (cache->ranges[i].size == 0)
+			return i;
+	}
+	__assert(!"get entry number called from full cache");
+}
+
+static vmmrange_t *allocrange() {
+	vmmcache_t *cache = cachelist;
+	vmmrange_t *range = NULL;
+	while (cache) {
+		// TODO lock
+		if (cache->header.freecount > 0) {
+			--cache->header.freecount;
+			uintmax_t r = getentrynumber(cache);
+			cache->header.firstfree = r + 1;
+			range = &cache->ranges[r];
+			range->size = -1; // set as allocated temporarily
+			break;
+		} else if (cache->header.next == NULL)
+			cache->header.next = newcache();
+
+		cache = cache->header.next;
+		// TODO unlock
+	}
+
+	// TODO unlock cache
+
+	return range;
+}
+
+// ranges are separated into kernel and user. the kernel has a temporary user context
+vmmcontext_t vmm_kernelctx;
+static vmmspace_t kernelspace = {
+	.start = KERNELSPACE_START,
+	.end = KERNELSPACE_END
+};
+
+// returns a pointer to the space of vaddr
+static vmmspace_t *getspace(void *vaddr) {
+	if (USERSPACE_START <= vaddr && vaddr <= USERSPACE_END)
+		return &_cpu()->vmmctx->space;
+	else if (KERNELSPACE_START <= vaddr && vaddr <= KERNELSPACE_END)
+		return &kernelspace;
+	else
+		return NULL;
+}
+
+
+// get a range from an address
+static vmmrange_t *getrange(vmmspace_t *space, void *addr) {
+	vmmrange_t *range = space->ranges;
+	while (range) {
+		if (addr >= range->start && addr < RANGE_TOP(range))
+			break;
+		range = range->next;
+	}
+	return range;
+}
+
+// get start of range that fits specific size from specific offset
+static void *getfreerange(vmmspace_t *space, void *addr, size_t size) {
+	vmmrange_t *range = space->ranges;
+	if (addr == NULL)
+		addr = space->start;
+
+	// if theres no ranges
+	if (range == NULL)
+		return addr;
+
+	// if theres free space before the first range
+	if (range->start != space->start && addr < range->start && (uintptr_t)range->start - (uintptr_t)addr >= size)
+		return addr;
+
+	while (range->next) {
+		void *rangetop = RANGE_TOP(range);
+		if (addr < rangetop)
+			addr = rangetop;
+
+		if (addr < range->next->start) {
+			size_t freesize = (uintptr_t)range->next->start - (uintptr_t)addr;
+			if (freesize >= size)
+				return addr;
+		}
+		range = range->next;
+	}
+
+	// if theres free space after the last range
+	void *rangetop = RANGE_TOP(range);
+	if (addr < rangetop)
+		addr = rangetop;
+
+	if (addr != space->end && (uintptr_t)space->end - (uintptr_t)addr >= size)
+		return addr;
+
+	return NULL;
+}
+
+static void insertrange(vmmspace_t *space, vmmrange_t *newrange) {
+	// TODO better handle fragmentation
+	vmmrange_t *range = space->ranges;
+
+	if (range == NULL) {
+		space->ranges = newrange;
+		newrange->next = NULL;
+		newrange->prev = NULL;
+		return;
+	}
+
+	if (RANGE_TOP(newrange) < range->start) {
+		space->ranges = newrange;
+		newrange->next = range;
+		range->prev = newrange;
+		return;
+	}
+
+	while (range->next) {
+		if (newrange->start >= RANGE_TOP(range) && newrange->start < range->next->start) {
+			newrange->next = range->next;
+			newrange->prev = range;
+			range->next = newrange;
+			return;
+		}
+		range = range->next;
+	}
+
+	range->next = newrange;
+	newrange->prev = range;
+	newrange->next = NULL;
+}
+
+void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, void *private) {
+	addr = (void *)ROUND_DOWN((uintptr_t)addr, PAGE_SIZE);
+	if (flags & VMM_FLAGS_PAGESIZE)
+		size *= PAGE_SIZE;
+	else
+		size = ROUND_UP(size, PAGE_SIZE);
+
+	if (size == 0)
+		return NULL;
+
+	vmmspace_t *space = getspace(addr);
+	if (space == NULL)
+		return NULL;
+
+	// TODO lock space
+
+	void *start = getfreerange(space, addr, size);
+	void *retaddr = NULL;
+	if (((flags & VMM_FLAGS_EXACT) && start != addr) || start == NULL)
+		goto cleanup;
+
+	vmmrange_t *range = allocrange();
+	if (range == NULL)
+		goto cleanup;
+
+	range->start = start;
+	range->size = size;
+	range->flags = VMM_PERMANENT_FLAGS_MASK & flags;
+	range->mmuflags = mmuflags;
+
+	// TODO file mapping and CoW
+
+	if (flags & VMM_FLAGS_PHYSICAL) {
+		// map to allocated virtual memory
+		for (uintmax_t i = 0; i < size; i += PAGE_SIZE) {
+			if (arch_mmu_map(_cpu()->vmmctx->pagetable, (void *)((uintptr_t)private + i), (void *)((uintptr_t)start + i), mmuflags) == false) {
+				for (uintmax_t j = 0; j < size; j += PAGE_SIZE)
+					arch_mmu_unmap(_cpu()->vmmctx->pagetable, (void *)((uintptr_t)start + j));
+
+				goto cleanup;
+			}
+		}
+	} else if (flags & VMM_FLAGS_ALLOCATE) {
+		// allocate to virtual memory
+		for (uintmax_t i = 0; i < size; i += PAGE_SIZE) {
+			void *allocated = pmm_alloc(1, PMM_SECTION_DEFAULT);
+			if (arch_mmu_map(_cpu()->vmmctx->pagetable, allocated, (void *)((uintptr_t)start + i), mmuflags) == false) {
+				for (uintmax_t j = 0; j < size; j += PAGE_SIZE) {
+						void *virt = (void *)((uintptr_t)start + i);
+						void *physical = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, virt);
+
+						if (physical) {
+							pmm_free(physical, 1);
+							arch_mmu_unmap(_cpu()->vmmctx->pagetable, virt);
+						}
+				}
+
+				goto cleanup;
+			}
+		}
+	}
+
+	insertrange(space, range);
+
+	retaddr = start;
+
+	cleanup:
+	if (start == NULL && range) {
+		// TODO free range
+	}
+	// TODO unlock
+	return retaddr;
+}
+
+void vmm_unmap(void *addr, size_t size, int flags);
+
+vmmcontext_t *vmm_newcontext() {
+	__assert(!"not yet");
+}
+
+void vmm_switchcontext(vmmcontext_t *ctx) {
+	arch_mmu_switch(ctx->pagetable);
+	_cpu()->vmmctx = ctx;
+}
+
+static void printspace(vmmspace_t *space) {
+	printf("vmm: ranges:\n");
+	vmmrange_t *range = space->ranges;
+	while (range) {
+		printf("vmm: address %p size %p flags %p\n", range->start, range->size, range->flags);
+		range = range->next;
+	}
+}
+
+extern void *_text_start;
+extern void *_data_start;
+extern void *_rodata_start;
+extern void *_text_end;
+extern void *_data_end;
+extern void *_rodata_end;
+
+extern volatile struct limine_memmap_request pmm_liminemap;
+
+void vmm_init() {
+	__assert(sizeof(vmmcache_t) <= PAGE_SIZE);
+
+	cachelist = newcache();
+	vmm_kernelctx.pagetable = arch_mmu_newtable();
+	__assert(cachelist && vmm_kernelctx.pagetable);
+
+	vmm_kernelctx.space.start = USERSPACE_START;
+	vmm_kernelctx.space.end = USERSPACE_END;
+
+	vmm_switchcontext(&vmm_kernelctx);
+
+	for (uint64_t i = 0; i < pmm_liminemap.response->entry_count; ++i) {
+		struct limine_memmap_entry *e = pmm_liminemap.response->entries[i];
+		__assert(vmm_map(MAKE_HHDM(e->base), e->length, 0, ARCH_MMU_FLAG_READ | ARCH_MMU_FLAG_WRITE | ARCH_MMU_FLAG_NOEXEC, NULL));
+	}
+
+	__assert(vmm_map(&_text_start, (uintptr_t)&_text_end - (uintptr_t)&_text_start, 0, ARCH_MMU_FLAG_READ, NULL));
+	__assert(vmm_map(&_rodata_start, (uintptr_t)&_rodata_end - (uintptr_t)&_rodata_start, 0, ARCH_MMU_FLAG_READ | ARCH_MMU_FLAG_NOEXEC, NULL));
+	__assert(vmm_map(&_data_start, (uintptr_t)&_data_end - (uintptr_t)&_data_start, 0, ARCH_MMU_FLAG_READ | ARCH_MMU_FLAG_WRITE | ARCH_MMU_FLAG_NOEXEC, NULL));
+
+	printspace(&kernelspace);
+}