vmmcache: initial implementation

kernel-src/mm/pgcache.c

@@ -1,3 +0,0 @@
-void pgcache_init() {
-	
-}

kernel-src/mm/pmm.c

@@ -5,6 +5,7 @@
 #include <arch/mmu.h>
 #include <mutex.h>
 #include <util.h>
+#include <kernel/vmmcache.h>
 
 uintptr_t hhdmbase;
 static size_t memorysize;
@@ -101,9 +102,29 @@ static void removefromfreelist(page_t *page) {
 	--freepagecount;
 }
 
+static void internalhold(page_t *page) {
+	__atomic_add_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST);
+	if (page->refcount == 1) {
+		// this is only valid on standby pages, in case of free pages its an use after free
+		__assert((page->flags & PAGE_FLAGS_FREE) == 0);
+		removefromfreelist(page);
+	}
+}
+
+page_t *pmm_getpage(void *address) {
+	return &pages[((uintptr_t)address / PAGE_SIZE)];
+}
+
+void *pmm_getpageaddress(page_t *page) {
+	return (void *)(PAGE_GETID(page) * PAGE_SIZE);
+}
+
 void pmm_hold(void *addr) {
 	page_t *page = &pages[((uintptr_t)addr / PAGE_SIZE)];
-	__atomic_add_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST);
+
+	MUTEX_ACQUIRE(&freelistmutex, false);
+	internalhold(page);
+	MUTEX_RELEASE(&freelistmutex);
 }
 
 void pmm_release(void *addr) {
@@ -124,13 +145,14 @@ static void doalloc(page_t *page) {
 	__assert(page->refcount == 0);
 	memset(page, 0, sizeof(page_t));
 	page->refcount = 1;
-	page->flags &= ~PAGE_FLAGS_FREE;
 }
 
 void *pmm_allocpage(int section) {
+	retry:
 	MUTEX_ACQUIRE(&freelistmutex, false);
 	page_t *page = NULL;
 
+	// try to take a free anonymous page
 	for (int i = section; i >= 0; --i) {
 		page = freelists[i];
 		if (page) {
@@ -139,9 +161,34 @@ void *pmm_allocpage(int section) {
 		}
 	}
 
-	// TODO get page from standby list and call to remove from cache
+	bool cachepage = false;
+
+	// if that wasn't possible, try to take from the cache standby list
+	// TODO do this FIFO rather than LIFO
+	if (page == NULL) {
+		for (int i = section; i >= 0; --i) {
+			page = standbylists[i];
+			if (page) {
+				cachepage = true;
+				internalhold(page);
+				break;
+			}
+		}
+	}
 
 	MUTEX_RELEASE(&freelistmutex);
+
+	if (cachepage && vmmcache_takepage(page) == EAGAIN) {
+		// someone already got the page from the cache between us holding it and taking it
+		page = NULL;
+		// retry it from the start, as an anonymous page could have been released while the lock was not held
+		goto retry;
+	} else if (cachepage) {
+		// we got the page from the cache, all is good and we hold the only reference to it.
+		// set the refcount to 0, as expected by the doalloc call
+		page->refcount = 0;
+	}
+
 	void *address = NULL;
 	if (page) {
 		address = (void *)(PAGE_GETID(page) * PAGE_SIZE);
@@ -207,6 +254,8 @@ void pmm_init() {
 	MUTEX_INIT(&freelistmutex);
 }
 
+// XXX pmm_alloc won't be able to take pages from the page cache when the allocation size is over 1 page
+
 void *pmm_alloc(size_t size, int section) {
 	__assert(size);
 	// pmm_allocpage is more suited for single page allocations, so use that instead

kernel-src/mm/vmm.c

@@ -6,6 +6,7 @@
 #include <limine.h>
 #include <string.h>
 #include <kernel/slab.h>
+#include <kernel/vmmcache.h>
 
 #define RANGE_TOP(x) (void *)((uintptr_t)x->start + x->size)
 
@@ -217,9 +218,18 @@ static void destroyrange(vmmrange_t *range, uintmax_t _offset, size_t size, int
 		proc_t *proc = thread ? thread->proc : NULL;
 		cred_t *cred = proc ? &proc->cred : NULL;
 
-		if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED)) {
-			__assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
+		if ((range->flags & VMM_FLAGS_FILE) && ((range->flags & VMM_FLAGS_SHARED) || range->vnode->type == V_TYPE_CHDEV)) {
+			// shared file mapping or character device mapping
+			if (range->vnode->type == V_TYPE_CHDEV) {
+				// character device mapping
+				__assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
+			} else if (arch_mmu_iswritable(_cpu()->vmmctx->pagetable, vaddr) == false || arch_mmu_isdirty(_cpu()->vmmctx->pagetable, vaddr) == false) {
+				// dirty page cache mapping
+				vmmcache_makedirty(pmm_getpage(physical));
+				pmm_release(physical);
+			}
 		} else {
+			// anonymous or private non character device mapping
 			pmm_release(physical);
 			arch_mmu_unmap(_cpu()->vmmctx->pagetable, vaddr);
 		}
@@ -289,6 +299,15 @@ static void unmap(vmmspace_t *space, void *address, size_t size) {
 
 }
 
+static void printspace(vmmspace_t *space) {
+	printf("vmm: ranges:\n");
+	vmmrange_t *range = space->ranges;
+	while (range) {
+		printf("vmm: address %p size %lx flags %x\n", range->start, range->size, range->flags);
+		range = range->next;
+	}
+}
+
 static void *zeropage;
 
 bool vmm_pagefault(void *addr, bool user, int actions) {
@@ -335,8 +354,28 @@ bool vmm_pagefault(void *addr, bool user, int actions) {
 		// page not present in the page tables
 		if (range->flags & VMM_FLAGS_FILE) {
 			uintmax_t mapoffset = (uintptr_t)addr - (uintptr_t)range->start;
-			__assert(VOP_MMAP(range->vnode, addr, range->offset + mapoffset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
-			status = true;
+			if (range->vnode->type != V_TYPE_BLKDEV && range->vnode->type != V_TYPE_REGULAR) {
+				// map non cacheable vnodes
+				__assert(VOP_MMAP(range->vnode, addr, range->offset + mapoffset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
+				status = true;
+			} else {
+				// cacheable vnode
+				page_t *res = NULL;
+				int error = vmmcache_getpage(range->vnode, range->offset + mapoffset, &res);
+				if (error == ENXIO) {
+					// TODO if error returned == ENXIO, raise an urgent SIGBUS on the calling thread
+					__assert(!"TODO: sigbus");
+				} else if (error) {
+					printf("vmm: error on vmmcache_getpage(): %d\n", error);
+					status = false;
+				} else {
+					status = arch_mmu_map(_cpu()->vmmctx->pagetable, pmm_getpageaddress(res), addr, range->mmuflags & ~ARCH_MMU_FLAGS_WRITE);
+					if (!status) {
+						printf("vmm: out of memory to map file into address space\n");
+						pmm_release(pmm_getpageaddress(res));
+					}
+				}
+			}
 		} else {
 			// anonymous memory. map the zero'd page
 			status = arch_mmu_map(_cpu()->vmmctx->pagetable, zeropage, addr, range->mmuflags & ~ARCH_MMU_FLAGS_WRITE);
@@ -346,12 +385,16 @@ bool vmm_pagefault(void *addr, bool user, int actions) {
 				pmm_hold(zeropage);
 			}
 		}
-	} else if (arch_mmu_iswritable(_cpu()->vmmctx->pagetable, addr) == false){
+	} else if (arch_mmu_iswritable(_cpu()->vmmctx->pagetable, addr) == false) {
 		// page present but not writeable in the page tables
+		void *oldphys = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr);
 		if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED)) {
-			// TODO shared file mappings (set as dirty and set as writable on PTE)
+			arch_mmu_remap(_cpu()->vmmctx->pagetable, oldphys, addr, range->mmuflags);
+			if (range->vnode->type != V_TYPE_CHDEV)
+				vmmcache_makedirty(pmm_getpage(oldphys));
+
+			status = true;
 		} else {
-			void *oldphys = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr);
 			// do copy on write
 			void *newphys = pmm_allocpage(PMM_SECTION_DEFAULT);
 			if (newphys == NULL) {
@@ -360,7 +403,9 @@ bool vmm_pagefault(void *addr, bool user, int actions) {
 			} else {
 				memcpy(MAKE_HHDM(newphys), MAKE_HHDM(oldphys), PAGE_SIZE);
 				arch_mmu_remap(_cpu()->vmmctx->pagetable, newphys, addr, range->mmuflags);
-				pmm_release(oldphys);
+				if ((range->flags & VMM_FLAGS_FILE) == 0 || range->vnode->type != V_TYPE_CHDEV)
+					pmm_release(oldphys);
+
 				status = true;
 			}
 		}
@@ -429,6 +474,7 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags,
 	if (flags & VMM_FLAGS_FILE) {
 		// XXX make sure that writes can't happen to executable memory mapped files and check file permissions
 		vmmfiledesc_t *desc = private;
+		__assert((desc->offset % PAGE_SIZE) == 0);
 		range->vnode = desc->node;
 		range->offset = desc->offset;
 		VOP_HOLD(desc->node);
@@ -602,15 +648,6 @@ void vmm_switchcontext(vmmcontext_t *ctx) {
 	arch_mmu_switch(ctx->pagetable);
 }
 
-static void printspace(vmmspace_t *space) {
-	printf("vmm: ranges:\n");
-	vmmrange_t *range = space->ranges;
-	while (range) {
-		printf("vmm: address %p size %lx flags %x\n", range->start, range->size, range->flags);
-		range = range->next;
-	}
-}
-
 extern void *_text_start;
 extern void *_data_start;
 extern void *_rodata_start;

kernel-src/mm/vmmcache.c

@@ -0,0 +1,316 @@
+#include <kernel/vmmcache.h>
+#include <kernel/vmm.h>
+#include <util.h>
+#include <logging.h>
+
+#define TABLE_SIZE 4096
+#define WRITER_TICK_SECONDS 1
+
+static mutex_t mutex;
+static page_t **table;
+
+static thread_t *writerthread;
+static semaphore_t sync;
+
+#define HOLD_LOCK() \
+	MUTEX_ACQUIRE(&mutex, false);
+
+#define RELEASE_LOCK() \
+	MUTEX_RELEASE(&mutex);
+
+static inline uint64_t fnv1ahash(void *buffer, size_t size);
+static uintmax_t getentry(vnode_t *vnode, uintmax_t offset) {
+	struct {
+		vnode_t *vnode;
+		uintmax_t offset;
+	} tmp;
+
+	tmp.vnode = vnode;
+	tmp.offset = offset;
+
+	return fnv1ahash(&tmp, sizeof(tmp)) % TABLE_SIZE;
+}
+
+// assumes lock is held
+static page_t *findpage(vnode_t *vnode, uintmax_t offset) {
+	uintmax_t entry = getentry(vnode, offset);
+
+	page_t *page = table[entry];
+	while (page) {
+		if (page->backing == vnode && page->offset == offset)
+			break;
+		page = page->hashnext;
+	}
+
+	return page;
+}
+
+// assumes lock is held
+static void putpage(page_t *page) {
+	uintmax_t entry = getentry(page->backing, page->offset);
+	// add to table list
+	page->hashprev = NULL;
+	page->hashnext = table[entry];
+	if (table[entry])
+		table[entry]->hashprev = page;
+
+	table[entry] = page;
+
+	// add to vnode list
+	// XXX this could be a bit slow if a large portion of the file is in memory and it gets purged
+	page->vnodeprev = NULL;
+	page->vnodenext = page->backing->pages;
+	if (page->backing->pages)
+		page->backing->pages->vnodeprev = page;
+
+	page->backing->pages = page;
+}
+
+// assumes lock is held
+static void removepage(page_t *page) {
+	uintmax_t entry = getentry(page->backing, page->offset);
+
+	// remove from table list
+	if (page->hashnext)
+		page->hashnext->hashprev = page->hashprev;
+
+	if (page->hashprev)
+		page->hashprev->hashnext = page->hashnext;
+	else
+		table[entry] = page->hashnext;
+
+	page->hashnext = NULL;
+	page->hashprev = NULL;
+
+	// remove from vnode list
+	if (page->vnodenext)
+		page->vnodenext->vnodeprev = page->vnodeprev;
+
+	if (page->vnodeprev)
+		page->vnodeprev->vnodenext = page->vnodenext;
+	else
+		page->backing->pages = page->vnodenext;
+
+	page->vnodenext = NULL;
+	page->vnodeprev = NULL;
+}
+
+int vmmcache_getpage(vnode_t *vnode, uintmax_t offset, page_t **res) {
+	__assert(vnode->type == V_TYPE_REGULAR || vnode->type == V_TYPE_BLKDEV);
+	__assert((offset % PAGE_SIZE) == 0);
+	retry_err:
+	HOLD_LOCK();
+
+	page_t *newpage = NULL;
+	volatile page_t *page = findpage(vnode, offset);
+	retry:
+	if (page) {
+		// page is present in the page cache
+		pmm_hold(pmm_getpageaddress((page_t *)page));
+		RELEASE_LOCK();
+
+		// in the case of a retry, release the allocated page here
+		if (newpage)
+			pmm_release(newpage);
+
+		// wait for page to be ready
+		while ((page->flags & (PAGE_FLAGS_READY | PAGE_FLAGS_ERROR)) == 0)
+			sched_yield(); // TODO proper sleeping mechanism here
+
+		if (page->flags & PAGE_FLAGS_ERROR) {
+			// the thread handling the page in failed to read it, we should retry it and see whats up
+			pmm_release((page_t *)page);
+			goto retry_err;
+		}
+
+		*res = (page_t *)page;
+	} else {
+		// page is not present in the cache, we will have to load it in
+		RELEASE_LOCK();
+
+		void *address = pmm_allocpage(PMM_SECTION_DEFAULT);
+		if (address == NULL)
+			return ENOMEM;
+
+		newpage = pmm_getpage(address);
+		
+		HOLD_LOCK();
+
+		// while the lock wasn't being held, the page could have potentially been added to the cache
+		// check for it again and return from the function as if it was always there in the first place
+		page = findpage(vnode, offset);
+		if (page)
+			goto retry;
+
+		newpage->backing = vnode;
+		newpage->offset = offset;
+
+		// add it to the page cache
+		putpage(newpage);
+
+		RELEASE_LOCK();
+
+		int error = VOP_GETPAGE(vnode, offset, newpage);
+		// TODO wake up the sleeping threads when implemented
+		if (error) {
+			// an error happened with GETPAGE, remove the page from the cache,
+			// tell the sleeping threads that something happened and free the page
+			// by setting backing to null so it gets treated as an anonymous page again
+			HOLD_LOCK();
+			removepage(newpage);
+
+			newpage->flags |= PAGE_FLAGS_ERROR;
+			newpage->backing = NULL;
+			newpage->offset = 0;
+
+			RELEASE_LOCK();
+			pmm_release(newpage);
+			return error;
+		}
+
+		newpage->flags |= PAGE_FLAGS_READY;
+		*res = newpage;
+	}
+
+	RELEASE_LOCK();
+	return 0;
+}
+
+int vmmcache_takepage(page_t *page) {
+	HOLD_LOCK();
+	// someone called vmmcache_getpage() and got this page while the lock wasn't held
+	// return an error status to the caller
+	if (page->refcount > 1) {
+		RELEASE_LOCK();
+		return EAGAIN;
+	}
+
+	__assert(page->refcount == 1);
+	__assert((page->flags & PAGE_FLAGS_DIRTY) == 0);
+
+	// XXX maybe just not allow truncated pages to take up space like this?
+	if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
+		// the page needs to be removed from the cache to continue
+		removepage(page);
+	}
+
+	RELEASE_LOCK();
+	return 0;
+}
+
+int vmmcache_truncate(vnode_t *vnode, uintmax_t offset) {
+	HOLD_LOCK();
+	page_t *pagelist = NULL;
+	page_t *page = vnode->pages;
+
+	while (page) {
+		page_t *oldpage = page;
+		page = page->vnodenext;
+
+		// only truncate past a certain offset
+		if (oldpage->offset < offset)
+			continue;
+
+		oldpage->flags |= PAGE_FLAGS_TRUNCATED;
+		removepage(oldpage);
+		oldpage->vnodenext = pagelist;
+		pagelist = oldpage;
+	}
+
+	RELEASE_LOCK();
+
+	// make sure to unref if they are pinned
+	while (pagelist) {
+		page_t *page = pagelist;
+		pagelist = pagelist->vnodenext;
+		if (page->flags & PAGE_FLAGS_PINNED)
+			pmm_release(pmm_getpageaddress(page));
+	}
+
+	return 0;
+}
+
+int vmmcache_syncvnode(vnode_t *vnode) {
+	__assert(!"unimplemented");
+}
+
+int vmmcache_sync() {
+	semaphore_signal(&sync);
+	return 0;
+}
+
+static page_t *dirtylist;
+
+int vmmcache_makedirty(page_t *page) {
+	HOLD_LOCK();
+
+	if ((page->flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_TRUNCATED)) == 0) {
+		// page is neither dirty nor truncated, add to dirty list and hold the page and vnode
+		page->flags |= PAGE_FLAGS_DIRTY;
+
+		page->writeprev = NULL;
+		page->writenext = dirtylist;
+		if (dirtylist)
+			dirtylist->writeprev = page;
+
+		dirtylist = page;
+		pmm_hold(pmm_getpageaddress(page));
+		__assert(page->backing);
+		VOP_HOLD(page->backing);
+	}
+
+	RELEASE_LOCK();
+	return 0;
+}
+
+static void tick(context_t *, dpcarg_t arg) {
+	vmmcache_sync();
+}
+
+static void writer() {
+	timerentry_t timerentry;
+	// this will be inserted on some random cpu's timer, but it will always work after that
+	interrupt_set(false);
+	timer_insert(_cpu()->timer, &timerentry, tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true);
+	interrupt_set(true);
+	for (;;) {
+		HOLD_LOCK();
+		// TODO make FIFO rather than LIFO
+		volatile page_t *page = dirtylist;
+		if (page == NULL) {
+			RELEASE_LOCK();
+			semaphore_wait(&sync, false);
+			continue;
+		}
+
+		__assert(page->flags & PAGE_FLAGS_DIRTY);
+		page->flags &= ~PAGE_FLAGS_DIRTY;
+
+		dirtylist = page->writenext;
+		if (dirtylist)
+			dirtylist->writeprev = NULL;
+
+		page->writenext = NULL;
+
+		RELEASE_LOCK();
+		if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
+			VOP_PUTPAGE(page->backing, page->offset, (page_t *)page);
+			VOP_RELEASE(page->backing);
+		}
+
+		pmm_release(pmm_getpageaddress((page_t *)page));
+	}
+}
+
+void vmmcache_init() {
+	MUTEX_INIT(&mutex);
+	table = vmm_map(NULL, TABLE_SIZE * sizeof(page_t *), VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
+	__assert(table);
+	memset(table, 0, TABLE_SIZE * sizeof(page_t *));
+
+	SEMAPHORE_INIT(&sync, 0);
+	writerthread = sched_newthread(writer, PAGE_SIZE * 4, 1, NULL, NULL);
+	__assert(writerthread);
+	sched_queue(writerthread);
+	vmmcache_sync();
+}