page cache rewrite

kernel-src/fs/abc.c

@@ -301,4 +301,4 @@ static void abc_init_routine(void) {
 	__assert(abc_block_cache);
 }
 
-INIT_ROUTINE_DEFINE(abc, INIT_ROUTINE_FLAGS_NONE, abc_init_routine, scheduler, trie);
+INIT_ROUTINE_DEFINE(abc, INIT_ROUTINE_FLAGS_NONE, abc_init_routine, mm_cache, scheduler, trie);

kernel-src/fs/ext2.c

@@ -1822,7 +1822,6 @@ static int ext2_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 static int ext2_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 	// only regular files get cached
 	__assert(node->type == V_TYPE_REGULAR);
-	size_t writec;
 	iovec_t iovec = {
 		.addr = MAKE_HHDM(mm_get_page_address(page)),
 		.len = PAGE_SIZE
@@ -1830,21 +1829,8 @@ static int ext2_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 
 	iovec_iterator_t iovec_iterator;
 	iovec_iterator_init(&iovec_iterator, &iovec, 1);
-	int error = VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &writec, NULL);
-
-	// its possible that writec is zero here. the condition where this is possible is when
-	// the file is truncated after the check in the page sync function but before it actually is
-	// written back to disk. therefore, we check that the page IS infact truncated if this happens.
-	// if it isn't, something else happened and its not safe to continue.
-	//
-	// TODO: since this check will likely be copied between different filesystem drivers, it could be interesting to have
-	// ext2_putpage take in a (size_t *) pointer and have this check be in whatever function calls VOP_PUTPAGE.
-
-	if (writec == 0) {
-		__assert(page->flags & PAGE_FLAGS_TRUNCATED);
-	}
-
-	return error;
+	size_t writec;
+	return VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &writec, NULL);
 }
 
 static int ext2_rename(vnode_t *sourcedir, vnode_t *source, char *oldname, vnode_t *targetdir, char *newname, int flags) {
@@ -1964,13 +1950,11 @@ static int ext2_rename(vnode_t *sourcedir, vnode_t *source, char *oldname, vnode
 }
 
 static int ext2_sync(vnode_t *vnode) {
-	int e = mm_cache_sync_vnode(vnode, 0, UINT64_MAX);
+	int e = mm_cache_sync_vnode(vnode);
 	ext2fs_t *fs = (ext2fs_t *)vnode->vfs;
 	abc_sync(&fs->abc);
 	// TODO don't sync the entire disk but rather only the inodes and blocks
-	VOP_LOCK(fs->backing);
-	int e2 = mm_cache_sync_vnode(fs->backing, 0, UINT64_MAX);
-	VOP_UNLOCK(fs->backing);
+	int e2 = mm_cache_sync_vnode(fs->backing);
 	// only the first errors are reported
 	return e ? e : e2;
 }
@@ -2007,7 +1991,6 @@ static int ext2_inactive(vnode_t *vnode) {
 		__assert(hashtable_remove(&fs->inodetable, &node->id, sizeof(node->id)) == 0);
 		MUTEX_RELEASE(&fs->inodetablelock);
 
-		mm_cache_truncate(vnode, 0);
 		freeinode(fs, &node->inode, node->id);
 
 		slab_free(nodecache, node);

kernel-src/fs/fat/fatfs.c

@@ -200,9 +200,7 @@ static int fatfs_unmount(vfs_t *vfs) {
 static int fatfs_sync(vfs_t *vfs) {
 	fatfs_t *fatfs = (fatfs_t *)vfs;
 	abc_sync(&fatfs->abc);
-	VOP_LOCK(fatfs->backing);
-	int error = mm_cache_sync_vnode(fatfs->backing, 0, UINT64_MAX);
-	VOP_UNLOCK(fatfs->backing);
+	int error = mm_cache_sync_vnode(fatfs->backing);
 	return error;
 }
 
@@ -264,19 +262,16 @@ static int fatfs_syncvnode(vnode_t *vnode) {
 	fatnode_t *fatnode = (fatnode_t *)vnode;
 
 	// sync file data
-	int e = mm_cache_sync_vnode(vnode, 0, UINT64_MAX);
+	int e = mm_cache_sync_vnode(vnode);
 	abc_sync(&fs->abc);
 
-	VOP_LOCK(fs->backing);
 	// sync the dent
 	int e2 = 0;
 	if (fatnode->dent_disk_offset)
-		e2 = mm_cache_sync_vnode(fs->backing, fatnode->dent_disk_offset, sizeof(fatfs_dent_t));
+		e2 = mm_cache_sync_vnode(fs->backing);
 
 	// and sync the fats
-	int e3 = mm_cache_sync_vnode(fs->backing, fs->fat_offset, fs->fat_size * fs->fat_count);
-
-	VOP_UNLOCK(fs->backing);
+	int e3 = mm_cache_sync_vnode(fs->backing);
 
 	// only the first errors are reported
 	if (e)
@@ -391,7 +386,6 @@ static int fatfs_read(vnode_t *vnode, iovec_iterator_t *iovec_iterator, size_t s
 static int fatfs_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 	// only regular files get cached
 	__assert(node->type == V_TYPE_REGULAR);
-	size_t write_count;
 	iovec_t iovec = {
 		.addr = MAKE_HHDM(mm_get_page_address(page)),
 		.len = PAGE_SIZE
@@ -399,23 +393,11 @@ static int fatfs_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 
 	iovec_iterator_t iovec_iterator;
 	iovec_iterator_init(&iovec_iterator, &iovec, 1);
-	int error = VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &write_count, NULL);
-
-	// its possible that write_count is zero here. the condition where this is possible is when
-	// the file is truncated after the check in the page sync function but before it actually is
-	// written back to disk. therefore, we check that the page IS infact truncated if this happens.
-	// if it isn't, something else happened and its not safe to continue.
-	//
-	// TODO: since this check will likely be copied between different filesystem drivers, it could be interesting to have
-	// fatfs_putpage take in a (size_t *) pointer and have this check be in whatever function calls VOP_PUTPAGE.
-
-	if (write_count == 0) {
-		__assert(page->flags & PAGE_FLAGS_TRUNCATED);
-	}
-
-	return error;
+	size_t write_count;
+	return VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &write_count, NULL);
 }
 
+
 static int fatfs_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 	// only regular files get cached
 	__assert(node->type == V_TYPE_REGULAR);
@@ -970,7 +952,6 @@ static int fatfs_inactive(vnode_t *vnode) {
 		__assert(!"fatfs VOP_INACTIVE() with hardlink count >0 is currently unimplemented");
 	} else {
 		// unlinked file having resources released when kernel refcount hits zero
-		mm_cache_truncate(vnode, 0);
 		fatfs_resize_file(fatfs, fatnode, 0);
 	}
 

kernel-src/fs/tmpfs.c

@@ -224,6 +224,36 @@ static int tmpfs_root(vfs_t *vfs, vnode_t **vnode) {
 	return 0;
 }
 
+static int tmpfs_rw(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, size_t *done, bool write) {
+	*done = 0;
+	while (*done < size) {
+		uintmax_t current_offset = offset + *done;
+		uintmax_t page_offset = current_offset / PAGE_SIZE * PAGE_SIZE;
+		size_t page_start = current_offset % PAGE_SIZE;
+		size_t copy_size = min(PAGE_SIZE - page_start, size - *done);
+
+		page_t *page;
+		int error = mm_cache_get_page(node, page_offset, 0, &page);
+		if (error)
+			return error;
+
+		void *address = MAKE_HHDM(mm_get_page_address(page));
+		if (write)
+			error = iovec_iterator_copy_to_buffer(iovec_iterator, (void *)((uintptr_t)address + page_start), copy_size);
+		else
+			error = iovec_iterator_copy_from_buffer(iovec_iterator, (void *)((uintptr_t)address + page_start), copy_size);
+		mm_release_page(FROM_HHDM(address));
+		if (error)
+			return error;
+
+		*done += copy_size;
+	}
+
+	if (write && *done)
+		((tmpfsnode_t *)node)->attr.mtime = timekeeper_time();
+	return 0;
+}
+
 static int tmpfs_read(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, int flags, size_t *readc, cred_t *cred) {
 	if (node->type == V_TYPE_DIR)
 		return EISDIR;
@@ -231,7 +261,13 @@ static int tmpfs_read(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t si
 	if (node->type != V_TYPE_REGULAR)
 		return EINVAL;
 
-	__assert(!"tmpfs_read is handled by the page cache");
+	tmpfsnode_t *tmpfsnode = (tmpfsnode_t *)node;
+	if (offset >= tmpfsnode->attr.size) {
+		*readc = 0;
+		return 0;
+	}
+	size = min(size, tmpfsnode->attr.size - offset);
+	return tmpfs_rw(node, iovec_iterator, size, offset, readc, false);
 }
 
 static int tmpfs_write(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, int flags, size_t *writec, cred_t *cred) {
@@ -241,7 +277,10 @@ static int tmpfs_write(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t s
 	if (node->type != V_TYPE_REGULAR)
 		return EINVAL;
 
-	__assert(!"tmpfs_read is handled by the page cache");
+	if (size + offset < offset)
+		return EOVERFLOW;
+	__assert(size + offset <= ((tmpfsnode_t *)node)->attr.size);
+	return tmpfs_rw(node, iovec_iterator, size, offset, writec, true);
 }
 
 static int tmpfs_access(vnode_t *vnode, mode_t mode, cred_t *cred) {
@@ -501,9 +540,6 @@ static int tmpfs_inactive(vnode_t *node) {
 	arch_e9_puts("tmpfs_inactive\n");
 	tmpfsnode_t *tmp_node = (tmpfsnode_t *)node;
 	__assert(tmp_node->attr.nlinks == (node->type == V_TYPE_DIR ? 1 : 0));
-	if (node->type == V_TYPE_REGULAR) {
-		mm_cache_truncate(node, 0);
-	}
 	freenode(tmp_node);
 	return 0;
 }
@@ -535,7 +571,7 @@ static int tmpfs_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
 	void *phyhhdm = MAKE_HHDM(phy);
 
 	mm_hold_page(phy);
-	page->flags |= PAGE_FLAGS_PINNED;
+	__atomic_fetch_or(&page->flags, PAGE_FLAGS_PINNED, __ATOMIC_RELAXED);
 	memset(phyhhdm, 0, PAGE_SIZE);
 	return 0;
 }

kernel-src/fs/vfs.c

@@ -152,6 +152,9 @@ int vfs_sync(void) {
 }
 
 void vfs_inactive(vnode_t *vnode) {
+	if (vfs_iscacheable(vnode))
+		mm_cache_truncate(vnode, 0);
+
 	if (vnode->type == V_TYPE_SOCKET && vnode->socketbinding) {
 		localsock_leavebinding(vnode);
 	} else if (vnode->type == V_TYPE_FIFO && vnode->fifobinding) {
@@ -438,33 +441,14 @@ static void bytestopages(uintmax_t offset, size_t size, uintmax_t *pageoffset, s
 	*pagecount = toppage - *pageoffset;
 }
 
-static int writenocache(vnode_t *node, page_t *page, uintmax_t pageoffset) {
-	// don't keep the pages in the cache!
-	// if its a file, do the proper filesystem sync.
-	// if its a block device, sync only the page we just dirtied
-	VOP_LOCK(node);
-	int e;
-	if (node->type == V_TYPE_REGULAR)
-		e = VOP_SYNC(node);
-	else
-		e = mm_cache_sync_vnode(node, pageoffset, PAGE_SIZE);
-	VOP_UNLOCK(node);
-
-	// try to turn it into anonymous memory
-	mm_cache_evict(page);
-
-	return e;
-}
-
 int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, size_t *written, int flags) {
 	int err = 0;
-	if (vfs_iscacheable(node)) {
+	bool cacheable = vfs_iscacheable(node);
+	if (cacheable) {
 		*written = 0;
-		// can't write a size 0 buffer
 		if (size == 0)
 			return 0;
 
-		// overflow
 		if (size + offset < offset)
 			return EINVAL;
 
@@ -502,7 +486,9 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size
 
 			size = min(size + offset, bytesize) - offset;
 		}
+	}
 
+	if (cacheable && (flags & V_FFLAGS_NOCACHE) == 0) {
 		uintmax_t pageoffset, pagecount, startoffset;
 		bytestopages(offset, size, &pageoffset, &pagecount, &startoffset);
 		page_t *page = NULL;
@@ -525,15 +511,10 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size
 			mm_cache_make_dirty(page);
 			*written += writesize;
 
-			if (flags & V_FFLAGS_NOCACHE)
-				err = writenocache(node, page, pageoffset * PAGE_SIZE);
-
 			pageoffset += 1;
 			pagecount -= 1;
 
 			mm_release_page(FROM_HHDM(address));
-			if (err)
-				goto leave;
 		}
 
 		for (uintmax_t offset = 0; offset < pagecount * PAGE_SIZE; offset += PAGE_SIZE) {
@@ -554,30 +535,30 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size
 			mm_cache_make_dirty(page);
 			*written += writesize;
 
-			if (flags & V_FFLAGS_NOCACHE)
-				err = writenocache(node, page, pageoffset * PAGE_SIZE + offset);
-
 			mm_release_page(FROM_HHDM(address));
-
-			if (err)
-				goto leave;
 		}
 
-		leave:
-		MUTEX_RELEASE(&node->size_lock);
+		if (flags & V_FFLAG_MUST_SYNC)
+			err = mm_cache_sync_vnode(node);
+
 	} else {
-		// special file, just write as its not being cached
+		// Non-cacheable objects and direct I/O bypass the page cache.
 		VOP_LOCK(node);
 		err = VOP_WRITE(node, iovec_iterator, size, offset, flags, written, getcred());
 		VOP_UNLOCK(node);
 	}
 
+	leave:
+	if (cacheable)
+		MUTEX_RELEASE(&node->size_lock);
+
 	return err;
 }
 
 int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, size_t *bytesread, int flags) {
 	int err = 0;
-	if (vfs_iscacheable(node)) {
+	bool cacheable = vfs_iscacheable(node);
+	if (cacheable) {
 		*bytesread = 0;
 		// can't read 0 bytes from the cache
 		if (size == 0)
@@ -615,7 +596,9 @@ int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size,
 			goto leave;
 
 		size = min(size + offset, nodesize) - offset;
+	}
 
+	if (cacheable && (flags & V_FFLAGS_NOCACHE) == 0) {
 		uintmax_t pageoffset, pagecount, startoffset;
 		bytestopages(offset, size, &pageoffset, &pagecount, &startoffset);
 		page_t *page = NULL;
@@ -637,11 +620,6 @@ int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size,
 
 			*bytesread += readsize;
 
-			if (flags & V_FFLAGS_NOCACHE) {
-				// try to turn it into anonymous memory
-				mm_cache_evict(page);
-			}
-
 			pageoffset += 1;
 			pagecount -= 1;
 
@@ -664,20 +642,18 @@ int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size,
 			}
 
 			*bytesread += readsize;
-			if (flags & V_FFLAGS_NOCACHE) {
-				// try to turn it into anonymous memory
-				mm_cache_evict(page);
-			}
 			mm_release_page(FROM_HHDM(address));
 		}
-		leave:
-		MUTEX_RELEASE(&node->size_lock);
 	} else {
-		// special file, just read as size doesn't matter
+		// Non-cacheable objects and direct I/O bypass the page cache.
 		VOP_LOCK(node);
 		err = VOP_READ(node, iovec_iterator, size, offset, flags, bytesread, getcred());
 		VOP_UNLOCK(node);
 	}
+
+	leave:
+	if (cacheable)
+		MUTEX_RELEASE(&node->size_lock);
 	return err;
 }
 

kernel-src/include/kernel/file.h

@@ -73,8 +73,10 @@ static inline int fileflagstovnodeflags(int flags) {
 		vnflags |= V_FFLAGS_NONBLOCKING;
 	if (flags & O_NOCTTY)
 		vnflags |= V_FFLAGS_NOCTTY;
-	if (flags & (O_DIRECT | O_SYNC | O_DSYNC))
+	if (flags & O_DIRECT)
 		vnflags |= V_FFLAGS_NOCACHE;
+	if (flags & (O_SYNC | O_DSYNC))
+		vnflags |= V_FFLAG_MUST_SYNC;
 
 	return vnflags;
 }

kernel-src/include/kernel/mm.h

@@ -99,15 +99,14 @@ void mm_init();
 
 extern size_t mm_cache_cached_pages;
 
-void mm_cache_init(void);
+#define MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE 1
 int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res);
 int mm_cache_take_page(page_t *page);
-int mm_cache_make_dirty(page_t *page);
+void mm_cache_make_dirty(page_t *page);
 int mm_cache_truncate(vnode_t *vnode, uintmax_t offset);
-int mm_cache_sync_vnode(vnode_t *vnode, uintmax_t startoffset, size_t size);
-int mm_cache_push_page(vnode_t *vnode, uintmax_t offset, page_t *page);
+int mm_cache_sync_vnode(vnode_t *vnode);
 int mm_cache_sync(void);
-int mm_cache_evict(page_t *page);
+void mm_cache_init_writer(void);
 
 void mm_range_init(void);
 mm_range_t *mm_alloc_range(void);

kernel-src/include/kernel/page.h

@@ -4,6 +4,7 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <stdbool.h>
+#include <list.h>
 
 #define MEMORY_SECTION_COUNT 3
 #define MEMORY_SECTION_1MB 0
@@ -16,24 +17,17 @@
 #define PAGE_FLAGS_DIRTY 8
 #define PAGE_FLAGS_READY 16
 #define PAGE_FLAGS_ERROR 32
-#define PAGE_FLAGS_VNODE_SYNCING 64
+#define PAGE_FLAGS_SYNCING 64
 
 typedef struct page_t {
 	struct vnode_t *backing;
 	uintmax_t offset;
-	struct page_t *hash_next;
-	struct page_t *hash_prev;
-	struct page_t *vnode_next;
-	struct page_t *vnode_prev;
 	union {
 		struct {
 			struct page_t *free_next;
 			struct page_t *free_prev;
 		};
-		struct {
-			struct page_t *write_next;
-			struct page_t *write_prev;
-		};
+		list_node_t dirty_list_node;
 	};
 	uintmax_t refcount;
 	uintmax_t lock_count;

kernel-src/include/kernel/vfs.h

@@ -10,7 +10,10 @@
 #include <errno.h>
 #include <kernel/cred.h>
 #include <kernel/event.h>
+#include <trie.h>
 #include <errno.h>
+#include <kernel/page.h>
+#include <list.h>
 
 #define V_ATTR_MODE	1
 #define V_ATTR_UID	2
@@ -120,6 +123,7 @@ typedef struct advlock_t {
 #define V_FFLAGS_NOCTTY 32
 #define V_FFLAGS_NOCACHE 64
 #define V_FFLAGS_CLOSE_URGENT 128
+#define V_FFLAG_MUST_SYNC 256
 
 typedef struct vnode_t {
 	struct vops_t *ops;
@@ -135,7 +139,14 @@ typedef struct vnode_t {
 		void *fifobinding;
 	};
 
-	struct page_t *pages;
+	bool dirty;
+	mutex_t writeback_mutex;
+	mutex_t dirty_list_mutex;
+	list_node_t vnode_dirty_list_node;
+	list_t dirty_pages;
+
+	pushlock_t pages_lock;
+	trie_t pages;
 
 	mutex_t adv_mutex;
 	advlock_t *advlock;
@@ -177,6 +188,7 @@ typedef struct vops_t {
 	int (*rename)(vnode_t *sourcedir, vnode_t *source, char *oldname, vnode_t *targetdir, char *newname, int flags);
 	int (*getpage)(vnode_t *node, uintmax_t offset, struct page_t *page);
 	int (*putpage)(vnode_t *node, uintmax_t offset, struct page_t *page);
+	// node must be unlocked on entry.
 	int (*sync)(vnode_t *node);
 	int (*advlock)(vnode_t *node, int op, advlock_t *advlock);
 	int (*lock)(vnode_t *node);
@@ -208,7 +220,12 @@ typedef struct vops_t {
 	(vn)->type = t; \
 	(vn)->vfs = v; \
 	(vn)->vfsmounted = NULL; \
-        (vn)->pages = NULL;
+	trie_init(&(vn)->pages); \
+	(vn)->pages_lock = 0; \
+	MUTEX_INIT(&(vn)->writeback_mutex); \
+	MUTEX_INIT(&(vn)->dirty_list_mutex); \
+	(vn)->dirty = false; \
+	list_init(&(vn)->dirty_pages);
 
 #define VOP_LOCK(v) (v)->ops->lock(v)
 #define VOP_UNLOCK(v) (v)->ops->unlock(v)

kernel-src/include/trie.h

@@ -39,7 +39,7 @@ int trie_insert_preallocated(trie_t *trie, uint64_t key, void *value, trie_preal
 void trie_free_preallocation(trie_preallocation_t *preallocation);
 int trie_lookup(trie_t *trie, uint64_t key, void **ret);
 int trie_remove(trie_t *trie, uint64_t key);
-void trie_truncate(trie_t *trie, uint64_t max_key);
+void trie_truncate(trie_t *trie, uint64_t min_key);
 void trie_iterate(trie_t *trie, uint64_t min_key, uint64_t max_key, trie_iterate_fn_t);
 
 #endif

kernel-src/lib/trie.c

@@ -285,13 +285,13 @@ static void trie_recursively_free(trie_node_t *trie_node, uint8_t height) {
 	slab_free(trie_node_cache, trie_node);
 }
 
-static void trie_truncate_internal(trie_node_t *trie_node, uint64_t max_key, uint8_t height) {
-	uint8_t offset = get_offset(max_key, height);
+static void trie_truncate_internal(trie_node_t *trie_node, uint64_t min_key, uint8_t height) {
+	uint8_t offset = get_offset(min_key, height);
 
 	if (height) {
 		trie_node_t *child_node = trie_node->nodes[offset];
 		if (child_node) {
-			trie_truncate_internal(child_node, max_key, height - 1);
+			trie_truncate_internal(child_node, min_key, height - 1);
 
 			if (child_node->count == 0) {
 				slab_free(trie_node_cache, child_node);
@@ -313,11 +313,8 @@ static void trie_truncate_internal(trie_node_t *trie_node, uint64_t max_key, uin
 	}
 }
 
-void trie_truncate(trie_t *trie, uint64_t max_key) {
-	if (key_height(max_key) > trie->height)
-		return;
-
-	if (max_key == 0) {
+void trie_truncate(trie_t *trie, uint64_t min_key) {
+	if (min_key == 0) {
 		if (trie->height)
 			trie_recursively_free(trie->root, trie->height - 1);
 
@@ -326,20 +323,10 @@ void trie_truncate(trie_t *trie, uint64_t max_key) {
 		return;
 	}
 
-	if (max_key == 1) {
-		void *item;
-		if (trie_lookup(trie, 0, &item))
-			item = NULL;
-
-		if (trie->height)
-			trie_recursively_free(trie->root, trie->height - 1);
-
-		trie->root_item = item;
-		trie->height = 0;
+	if (trie->height == 0 || key_height(min_key) > trie->height)
 		return;
-	}
 
-	trie_truncate_internal(trie->root, max_key, trie->height - 1);
+	trie_truncate_internal(trie->root, min_key, trie->height - 1);
 
 	trie_trim(trie);
 }

kernel-src/mm/cache.c

@@ -1,503 +1,220 @@
 #include <kernel/mm.h>
-#include <util.h>
+#include <semaphore.h>
 #include <logging.h>
-#include <kernel/timekeeper.h>
-#include <kernel/event.h>
 #include <kernel/init.h>
 
-#define TABLE_SIZE 4096
-#define WRITER_TICK_SECONDS 15
+typedef struct {
+	list_node_t list_node;
+	semaphore_t semaphore;
+	page_t *page;
+} page_wait_block_t;
 
-static mutex_t mutex;
-static page_t **table;
+typedef struct {
+	mutex_t mutex;
+	list_t wait_block_list;
+} page_wait_block_header_t;
 
-static thread_t *writer_thread;
-static semaphore_t sync;
-static eventheader_t sync_event;
-static eventheader_t page_ready_event;
+#define WAIT_BLOCK_COUNT 128
+
+static page_wait_block_header_t wait_blocks[WAIT_BLOCK_COUNT];
 size_t mm_cache_cached_pages;
 
-#define HOLD_LOCK() \
-	MUTEX_ACQUIRE(&mutex);
+static page_wait_block_header_t *page_wait_pick_header(page_t *page) {
+	return &wait_blocks[(uintptr_t)mm_get_page_address(page) / PAGE_SIZE % WAIT_BLOCK_COUNT]; // TODO: maybe use a hash
+}
 
-#define RELEASE_LOCK() \
-	MUTEX_RELEASE(&mutex);
+static void page_wait_attach(page_t *page, page_wait_block_t *page_wait_block) {
+	page_wait_block_header_t *header = page_wait_pick_header(page);
 
-static inline uint64_t fnv1a_hash(void *buffer, size_t size) {
-	uint8_t *ptr = buffer;
-	uint8_t *top = ptr + size;
-	uint64_t h = FNV1OFFSET;
+	SEMAPHORE_INIT(&page_wait_block->semaphore, 0);
+	page_wait_block->page = page;
 
-	while (ptr < top) {
-		h ^= *ptr++;
-		h *= FNV1PRIME;
+	MUTEX_ACQUIRE(&header->mutex);
+	list_push_back(&header->wait_block_list, &page_wait_block->list_node);
+	MUTEX_RELEASE(&header->mutex);
+}
+
+static void page_wait_detach(page_wait_block_t *page_wait_block) {
+	page_wait_block_header_t *header = page_wait_pick_header(page_wait_block->page);
+
+	MUTEX_ACQUIRE(&header->mutex);
+	list_remove(&header->wait_block_list, &page_wait_block->list_node);
+	MUTEX_RELEASE(&header->mutex);
+}
+
+static void page_wait(page_wait_block_t *page_wait_block) {
+	semaphore_wait(&page_wait_block->semaphore, false);
+}
+
+static void page_notify_waiters(page_t *page) {
+	page_wait_block_header_t *header = page_wait_pick_header(page);
+
+	MUTEX_ACQUIRE(&header->mutex);
+
+	list_for_each(&header->wait_block_list, list_node) {
+		page_wait_block_t *block = (page_wait_block_t *)list_node;
+
+		if (block->page == page)
+			semaphore_signal(&block->semaphore);
 	}
 
-	return h;
+	MUTEX_RELEASE(&header->mutex);
 }
 
-static uintmax_t get_entry(vnode_t *vnode, uintmax_t offset) {
-	struct {
-		vnode_t *vnode;
-		uintmax_t offset;
-	} tmp;
-
-	tmp.vnode = vnode;
-	tmp.offset = offset;
-
-	return fnv1a_hash(&tmp, sizeof(tmp)) % TABLE_SIZE;
+// expects vnode pushlock with shared+ acquire
+static page_t *lookup_page(vnode_t *vnode, uintmax_t offset) {
+	void *v;
+	return trie_lookup(&vnode->pages, offset / PAGE_SIZE, &v) ? NULL : v;
 }
 
-// assumes lock is held
-static page_t *find_page(vnode_t *vnode, uintmax_t offset) {
-	uintmax_t entry = get_entry(vnode, offset);
+// expects vnode pushlock with exclusive acquire
+static int insert_page(vnode_t *vnode, uintmax_t offset, page_t *page, trie_preallocation_t *preallocation) {
+	int error = trie_insert_preallocated(&vnode->pages, offset / PAGE_SIZE, page, preallocation);
+	if (error == 0)
+		__atomic_add_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED);
+	return error;
+}
 
-	page_t *page = table[entry];
-	while (page) {
-		if (page->backing == vnode && page->offset == offset)
-			break;
-		page = page->hash_next;
+static void remove_page(vnode_t *vnode, uintmax_t offset) {
+	__assert(trie_remove(&vnode->pages, offset / PAGE_SIZE) == 0);
+	__atomic_sub_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED);
+}
+
+static int wait_for_ready(page_t *page) {
+	for (;;) {
+		int flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE);
+		if (flags & PAGE_FLAGS_READY) {
+			if ((flags & PAGE_FLAGS_ERROR) == 0)
+				return 0;
+
+			void *phys = mm_get_page_address(page);
+			mm_release_page(phys);
+			return EIO;
+		}
+
+		page_wait_block_t page_wait_block;
+		page_wait_attach(page, &page_wait_block);
+		if (__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_READY) {
+			page_wait_detach(&page_wait_block);
+			continue;
+		}
+
+		page_wait(&page_wait_block);
+		page_wait_detach(&page_wait_block);
 	}
-
-	return page;
-}
-
-// assumes lock is held
-static void put_page(page_t *page) {
-	uintmax_t entry = get_entry(page->backing, page->offset);
-	// add to table list
-	page->hash_prev = NULL;
-	page->hash_next = table[entry];
-	if (table[entry])
-		table[entry]->hash_prev = page;
-
-	table[entry] = page;
-
-	// add to vnode list
-	// XXX this could be a bit slow if a large portion of the file is in memory and it gets purged
-	page->vnode_prev = NULL;
-	page->vnode_next = page->backing->pages;
-	if (page->backing->pages)
-		page->backing->pages->vnode_prev = page;
-
-	page->backing->pages = page;
-	++mm_cache_cached_pages;
-}
-
-// assumes lock is held
-static void remove_page(page_t *page) {
-	uintmax_t entry = get_entry(page->backing, page->offset);
-
-	// remove from table list
-	if (page->hash_next)
-		page->hash_next->hash_prev = page->hash_prev;
-
-	if (page->hash_prev)
-		page->hash_prev->hash_next = page->hash_next;
-	else
-		table[entry] = page->hash_next;
-
-	page->hash_next = NULL;
-	page->hash_prev = NULL;
-
-	// remove from vnode list
-	if (page->vnode_next)
-		page->vnode_next->vnode_prev = page->vnode_prev;
-
-	if (page->vnode_prev)
-		page->vnode_prev->vnode_next = page->vnode_next;
-	else
-		page->backing->pages = page->vnode_next;
-
-	page->vnode_next = NULL;
-	page->vnode_prev = NULL;
-	--mm_cache_cached_pages;
 }
 
+// TODO: clustered page in
 int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res) {
-	(void) flags;
-	__assert(vnode->type == V_TYPE_REGULAR || vnode->type == V_TYPE_BLKDEV);
-	__assert((offset % PAGE_SIZE) == 0);
-	retry_err:
-	HOLD_LOCK();
+	pushlock_acquire_shared(&vnode->pages_lock);
 
-	page_t *new_page = NULL;
-	volatile page_t *page = find_page(vnode, offset);
-	retry:
+	page_t *page = lookup_page(vnode, offset);
+	if (page)
+		mm_hold_page(mm_get_page_address(page));
+
+	pushlock_release_shared(&vnode->pages_lock);
+
+retry_page:
 	if (page) {
-		// page is present in the page cache
-		mm_hold_page(mm_get_page_address((page_t *)page));
-		RELEASE_LOCK();
+		*res = page;
+		return wait_for_ready(page);
+	} else if (flags & MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE)
+		return ENOENT;
 
-		// in the case of a retry, release the allocated page here
-		if (new_page)
-			mm_release_page(mm_get_page_address(new_page));
+	void *phys = mm_alloc_page(MEMORY_SECTION_DEFAULT);
+	if (phys == NULL)
+		return ENOMEM;
 
-		eventlistener_t listener;
-		EVENT_INITLISTENER(&listener);
-		EVENT_ATTACH(&listener, &page_ready_event);
+	page = mm_get_page(phys);
 
-		// wait for page to be ready
-		while ((page->flags & (PAGE_FLAGS_READY | PAGE_FLAGS_ERROR)) == 0)
-			EVENT_WAIT(&listener, 0);
+	trie_preallocation_t *trie_preallocation = trie_preallocate();
+	if (trie_preallocation == NULL) {
+		mm_release_page(phys);
+		return ENOMEM;
+	}
 
-		EVENT_DETACHALL(&listener);
+	pushlock_acquire_exclusive(&vnode->pages_lock);
+	page_t *tmp = lookup_page(vnode, offset);
+	if (tmp) {
+		mm_hold_page(mm_get_page_address(tmp));
+		pushlock_release_exclusive(&vnode->pages_lock);
+		trie_free_preallocation(trie_preallocation);
+		mm_release_page(phys);
+		page = tmp;
+		goto retry_page;
+	}
 
-		if (page->flags & PAGE_FLAGS_ERROR) {
-			// the thread handling the page in failed to read it, we should retry it and see whats up
-			mm_release_page(mm_get_page_address((page_t *)page));
-			goto retry_err;
-		}
+	__assert(insert_page(vnode, offset, page, trie_preallocation) == 0);
+	pushlock_release_exclusive(&vnode->pages_lock);
 
-		*res = (page_t *)page;
+	VOP_LOCK(vnode);
+	int error = VOP_GETPAGE(vnode, offset, page);
+	VOP_UNLOCK(vnode);
+	if (error) {
+		pushlock_acquire_exclusive(&vnode->pages_lock);
+		remove_page(vnode, offset);
+		pushlock_release_exclusive(&vnode->pages_lock);
 	} else {
-		// page is not present in the cache, we will have to load it in
-		RELEASE_LOCK();
-
-		void *address = mm_alloc_page(MEMORY_SECTION_DEFAULT);
-		if (address == NULL)
-			return ENOMEM;
-
-		new_page = mm_get_page(address);
-
-		HOLD_LOCK();
-
-		// while the lock wasn't being held, the page could have potentially been added to the cache
-		// check for it again and return from the function as if it was always there in the first place
-		page = find_page(vnode, offset);
-		if (page)
-			goto retry;
-
-		new_page->backing = vnode;
-		new_page->offset = offset;
-
-		// add it to the page cache
-		put_page(new_page);
-
-		RELEASE_LOCK();
-
-		VOP_LOCK(vnode);
-		int error = VOP_GETPAGE(vnode, offset, new_page);
-		VOP_UNLOCK(vnode);
-
-		if (error) {
-			// an error happened with GETPAGE, remove the page from the cache,
-			// tell the sleeping threads that something happened and free the page
-			// by setting backing to null so it gets treated as an anonymous page again
-			HOLD_LOCK();
-			remove_page(new_page);
-
-			new_page->flags |= PAGE_FLAGS_ERROR;
-			new_page->backing = NULL;
-			new_page->offset = 0;
-
-			RELEASE_LOCK();
-			mm_release_page(mm_get_page_address(new_page));
-			EVENT_SIGNAL(&page_ready_event);
-			return error;
-		}
-
-		HOLD_LOCK();
-		new_page->flags |= PAGE_FLAGS_READY;
-		RELEASE_LOCK();
-
-		EVENT_SIGNAL(&page_ready_event);
-		*res = new_page;
+		page->backing = vnode;
+		page->offset = offset;
 	}
 
-	return 0;
+	__atomic_or_fetch(&page->flags, PAGE_FLAGS_READY | (error ? PAGE_FLAGS_ERROR : 0), __ATOMIC_RELEASE);
+	page_notify_waiters(page);
+	*res = page;
+	trie_free_preallocation(trie_preallocation);
+	if (error) {
+		mm_release_page(phys);
+	}
+
+	return error;
 }
 
-// adds a page to the cache in a specific offset if its not already there
-int mm_cache_push_page(vnode_t *vnode, uintmax_t offset, page_t *page) {
-	__assert((offset % PAGE_SIZE) == 0);
-	HOLD_LOCK();
-
-	page_t *page_test = find_page(vnode, offset);
-	if (page_test) {
-		RELEASE_LOCK();
-		return EAGAIN;
-	}
-
-	page->backing = vnode;
-	page->offset = offset;
-	page->flags |= PAGE_FLAGS_READY;
-
-	put_page(page);
-
-	RELEASE_LOCK();
-	return 0;
-}
-
-// removes a page from the cache AND turns it into anonymous memory
-int mm_cache_evict(page_t *page) {
-	HOLD_LOCK();
-	if (page->refcount > 1) {
-		RELEASE_LOCK();
-		return EAGAIN;
-	}
-	__assert(page->refcount == 1);
-	__assert((page->flags & PAGE_FLAGS_DIRTY) == 0);
-
-	if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
-		// the page needs to be removed from the cache to continue
-		remove_page(page);
-	}
-
-	page->flags = 0;
-	page->backing = NULL;
-	page->offset = 0;
-
-	RELEASE_LOCK();
-	return 0;
-}
-
-// removes a page from the cache *but doesn't do anything to it*
+// removes a page from the page cache if only the caller holds a reference
+// keeps the page in the same state as if it were in the page cache
 int mm_cache_take_page(page_t *page) {
-	HOLD_LOCK();
-	// someone called mm_cache_get_page() and got this page while the lock wasn't held
-	// return an error status to the caller
+	pushlock_acquire_exclusive(&page->backing->pages_lock);
 	if (page->refcount > 1) {
-		RELEASE_LOCK();
+		pushlock_release_exclusive(&page->backing->pages_lock);
 		return EAGAIN;
 	}
 
-	__assert(page->refcount == 1);
-	__assert((page->flags & PAGE_FLAGS_DIRTY) == 0);
+	__assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_DIRTY) == 0);
+	// TODO: remove the need for this check (i.e. transform truncated pages into anonymous pages)
+	if ((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_TRUNCATED) == 0)
+		remove_page(page->backing, page->offset);
 
-	// XXX maybe just not allow truncated pages to take up space like this?
-	if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
-		// the page needs to be removed from the cache to continue
-		remove_page(page);
-	}
-
-	RELEASE_LOCK();
+	pushlock_release_exclusive(&page->backing->pages_lock);
 	return 0;
 }
 
+static void truncate_iterate(void *p) {
+	page_t *page = p;
+	__atomic_sub_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED);
+	if (__atomic_or_fetch(&page->flags, PAGE_FLAGS_TRUNCATED, __ATOMIC_RELEASE) & PAGE_FLAGS_PINNED) {
+		mm_release_page(mm_get_page_address(page));
+	}
+}
+
 int mm_cache_truncate(vnode_t *vnode, uintmax_t offset) {
-	HOLD_LOCK();
-	page_t *page_list = NULL;
-	page_t *page = vnode->pages;
+	uintmax_t min_page = ROUND_UP(offset, PAGE_SIZE) / PAGE_SIZE;
 
-	while (page) {
-		page_t *old_page = page;
-		page = page->vnode_next;
+	pushlock_acquire_exclusive(&vnode->pages_lock);
 
-		// only truncate past a certain offset
-		if (old_page->offset < offset)
-			continue;
-
-		old_page->flags |= PAGE_FLAGS_TRUNCATED;
-		remove_page(old_page);
-		old_page->vnode_next = page_list;
-		page_list = old_page;
-	}
-
-	RELEASE_LOCK();
-
-	// make sure to unref if they are pinned
-	while (page_list) {
-		page_t *page = page_list;
-		page_list = page_list->vnode_next;
-		if (page->flags & PAGE_FLAGS_PINNED)
-			mm_release_page(mm_get_page_address(page));
-	}
+	trie_iterate(&vnode->pages, min_page, UINT64_MAX, truncate_iterate);
+	trie_truncate(&vnode->pages, min_page);
 
+	pushlock_release_exclusive(&vnode->pages_lock);
 	return 0;
 }
 
-// called with lock held
-// returns with lock released
-// expects backing lock to be held
-static int sync_page(page_t *page, bool backing_lock) {
-	__assert(page->flags & PAGE_FLAGS_DIRTY);
-	page->flags &= ~PAGE_FLAGS_DIRTY;
-	RELEASE_LOCK();
-	int e = 0;
-	if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
-		if (backing_lock)
-			VOP_LOCK(page->backing);
-
-		e = VOP_PUTPAGE(page->backing, page->offset, (page_t *)page);
-
-		if (backing_lock)
-			VOP_UNLOCK(page->backing);
-		VOP_RELEASE(page->backing);
-	} else {
-		// page got truncated from the file while waiting to be written to disk
-		// its still holding a reference to the vnode, so release that
-		VOP_RELEASE(page->backing);
+static void mm_cache_init(void) {
+	for (size_t i = 0; i < WAIT_BLOCK_COUNT; ++i) {
+		MUTEX_INIT(&wait_blocks[i].mutex);
+		list_init(&wait_blocks[i].wait_block_list);
 	}
 
-	mm_release_page(mm_get_page_address((page_t *)page));
-	return e;
+	mm_cache_init_writer();
 }
 
-static page_t *dirty_list;
-static page_t *dirty_list_end;
-
-// expects vnode to be held
-int mm_cache_sync_vnode(vnode_t *vnode, uintmax_t offset, size_t size) {
-	offset = ROUND_DOWN(offset, PAGE_SIZE);
-	uintmax_t top = offset + size;
-	// overflow check
-	__assert(top > offset);
-	HOLD_LOCK();
-
-	// loop through all vnode pages in memory and check which ones are in the range and are dirty
-	// TODO create a proper vnode dirty list as to not have to loop through the ENTIRE thing in memory
-	page_t *page = vnode->pages;
-	page_t *vnode_dirty_list = NULL;
-	for (; page; page = page->vnode_next) {
-		if (page->offset < offset || page->offset >= top || (page->flags & PAGE_FLAGS_DIRTY) == 0 || (page->flags & PAGE_FLAGS_VNODE_SYNCING))
-			continue;
-
-		// remove from write list and add to an internal list using the write pointers
-		// in a singly linked list way
-		if (page->write_next)
-			page->write_next->write_prev = page->write_prev;
-		else
-			dirty_list_end = page->write_prev;
-
-		if (page->write_prev)
-			page->write_prev->write_next = page->write_next;
-		else
-			dirty_list = page->write_next;
-
-		page->write_next = vnode_dirty_list;
-		page->write_prev = NULL;
-		page->flags |= PAGE_FLAGS_VNODE_SYNCING;
-		vnode_dirty_list = page;
-	}
-
-	RELEASE_LOCK();
-
-	int e = 0;
-	while (vnode_dirty_list) {
-		// in the case of failure, only the first error to occur will be reported and we will not
-		// retry the write and keep on syncing the pages to disk
-		HOLD_LOCK();
-		page_t *page = vnode_dirty_list;
-		vnode_dirty_list = vnode_dirty_list->write_next;
-		page->write_next = NULL;
-		page->flags &= ~PAGE_FLAGS_VNODE_SYNCING;
-
-		// another thread could already have synced this page, verify if it is still dirty
-		if (page->flags & PAGE_FLAGS_DIRTY) {
-			int error = sync_page(page, false);
-
-			if (e == 0)
-				e = error;
-		} else {
-			RELEASE_LOCK();
-		}
-		// sync_page returns with lock released
-	}
-
-	return e;
-}
-
-int mm_cache_sync(void) {
-	eventlistener_t event_listener;
-	EVENT_INITLISTENER(&event_listener);
-	HOLD_LOCK();
-	if (dirty_list == NULL) {
-		// no dirty pages
-		RELEASE_LOCK();
-		return 0;
-	}
-
-	EVENT_ATTACH(&event_listener, &sync_event);
-	semaphore_signal(&sync);
-	RELEASE_LOCK();
-
-	EVENT_WAIT(&event_listener, 0);
-
-	EVENT_DETACHALL(&event_listener);
-	return 0;
-}
-
-// backing expected locked
-int mm_cache_make_dirty(page_t *page) {
-	bool made_dirty = false;
-	HOLD_LOCK();
-
-	if ((page->flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_TRUNCATED)) == 0) {
-		made_dirty = true;
-		// page is neither dirty nor truncated, add to dirty list and hold the page and vnode
-		page->flags |= PAGE_FLAGS_DIRTY;
-
-		if ((page->flags & PAGE_FLAGS_VNODE_SYNCING) == 0) {
-			page->write_prev = NULL;
-			page->write_next = dirty_list;
-			if (dirty_list)
-				dirty_list->write_prev = page;
-			else
-				dirty_list_end = page;
-
-			dirty_list = page;
-		}
-
-		mm_hold_page(mm_get_page_address(page));
-		__assert(page->backing);
-		VOP_HOLD(page->backing);
-	}
-
-	RELEASE_LOCK();
-	if (made_dirty) {
-		vattr_t attr;
-		attr.mtime = timekeeper_time();
-		VOP_SETATTR(page->backing, &attr, V_ATTR_MTIME, NULL);
-	}
-	return 0;
-}
-
-static void tick(context_t *, dpcarg_t arg) {
-	semaphore_signal(&sync);
-}
-
-static void writer() {
-	timerentry_t timer_entry;
-	// this will be inserted on some random cpu's timer, but it will always work after that
-	interrupt_set(false);
-	timer_insert(current_cpu()->timer, &timer_entry, tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true);
-	interrupt_set(true);
-	for (;;) {
-		HOLD_LOCK();
-		volatile page_t *page = dirty_list_end;
-		if (page == NULL) {
-			EVENT_SIGNAL(&sync_event);
-			RELEASE_LOCK();
-			semaphore_wait(&sync, false);
-			continue;
-		}
-
-		dirty_list_end = page->write_prev;
-		if (dirty_list_end)
-			dirty_list_end->write_next = NULL;
-		else
-			dirty_list = NULL;
-
-		page->write_prev = NULL;
-		// TODO notify error on mm_cache_sync_vnode
-		sync_page((page_t *)page, true);
-	}
-}
-
-void mm_cache_init(void) {
-	MUTEX_INIT(&mutex);
-	table = mm_map(NULL, TABLE_SIZE * sizeof(page_t *), MM_RANGE_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
-	__assert(table);
-	memset(table, 0, TABLE_SIZE * sizeof(page_t *));
-
-	SEMAPHORE_INIT(&sync, 0);
-	writer_thread = sched_newthread(writer, PAGE_SIZE * 16, 1, NULL, NULL);
-	__assert(writer_thread);
-	sched_queue(writer_thread);
-	mm_cache_sync();
-	EVENT_INITHEADER(&sync_event);
-	EVENT_INITHEADER(&page_ready_event);
-}
-
-INIT_ROUTINE_DEFINE(mm_cache, INIT_ROUTINE_FLAGS_NONE, mm_cache_init, scheduler);
+INIT_ROUTINE_DEFINE(mm_cache, INIT_ROUTINE_FLAGS_NONE, mm_cache_init, scheduler, trie);

kernel-src/mm/mm.c

@@ -163,11 +163,8 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) {
 		if ((!private || (!file_mapping && current_page->refcount == 1)) && mapped_address != zero_page) {
 			arch_mmu_remap(current_mm_context()->pagetable, mapped_address, vaddr, range->mmuflags);
 
-			if (file_mapping && cacheable) {
-				VOP_LOCK(range->vnode);
+			if (file_mapping && cacheable)
 				mm_cache_make_dirty(current_page);
-				VOP_UNLOCK(range->vnode);
-			}
 
 			*resulting_page = current_page;
 			return 0;
@@ -228,11 +225,8 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) {
 			return ENOMEM;
 		}
 
-		if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE)) {
-			VOP_LOCK(range->vnode);
+		if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE))
 			mm_cache_make_dirty(vn_page);
-			VOP_UNLOCK(range->vnode);
-		}
 
 		*resulting_page = vn_page;
 		return 0;

kernel-src/mm/page.c

@@ -119,7 +119,7 @@ static void remove_from_free_list(page_t *page) {
 static void internal_hold(page_t *page) {
 	if (++page->refcount == 1) {
 		// this is only valid on standby pages, in case of free pages its an use after free
-		__assert((page->flags & PAGE_FLAGS_FREE) == 0);
+		__assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_FREE) == 0);
 		remove_from_free_list(page);
 	}
 }
@@ -147,10 +147,10 @@ void mm_release_page(void *addr) {
 	__assert(page->refcount != 0);
 	if (--page->refcount == 0) {
 		__assert(!mm_is_page_locked(page));
-		__assert((page->flags & PAGE_FLAGS_DIRTY) == 0);
+		__assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_DIRTY) == 0);
 		insert_in_free_list(page);
 		if (page->backing == NULL)
-			page->flags |= PAGE_FLAGS_FREE;
+			__atomic_or_fetch(&page->flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED);
 	}
 	MUTEX_RELEASE(&free_list_mutex);
 }
@@ -236,7 +236,7 @@ void mm_force_free_page(void *address, size_t count) {
 		uintmax_t page_id = base_id + i;
 		PAGE_BOUNDARY_CHECK(page_id);
 		page_t *page = &pages[page_id];
-		page->flags |= PAGE_FLAGS_FREE;
+		__atomic_or_fetch(&page->flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED);
 		insert_in_free_list(page);
 	}
 	MUTEX_RELEASE(&free_list_mutex);
@@ -277,7 +277,7 @@ void mm_page_init() {
 		if (e->type == LIMINE_MEMMAP_USABLE) {
 			int first_usable_page = e == biggest ? ROUND_UP(e->base + page_count * sizeof(page_t), PAGE_SIZE) / PAGE_SIZE : e->base / PAGE_SIZE;
 			for (int i = first_usable_page; i < (e->base + e->length) / PAGE_SIZE; ++i) {
-				pages[i].flags |= PAGE_FLAGS_FREE;
+				__atomic_or_fetch(&pages[i].flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED);
 				insert_in_free_list(&pages[i]);
 			}
 		} else if (e->type == LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE) {
@@ -311,7 +311,7 @@ void *mm_alloc_pages(size_t size, int section) {
 			if (page >= page_count)
 				break;
 
-			if (pages[page].flags & PAGE_FLAGS_FREE) {
+			if (__atomic_load_n(&pages[page].flags, __ATOMIC_RELAXED) & PAGE_FLAGS_FREE) {
 				++found;
 			} else
 				found = 0;

kernel-src/mm/range.c

@@ -165,9 +165,7 @@ void mm_destroy_range(mm_range_t *range, uintmax_t _offset, size_t size, int fla
 			} else if (arch_mmu_iswritable(current_mm_context()->pagetable, vaddr)) {
 				// dirty page cache mapping
 				arch_mmu_unmap(current_mm_context()->pagetable, vaddr);
-				VOP_LOCK(range->vnode);
 				mm_cache_make_dirty(mm_get_page(physical));
-				VOP_UNLOCK(range->vnode);
 				mm_release_page(physical);
 			} else {
 				// non dirty page mapping
@@ -212,9 +210,7 @@ static void change_mmu_range(mm_range_t *range, void *base, size_t size, mmuflag
 			// removing write permissions from a writeable dirty shared mapped page, mark it as dirty, as
 			// it won't be marked dirty upon a mm_unmap after this
 			page_t *page = mm_get_page(physical);
-			VOP_LOCK(range->vnode);
 			mm_cache_make_dirty(page);
-			VOP_UNLOCK(range->vnode);
 		}
 
 		// we will only change the mapping if the permissions decreased

kernel-src/mm/writeback.c

@@ -0,0 +1,206 @@
+#include <kernel/mm.h>
+#include <kernel/event.h>
+#include <kernel/scheduler.h>
+#include <kernel/timekeeper.h>
+#include <arch/cpu.h>
+#include <logging.h>
+#include <mutex.h>
+#include <list.h>
+#include <util.h>
+
+#define WRITER_TICK_SECONDS 15
+
+static mutex_t vnode_dirty_list_mutex;
+static list_t vnode_dirty_list;
+static bool writer_syncing;
+static eventheader_t writer_wakeup_event;
+static eventheader_t sync_complete_event;
+
+// OK
+void mm_cache_make_dirty(page_t *page) {
+	vnode_t *vnode = page->backing;
+	MUTEX_ACQUIRE(&vnode->dirty_list_mutex);
+
+	int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED);
+	for (;;) {
+		if (flags & PAGE_FLAGS_TRUNCATED) {
+			MUTEX_RELEASE(&vnode->dirty_list_mutex);
+			return;
+		}
+
+		int new_flags = flags | PAGE_FLAGS_DIRTY;
+		if (__atomic_compare_exchange_n(&page->flags, &flags, new_flags, true, __ATOMIC_RELEASE, __ATOMIC_RELAXED))
+			break;
+	}
+
+	if (flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_SYNCING)) {
+		MUTEX_RELEASE(&vnode->dirty_list_mutex);
+		return;
+	}
+
+	mm_hold_page(mm_get_page_address(page));
+	list_push_back(&vnode->dirty_pages, &page->dirty_list_node);
+
+	if (!vnode->dirty) {
+		vnode->dirty = true;
+		VOP_HOLD(vnode);
+
+		MUTEX_ACQUIRE(&vnode_dirty_list_mutex);
+		list_push_back(&vnode_dirty_list, &vnode->vnode_dirty_list_node);
+		MUTEX_RELEASE(&vnode_dirty_list_mutex);
+	}
+
+	MUTEX_RELEASE(&vnode->dirty_list_mutex);
+
+	vattr_t attr;
+	attr.mtime = timekeeper_time();
+	VOP_LOCK(vnode);
+	VOP_SETATTR(vnode, &attr, V_ATTR_MTIME, NULL);
+	VOP_UNLOCK(vnode);
+}
+
+// OK
+static void mark_page_syncing(page_t *page) {
+	int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED);
+	for (;;) {
+		__assert(flags & PAGE_FLAGS_DIRTY);
+		__assert((flags & PAGE_FLAGS_SYNCING) == 0);
+
+		int new_flags = (flags & ~PAGE_FLAGS_DIRTY) | PAGE_FLAGS_SYNCING;
+		if (__atomic_compare_exchange_n(&page->flags, &flags, new_flags, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
+			break;
+	}
+}
+
+// OK
+// expects the per-vnode dirty list mutex to be held. returns true if the caller must release a vnode reference.
+static bool try_undirty_vnode(vnode_t *vnode) {
+	if (vnode->dirty == false)
+		return false;
+
+	MUTEX_ACQUIRE(&vnode_dirty_list_mutex);
+	list_remove(&vnode_dirty_list, &vnode->vnode_dirty_list_node);
+	MUTEX_RELEASE(&vnode_dirty_list_mutex);
+	vnode->dirty = false;
+	return true;
+}
+
+// OK
+int mm_cache_sync_vnode(vnode_t *vnode) {
+	int first_error = 0;
+	MUTEX_ACQUIRE(&vnode->writeback_mutex);
+
+	for (;;) {
+		MUTEX_ACQUIRE(&vnode->dirty_list_mutex);
+		list_node_t *page_node = list_pop_front(&vnode->dirty_pages);
+		if (page_node == NULL) {
+			bool was_dirty = try_undirty_vnode(vnode);
+			MUTEX_RELEASE(&vnode->dirty_list_mutex);
+			MUTEX_RELEASE(&vnode->writeback_mutex);
+
+			if (was_dirty) {
+				VOP_RELEASE(vnode);
+			}
+			return first_error;
+		}
+
+		page_t *page = container_of(page_node, page_t, dirty_list_node);
+		mark_page_syncing(page);
+		MUTEX_RELEASE(&vnode->dirty_list_mutex);
+
+		VOP_LOCK(vnode);
+
+		// TODO: remark as dirty on error
+		int error = 0;
+		if ((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_TRUNCATED) == 0)
+			error = VOP_PUTPAGE(vnode, page->offset, page);
+
+		VOP_UNLOCK(vnode);
+		if (first_error == 0)
+			first_error = error;
+
+		int flags = __atomic_and_fetch(&page->flags, ~PAGE_FLAGS_SYNCING, __ATOMIC_ACQUIRE);
+		if (flags & PAGE_FLAGS_DIRTY) {
+			MUTEX_ACQUIRE(&vnode->dirty_list_mutex);
+			list_push_back(&vnode->dirty_pages, &page->dirty_list_node);
+			MUTEX_RELEASE(&vnode->dirty_list_mutex);
+		} else {
+			mm_release_page(mm_get_page_address(page));
+		}
+	}
+}
+
+// OK
+int mm_cache_sync(void) {
+	MUTEX_ACQUIRE(&vnode_dirty_list_mutex);
+
+	if (vnode_dirty_list.head == NULL && !writer_syncing) {
+		MUTEX_RELEASE(&vnode_dirty_list_mutex);
+		return 0;
+	}
+
+	eventlistener_t listener;
+	EVENT_INITLISTENER(&listener);
+	EVENT_ATTACH(&listener, &sync_complete_event);
+
+	EVENT_SIGNAL(&writer_wakeup_event);
+	MUTEX_RELEASE(&vnode_dirty_list_mutex);
+
+	EVENT_WAIT(&listener, 0);
+	EVENT_DETACHALL(&listener);
+	return 0;
+}
+
+// OK
+static void writer_tick(context_t *, dpcarg_t) {
+	EVENT_SIGNAL(&writer_wakeup_event);
+}
+
+// OK
+static void writer(void) {
+	long ipl = interrupt_raiseipl(IPL_DPC);
+	timerentry_t timer_entry;
+	timer_insert(current_cpu()->timer, &timer_entry, writer_tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true);
+	interrupt_loweripl(ipl);
+
+	for (;;) {
+		eventlistener_t listener;
+		EVENT_INITLISTENER(&listener);
+
+		MUTEX_ACQUIRE(&vnode_dirty_list_mutex);
+		list_node_t *list_node = vnode_dirty_list.head;
+		if (list_node == NULL) {
+			writer_syncing = false;
+			EVENT_ATTACH(&listener, &writer_wakeup_event);
+			EVENT_SIGNAL(&sync_complete_event);
+			MUTEX_RELEASE(&vnode_dirty_list_mutex);
+
+			EVENT_WAIT(&listener, 0);
+			EVENT_DETACHALL(&listener);
+			continue;
+		}
+
+		vnode_t *vnode = container_of(list_node, vnode_t, vnode_dirty_list_node);
+		VOP_HOLD(vnode);
+
+		writer_syncing = true;
+		MUTEX_RELEASE(&vnode_dirty_list_mutex);
+
+		mm_cache_sync_vnode(vnode);
+
+		VOP_RELEASE(vnode);
+	}
+}
+
+// OK
+void mm_cache_init_writer(void) {
+	MUTEX_INIT(&vnode_dirty_list_mutex);
+	list_init(&vnode_dirty_list);
+	writer_syncing = false;
+	EVENT_INITHEADER(&writer_wakeup_event);
+	EVENT_INITHEADER(&sync_complete_event);
+
+	thread_t *writer_thread = sched_newthread(writer, PAGE_SIZE * 2, 0, NULL, NULL);
+	__assert(writer_thread);
+	sched_queue(writer_thread);
+}

kernel-src/sys/syscalls/sync.c

@@ -19,9 +19,7 @@ syscallret_t syscall_fsync(context_t *context, int fd) {
 		return ret;
 	}
 
-	VOP_LOCK(file->vnode);
 	ret.errno = VOP_SYNC(file->vnode);
-	VOP_UNLOCK(file->vnode);
 
 	fd_release(file);
 	ret.ret = ret.errno ? -1 : 0;