From 31f9ae17cf7bf21ed8f4d7de4968cc4831f050dc Mon Sep 17 00:00:00 2001 From: Matheus Date: Tue, 28 Jul 2026 16:33:55 -0300 Subject: [PATCH] page cache rewrite --- kernel-src/fs/abc.c | 2 +- kernel-src/fs/ext2.c | 25 +- kernel-src/fs/fat/fatfs.c | 33 +- kernel-src/fs/tmpfs.c | 48 ++- kernel-src/fs/vfs.c | 72 ++-- kernel-src/include/kernel/file.h | 4 +- kernel-src/include/kernel/mm.h | 9 +- kernel-src/include/kernel/page.h | 12 +- kernel-src/include/kernel/vfs.h | 21 +- kernel-src/include/trie.h | 2 +- kernel-src/lib/trie.c | 27 +- kernel-src/mm/cache.c | 613 +++++++++---------------------- kernel-src/mm/mm.c | 10 +- kernel-src/mm/page.c | 12 +- kernel-src/mm/range.c | 4 - kernel-src/mm/writeback.c | 206 +++++++++++ kernel-src/sys/syscalls/sync.c | 2 - 17 files changed, 494 insertions(+), 608 deletions(-) create mode 100644 kernel-src/mm/writeback.c diff --git a/kernel-src/fs/abc.c b/kernel-src/fs/abc.c index 59e32ad0..e5f5e996 100644 --- a/kernel-src/fs/abc.c +++ b/kernel-src/fs/abc.c @@ -301,4 +301,4 @@ static void abc_init_routine(void) { __assert(abc_block_cache); } -INIT_ROUTINE_DEFINE(abc, INIT_ROUTINE_FLAGS_NONE, abc_init_routine, scheduler, trie); +INIT_ROUTINE_DEFINE(abc, INIT_ROUTINE_FLAGS_NONE, abc_init_routine, mm_cache, scheduler, trie); diff --git a/kernel-src/fs/ext2.c b/kernel-src/fs/ext2.c index 2ea2689a..bbbbf6fb 100644 --- a/kernel-src/fs/ext2.c +++ b/kernel-src/fs/ext2.c @@ -1822,7 +1822,6 @@ static int ext2_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) { static int ext2_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) { // only regular files get cached __assert(node->type == V_TYPE_REGULAR); - size_t writec; iovec_t iovec = { .addr = MAKE_HHDM(mm_get_page_address(page)), .len = PAGE_SIZE @@ -1830,21 +1829,8 @@ static int ext2_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) { iovec_iterator_t iovec_iterator; iovec_iterator_init(&iovec_iterator, &iovec, 1); - int error = VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &writec, NULL); - - // its possible that writec is zero here. the condition where this is possible is when - // the file is truncated after the check in the page sync function but before it actually is - // written back to disk. therefore, we check that the page IS infact truncated if this happens. - // if it isn't, something else happened and its not safe to continue. - // - // TODO: since this check will likely be copied between different filesystem drivers, it could be interesting to have - // ext2_putpage take in a (size_t *) pointer and have this check be in whatever function calls VOP_PUTPAGE. - - if (writec == 0) { - __assert(page->flags & PAGE_FLAGS_TRUNCATED); - } - - return error; + size_t writec; + return VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &writec, NULL); } static int ext2_rename(vnode_t *sourcedir, vnode_t *source, char *oldname, vnode_t *targetdir, char *newname, int flags) { @@ -1964,13 +1950,11 @@ static int ext2_rename(vnode_t *sourcedir, vnode_t *source, char *oldname, vnode } static int ext2_sync(vnode_t *vnode) { - int e = mm_cache_sync_vnode(vnode, 0, UINT64_MAX); + int e = mm_cache_sync_vnode(vnode); ext2fs_t *fs = (ext2fs_t *)vnode->vfs; abc_sync(&fs->abc); // TODO don't sync the entire disk but rather only the inodes and blocks - VOP_LOCK(fs->backing); - int e2 = mm_cache_sync_vnode(fs->backing, 0, UINT64_MAX); - VOP_UNLOCK(fs->backing); + int e2 = mm_cache_sync_vnode(fs->backing); // only the first errors are reported return e ? e : e2; } @@ -2007,7 +1991,6 @@ static int ext2_inactive(vnode_t *vnode) { __assert(hashtable_remove(&fs->inodetable, &node->id, sizeof(node->id)) == 0); MUTEX_RELEASE(&fs->inodetablelock); - mm_cache_truncate(vnode, 0); freeinode(fs, &node->inode, node->id); slab_free(nodecache, node); diff --git a/kernel-src/fs/fat/fatfs.c b/kernel-src/fs/fat/fatfs.c index 7eaf3fa5..20cd081e 100644 --- a/kernel-src/fs/fat/fatfs.c +++ b/kernel-src/fs/fat/fatfs.c @@ -200,9 +200,7 @@ static int fatfs_unmount(vfs_t *vfs) { static int fatfs_sync(vfs_t *vfs) { fatfs_t *fatfs = (fatfs_t *)vfs; abc_sync(&fatfs->abc); - VOP_LOCK(fatfs->backing); - int error = mm_cache_sync_vnode(fatfs->backing, 0, UINT64_MAX); - VOP_UNLOCK(fatfs->backing); + int error = mm_cache_sync_vnode(fatfs->backing); return error; } @@ -264,19 +262,16 @@ static int fatfs_syncvnode(vnode_t *vnode) { fatnode_t *fatnode = (fatnode_t *)vnode; // sync file data - int e = mm_cache_sync_vnode(vnode, 0, UINT64_MAX); + int e = mm_cache_sync_vnode(vnode); abc_sync(&fs->abc); - VOP_LOCK(fs->backing); // sync the dent int e2 = 0; if (fatnode->dent_disk_offset) - e2 = mm_cache_sync_vnode(fs->backing, fatnode->dent_disk_offset, sizeof(fatfs_dent_t)); + e2 = mm_cache_sync_vnode(fs->backing); // and sync the fats - int e3 = mm_cache_sync_vnode(fs->backing, fs->fat_offset, fs->fat_size * fs->fat_count); - - VOP_UNLOCK(fs->backing); + int e3 = mm_cache_sync_vnode(fs->backing); // only the first errors are reported if (e) @@ -391,7 +386,6 @@ static int fatfs_read(vnode_t *vnode, iovec_iterator_t *iovec_iterator, size_t s static int fatfs_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) { // only regular files get cached __assert(node->type == V_TYPE_REGULAR); - size_t write_count; iovec_t iovec = { .addr = MAKE_HHDM(mm_get_page_address(page)), .len = PAGE_SIZE @@ -399,23 +393,11 @@ static int fatfs_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) { iovec_iterator_t iovec_iterator; iovec_iterator_init(&iovec_iterator, &iovec, 1); - int error = VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &write_count, NULL); - - // its possible that write_count is zero here. the condition where this is possible is when - // the file is truncated after the check in the page sync function but before it actually is - // written back to disk. therefore, we check that the page IS infact truncated if this happens. - // if it isn't, something else happened and its not safe to continue. - // - // TODO: since this check will likely be copied between different filesystem drivers, it could be interesting to have - // fatfs_putpage take in a (size_t *) pointer and have this check be in whatever function calls VOP_PUTPAGE. - - if (write_count == 0) { - __assert(page->flags & PAGE_FLAGS_TRUNCATED); - } - - return error; + size_t write_count; + return VOP_WRITE(node, &iovec_iterator, PAGE_SIZE, offset, 0, &write_count, NULL); } + static int fatfs_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) { // only regular files get cached __assert(node->type == V_TYPE_REGULAR); @@ -970,7 +952,6 @@ static int fatfs_inactive(vnode_t *vnode) { __assert(!"fatfs VOP_INACTIVE() with hardlink count >0 is currently unimplemented"); } else { // unlinked file having resources released when kernel refcount hits zero - mm_cache_truncate(vnode, 0); fatfs_resize_file(fatfs, fatnode, 0); } diff --git a/kernel-src/fs/tmpfs.c b/kernel-src/fs/tmpfs.c index 0464a08f..bcdb9bf3 100644 --- a/kernel-src/fs/tmpfs.c +++ b/kernel-src/fs/tmpfs.c @@ -224,6 +224,36 @@ static int tmpfs_root(vfs_t *vfs, vnode_t **vnode) { return 0; } +static int tmpfs_rw(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, size_t *done, bool write) { + *done = 0; + while (*done < size) { + uintmax_t current_offset = offset + *done; + uintmax_t page_offset = current_offset / PAGE_SIZE * PAGE_SIZE; + size_t page_start = current_offset % PAGE_SIZE; + size_t copy_size = min(PAGE_SIZE - page_start, size - *done); + + page_t *page; + int error = mm_cache_get_page(node, page_offset, 0, &page); + if (error) + return error; + + void *address = MAKE_HHDM(mm_get_page_address(page)); + if (write) + error = iovec_iterator_copy_to_buffer(iovec_iterator, (void *)((uintptr_t)address + page_start), copy_size); + else + error = iovec_iterator_copy_from_buffer(iovec_iterator, (void *)((uintptr_t)address + page_start), copy_size); + mm_release_page(FROM_HHDM(address)); + if (error) + return error; + + *done += copy_size; + } + + if (write && *done) + ((tmpfsnode_t *)node)->attr.mtime = timekeeper_time(); + return 0; +} + static int tmpfs_read(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, int flags, size_t *readc, cred_t *cred) { if (node->type == V_TYPE_DIR) return EISDIR; @@ -231,7 +261,13 @@ static int tmpfs_read(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t si if (node->type != V_TYPE_REGULAR) return EINVAL; - __assert(!"tmpfs_read is handled by the page cache"); + tmpfsnode_t *tmpfsnode = (tmpfsnode_t *)node; + if (offset >= tmpfsnode->attr.size) { + *readc = 0; + return 0; + } + size = min(size, tmpfsnode->attr.size - offset); + return tmpfs_rw(node, iovec_iterator, size, offset, readc, false); } static int tmpfs_write(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, int flags, size_t *writec, cred_t *cred) { @@ -241,7 +277,10 @@ static int tmpfs_write(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t s if (node->type != V_TYPE_REGULAR) return EINVAL; - __assert(!"tmpfs_read is handled by the page cache"); + if (size + offset < offset) + return EOVERFLOW; + __assert(size + offset <= ((tmpfsnode_t *)node)->attr.size); + return tmpfs_rw(node, iovec_iterator, size, offset, writec, true); } static int tmpfs_access(vnode_t *vnode, mode_t mode, cred_t *cred) { @@ -501,9 +540,6 @@ static int tmpfs_inactive(vnode_t *node) { arch_e9_puts("tmpfs_inactive\n"); tmpfsnode_t *tmp_node = (tmpfsnode_t *)node; __assert(tmp_node->attr.nlinks == (node->type == V_TYPE_DIR ? 1 : 0)); - if (node->type == V_TYPE_REGULAR) { - mm_cache_truncate(node, 0); - } freenode(tmp_node); return 0; } @@ -535,7 +571,7 @@ static int tmpfs_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) { void *phyhhdm = MAKE_HHDM(phy); mm_hold_page(phy); - page->flags |= PAGE_FLAGS_PINNED; + __atomic_fetch_or(&page->flags, PAGE_FLAGS_PINNED, __ATOMIC_RELAXED); memset(phyhhdm, 0, PAGE_SIZE); return 0; } diff --git a/kernel-src/fs/vfs.c b/kernel-src/fs/vfs.c index e556b366..810e6558 100644 --- a/kernel-src/fs/vfs.c +++ b/kernel-src/fs/vfs.c @@ -152,6 +152,9 @@ int vfs_sync(void) { } void vfs_inactive(vnode_t *vnode) { + if (vfs_iscacheable(vnode)) + mm_cache_truncate(vnode, 0); + if (vnode->type == V_TYPE_SOCKET && vnode->socketbinding) { localsock_leavebinding(vnode); } else if (vnode->type == V_TYPE_FIFO && vnode->fifobinding) { @@ -438,33 +441,14 @@ static void bytestopages(uintmax_t offset, size_t size, uintmax_t *pageoffset, s *pagecount = toppage - *pageoffset; } -static int writenocache(vnode_t *node, page_t *page, uintmax_t pageoffset) { - // don't keep the pages in the cache! - // if its a file, do the proper filesystem sync. - // if its a block device, sync only the page we just dirtied - VOP_LOCK(node); - int e; - if (node->type == V_TYPE_REGULAR) - e = VOP_SYNC(node); - else - e = mm_cache_sync_vnode(node, pageoffset, PAGE_SIZE); - VOP_UNLOCK(node); - - // try to turn it into anonymous memory - mm_cache_evict(page); - - return e; -} - int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, size_t *written, int flags) { int err = 0; - if (vfs_iscacheable(node)) { + bool cacheable = vfs_iscacheable(node); + if (cacheable) { *written = 0; - // can't write a size 0 buffer if (size == 0) return 0; - // overflow if (size + offset < offset) return EINVAL; @@ -502,7 +486,9 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size size = min(size + offset, bytesize) - offset; } + } + if (cacheable && (flags & V_FFLAGS_NOCACHE) == 0) { uintmax_t pageoffset, pagecount, startoffset; bytestopages(offset, size, &pageoffset, &pagecount, &startoffset); page_t *page = NULL; @@ -525,15 +511,10 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size mm_cache_make_dirty(page); *written += writesize; - if (flags & V_FFLAGS_NOCACHE) - err = writenocache(node, page, pageoffset * PAGE_SIZE); - pageoffset += 1; pagecount -= 1; mm_release_page(FROM_HHDM(address)); - if (err) - goto leave; } for (uintmax_t offset = 0; offset < pagecount * PAGE_SIZE; offset += PAGE_SIZE) { @@ -554,30 +535,30 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size mm_cache_make_dirty(page); *written += writesize; - if (flags & V_FFLAGS_NOCACHE) - err = writenocache(node, page, pageoffset * PAGE_SIZE + offset); - mm_release_page(FROM_HHDM(address)); - - if (err) - goto leave; } - leave: - MUTEX_RELEASE(&node->size_lock); + if (flags & V_FFLAG_MUST_SYNC) + err = mm_cache_sync_vnode(node); + } else { - // special file, just write as its not being cached + // Non-cacheable objects and direct I/O bypass the page cache. VOP_LOCK(node); err = VOP_WRITE(node, iovec_iterator, size, offset, flags, written, getcred()); VOP_UNLOCK(node); } + leave: + if (cacheable) + MUTEX_RELEASE(&node->size_lock); + return err; } int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, size_t *bytesread, int flags) { int err = 0; - if (vfs_iscacheable(node)) { + bool cacheable = vfs_iscacheable(node); + if (cacheable) { *bytesread = 0; // can't read 0 bytes from the cache if (size == 0) @@ -615,7 +596,9 @@ int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, goto leave; size = min(size + offset, nodesize) - offset; + } + if (cacheable && (flags & V_FFLAGS_NOCACHE) == 0) { uintmax_t pageoffset, pagecount, startoffset; bytestopages(offset, size, &pageoffset, &pagecount, &startoffset); page_t *page = NULL; @@ -637,11 +620,6 @@ int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, *bytesread += readsize; - if (flags & V_FFLAGS_NOCACHE) { - // try to turn it into anonymous memory - mm_cache_evict(page); - } - pageoffset += 1; pagecount -= 1; @@ -664,20 +642,18 @@ int vfs_read_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size, } *bytesread += readsize; - if (flags & V_FFLAGS_NOCACHE) { - // try to turn it into anonymous memory - mm_cache_evict(page); - } mm_release_page(FROM_HHDM(address)); } - leave: - MUTEX_RELEASE(&node->size_lock); } else { - // special file, just read as size doesn't matter + // Non-cacheable objects and direct I/O bypass the page cache. VOP_LOCK(node); err = VOP_READ(node, iovec_iterator, size, offset, flags, bytesread, getcred()); VOP_UNLOCK(node); } + + leave: + if (cacheable) + MUTEX_RELEASE(&node->size_lock); return err; } diff --git a/kernel-src/include/kernel/file.h b/kernel-src/include/kernel/file.h index d8e64063..18c5585d 100644 --- a/kernel-src/include/kernel/file.h +++ b/kernel-src/include/kernel/file.h @@ -73,8 +73,10 @@ static inline int fileflagstovnodeflags(int flags) { vnflags |= V_FFLAGS_NONBLOCKING; if (flags & O_NOCTTY) vnflags |= V_FFLAGS_NOCTTY; - if (flags & (O_DIRECT | O_SYNC | O_DSYNC)) + if (flags & O_DIRECT) vnflags |= V_FFLAGS_NOCACHE; + if (flags & (O_SYNC | O_DSYNC)) + vnflags |= V_FFLAG_MUST_SYNC; return vnflags; } diff --git a/kernel-src/include/kernel/mm.h b/kernel-src/include/kernel/mm.h index 18c57b6e..6d80bdd3 100644 --- a/kernel-src/include/kernel/mm.h +++ b/kernel-src/include/kernel/mm.h @@ -99,15 +99,14 @@ void mm_init(); extern size_t mm_cache_cached_pages; -void mm_cache_init(void); +#define MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE 1 int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res); int mm_cache_take_page(page_t *page); -int mm_cache_make_dirty(page_t *page); +void mm_cache_make_dirty(page_t *page); int mm_cache_truncate(vnode_t *vnode, uintmax_t offset); -int mm_cache_sync_vnode(vnode_t *vnode, uintmax_t startoffset, size_t size); -int mm_cache_push_page(vnode_t *vnode, uintmax_t offset, page_t *page); +int mm_cache_sync_vnode(vnode_t *vnode); int mm_cache_sync(void); -int mm_cache_evict(page_t *page); +void mm_cache_init_writer(void); void mm_range_init(void); mm_range_t *mm_alloc_range(void); diff --git a/kernel-src/include/kernel/page.h b/kernel-src/include/kernel/page.h index 597b9998..1c6f4589 100644 --- a/kernel-src/include/kernel/page.h +++ b/kernel-src/include/kernel/page.h @@ -4,6 +4,7 @@ #include #include #include +#include #define MEMORY_SECTION_COUNT 3 #define MEMORY_SECTION_1MB 0 @@ -16,24 +17,17 @@ #define PAGE_FLAGS_DIRTY 8 #define PAGE_FLAGS_READY 16 #define PAGE_FLAGS_ERROR 32 -#define PAGE_FLAGS_VNODE_SYNCING 64 +#define PAGE_FLAGS_SYNCING 64 typedef struct page_t { struct vnode_t *backing; uintmax_t offset; - struct page_t *hash_next; - struct page_t *hash_prev; - struct page_t *vnode_next; - struct page_t *vnode_prev; union { struct { struct page_t *free_next; struct page_t *free_prev; }; - struct { - struct page_t *write_next; - struct page_t *write_prev; - }; + list_node_t dirty_list_node; }; uintmax_t refcount; uintmax_t lock_count; diff --git a/kernel-src/include/kernel/vfs.h b/kernel-src/include/kernel/vfs.h index cbb16098..9ae8ab77 100644 --- a/kernel-src/include/kernel/vfs.h +++ b/kernel-src/include/kernel/vfs.h @@ -10,7 +10,10 @@ #include #include #include +#include #include +#include +#include #define V_ATTR_MODE 1 #define V_ATTR_UID 2 @@ -120,6 +123,7 @@ typedef struct advlock_t { #define V_FFLAGS_NOCTTY 32 #define V_FFLAGS_NOCACHE 64 #define V_FFLAGS_CLOSE_URGENT 128 +#define V_FFLAG_MUST_SYNC 256 typedef struct vnode_t { struct vops_t *ops; @@ -135,7 +139,14 @@ typedef struct vnode_t { void *fifobinding; }; - struct page_t *pages; + bool dirty; + mutex_t writeback_mutex; + mutex_t dirty_list_mutex; + list_node_t vnode_dirty_list_node; + list_t dirty_pages; + + pushlock_t pages_lock; + trie_t pages; mutex_t adv_mutex; advlock_t *advlock; @@ -177,6 +188,7 @@ typedef struct vops_t { int (*rename)(vnode_t *sourcedir, vnode_t *source, char *oldname, vnode_t *targetdir, char *newname, int flags); int (*getpage)(vnode_t *node, uintmax_t offset, struct page_t *page); int (*putpage)(vnode_t *node, uintmax_t offset, struct page_t *page); + // node must be unlocked on entry. int (*sync)(vnode_t *node); int (*advlock)(vnode_t *node, int op, advlock_t *advlock); int (*lock)(vnode_t *node); @@ -208,7 +220,12 @@ typedef struct vops_t { (vn)->type = t; \ (vn)->vfs = v; \ (vn)->vfsmounted = NULL; \ - (vn)->pages = NULL; + trie_init(&(vn)->pages); \ + (vn)->pages_lock = 0; \ + MUTEX_INIT(&(vn)->writeback_mutex); \ + MUTEX_INIT(&(vn)->dirty_list_mutex); \ + (vn)->dirty = false; \ + list_init(&(vn)->dirty_pages); #define VOP_LOCK(v) (v)->ops->lock(v) #define VOP_UNLOCK(v) (v)->ops->unlock(v) diff --git a/kernel-src/include/trie.h b/kernel-src/include/trie.h index 0f281c3d..f586faa6 100644 --- a/kernel-src/include/trie.h +++ b/kernel-src/include/trie.h @@ -39,7 +39,7 @@ int trie_insert_preallocated(trie_t *trie, uint64_t key, void *value, trie_preal void trie_free_preallocation(trie_preallocation_t *preallocation); int trie_lookup(trie_t *trie, uint64_t key, void **ret); int trie_remove(trie_t *trie, uint64_t key); -void trie_truncate(trie_t *trie, uint64_t max_key); +void trie_truncate(trie_t *trie, uint64_t min_key); void trie_iterate(trie_t *trie, uint64_t min_key, uint64_t max_key, trie_iterate_fn_t); #endif diff --git a/kernel-src/lib/trie.c b/kernel-src/lib/trie.c index 12b595ff..40a50621 100644 --- a/kernel-src/lib/trie.c +++ b/kernel-src/lib/trie.c @@ -285,13 +285,13 @@ static void trie_recursively_free(trie_node_t *trie_node, uint8_t height) { slab_free(trie_node_cache, trie_node); } -static void trie_truncate_internal(trie_node_t *trie_node, uint64_t max_key, uint8_t height) { - uint8_t offset = get_offset(max_key, height); +static void trie_truncate_internal(trie_node_t *trie_node, uint64_t min_key, uint8_t height) { + uint8_t offset = get_offset(min_key, height); if (height) { trie_node_t *child_node = trie_node->nodes[offset]; if (child_node) { - trie_truncate_internal(child_node, max_key, height - 1); + trie_truncate_internal(child_node, min_key, height - 1); if (child_node->count == 0) { slab_free(trie_node_cache, child_node); @@ -313,11 +313,8 @@ static void trie_truncate_internal(trie_node_t *trie_node, uint64_t max_key, uin } } -void trie_truncate(trie_t *trie, uint64_t max_key) { - if (key_height(max_key) > trie->height) - return; - - if (max_key == 0) { +void trie_truncate(trie_t *trie, uint64_t min_key) { + if (min_key == 0) { if (trie->height) trie_recursively_free(trie->root, trie->height - 1); @@ -326,20 +323,10 @@ void trie_truncate(trie_t *trie, uint64_t max_key) { return; } - if (max_key == 1) { - void *item; - if (trie_lookup(trie, 0, &item)) - item = NULL; - - if (trie->height) - trie_recursively_free(trie->root, trie->height - 1); - - trie->root_item = item; - trie->height = 0; + if (trie->height == 0 || key_height(min_key) > trie->height) return; - } - trie_truncate_internal(trie->root, max_key, trie->height - 1); + trie_truncate_internal(trie->root, min_key, trie->height - 1); trie_trim(trie); } diff --git a/kernel-src/mm/cache.c b/kernel-src/mm/cache.c index 45492ef4..8101315e 100644 --- a/kernel-src/mm/cache.c +++ b/kernel-src/mm/cache.c @@ -1,503 +1,220 @@ #include -#include +#include #include -#include -#include #include -#define TABLE_SIZE 4096 -#define WRITER_TICK_SECONDS 15 +typedef struct { + list_node_t list_node; + semaphore_t semaphore; + page_t *page; +} page_wait_block_t; -static mutex_t mutex; -static page_t **table; +typedef struct { + mutex_t mutex; + list_t wait_block_list; +} page_wait_block_header_t; -static thread_t *writer_thread; -static semaphore_t sync; -static eventheader_t sync_event; -static eventheader_t page_ready_event; +#define WAIT_BLOCK_COUNT 128 + +static page_wait_block_header_t wait_blocks[WAIT_BLOCK_COUNT]; size_t mm_cache_cached_pages; -#define HOLD_LOCK() \ - MUTEX_ACQUIRE(&mutex); +static page_wait_block_header_t *page_wait_pick_header(page_t *page) { + return &wait_blocks[(uintptr_t)mm_get_page_address(page) / PAGE_SIZE % WAIT_BLOCK_COUNT]; // TODO: maybe use a hash +} -#define RELEASE_LOCK() \ - MUTEX_RELEASE(&mutex); +static void page_wait_attach(page_t *page, page_wait_block_t *page_wait_block) { + page_wait_block_header_t *header = page_wait_pick_header(page); -static inline uint64_t fnv1a_hash(void *buffer, size_t size) { - uint8_t *ptr = buffer; - uint8_t *top = ptr + size; - uint64_t h = FNV1OFFSET; + SEMAPHORE_INIT(&page_wait_block->semaphore, 0); + page_wait_block->page = page; - while (ptr < top) { - h ^= *ptr++; - h *= FNV1PRIME; + MUTEX_ACQUIRE(&header->mutex); + list_push_back(&header->wait_block_list, &page_wait_block->list_node); + MUTEX_RELEASE(&header->mutex); +} + +static void page_wait_detach(page_wait_block_t *page_wait_block) { + page_wait_block_header_t *header = page_wait_pick_header(page_wait_block->page); + + MUTEX_ACQUIRE(&header->mutex); + list_remove(&header->wait_block_list, &page_wait_block->list_node); + MUTEX_RELEASE(&header->mutex); +} + +static void page_wait(page_wait_block_t *page_wait_block) { + semaphore_wait(&page_wait_block->semaphore, false); +} + +static void page_notify_waiters(page_t *page) { + page_wait_block_header_t *header = page_wait_pick_header(page); + + MUTEX_ACQUIRE(&header->mutex); + + list_for_each(&header->wait_block_list, list_node) { + page_wait_block_t *block = (page_wait_block_t *)list_node; + + if (block->page == page) + semaphore_signal(&block->semaphore); } - return h; + MUTEX_RELEASE(&header->mutex); } -static uintmax_t get_entry(vnode_t *vnode, uintmax_t offset) { - struct { - vnode_t *vnode; - uintmax_t offset; - } tmp; - - tmp.vnode = vnode; - tmp.offset = offset; - - return fnv1a_hash(&tmp, sizeof(tmp)) % TABLE_SIZE; +// expects vnode pushlock with shared+ acquire +static page_t *lookup_page(vnode_t *vnode, uintmax_t offset) { + void *v; + return trie_lookup(&vnode->pages, offset / PAGE_SIZE, &v) ? NULL : v; } -// assumes lock is held -static page_t *find_page(vnode_t *vnode, uintmax_t offset) { - uintmax_t entry = get_entry(vnode, offset); +// expects vnode pushlock with exclusive acquire +static int insert_page(vnode_t *vnode, uintmax_t offset, page_t *page, trie_preallocation_t *preallocation) { + int error = trie_insert_preallocated(&vnode->pages, offset / PAGE_SIZE, page, preallocation); + if (error == 0) + __atomic_add_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED); + return error; +} - page_t *page = table[entry]; - while (page) { - if (page->backing == vnode && page->offset == offset) - break; - page = page->hash_next; +static void remove_page(vnode_t *vnode, uintmax_t offset) { + __assert(trie_remove(&vnode->pages, offset / PAGE_SIZE) == 0); + __atomic_sub_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED); +} + +static int wait_for_ready(page_t *page) { + for (;;) { + int flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE); + if (flags & PAGE_FLAGS_READY) { + if ((flags & PAGE_FLAGS_ERROR) == 0) + return 0; + + void *phys = mm_get_page_address(page); + mm_release_page(phys); + return EIO; + } + + page_wait_block_t page_wait_block; + page_wait_attach(page, &page_wait_block); + if (__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_READY) { + page_wait_detach(&page_wait_block); + continue; + } + + page_wait(&page_wait_block); + page_wait_detach(&page_wait_block); } - - return page; -} - -// assumes lock is held -static void put_page(page_t *page) { - uintmax_t entry = get_entry(page->backing, page->offset); - // add to table list - page->hash_prev = NULL; - page->hash_next = table[entry]; - if (table[entry]) - table[entry]->hash_prev = page; - - table[entry] = page; - - // add to vnode list - // XXX this could be a bit slow if a large portion of the file is in memory and it gets purged - page->vnode_prev = NULL; - page->vnode_next = page->backing->pages; - if (page->backing->pages) - page->backing->pages->vnode_prev = page; - - page->backing->pages = page; - ++mm_cache_cached_pages; -} - -// assumes lock is held -static void remove_page(page_t *page) { - uintmax_t entry = get_entry(page->backing, page->offset); - - // remove from table list - if (page->hash_next) - page->hash_next->hash_prev = page->hash_prev; - - if (page->hash_prev) - page->hash_prev->hash_next = page->hash_next; - else - table[entry] = page->hash_next; - - page->hash_next = NULL; - page->hash_prev = NULL; - - // remove from vnode list - if (page->vnode_next) - page->vnode_next->vnode_prev = page->vnode_prev; - - if (page->vnode_prev) - page->vnode_prev->vnode_next = page->vnode_next; - else - page->backing->pages = page->vnode_next; - - page->vnode_next = NULL; - page->vnode_prev = NULL; - --mm_cache_cached_pages; } +// TODO: clustered page in int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res) { - (void) flags; - __assert(vnode->type == V_TYPE_REGULAR || vnode->type == V_TYPE_BLKDEV); - __assert((offset % PAGE_SIZE) == 0); - retry_err: - HOLD_LOCK(); + pushlock_acquire_shared(&vnode->pages_lock); - page_t *new_page = NULL; - volatile page_t *page = find_page(vnode, offset); - retry: + page_t *page = lookup_page(vnode, offset); + if (page) + mm_hold_page(mm_get_page_address(page)); + + pushlock_release_shared(&vnode->pages_lock); + +retry_page: if (page) { - // page is present in the page cache - mm_hold_page(mm_get_page_address((page_t *)page)); - RELEASE_LOCK(); + *res = page; + return wait_for_ready(page); + } else if (flags & MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE) + return ENOENT; - // in the case of a retry, release the allocated page here - if (new_page) - mm_release_page(mm_get_page_address(new_page)); + void *phys = mm_alloc_page(MEMORY_SECTION_DEFAULT); + if (phys == NULL) + return ENOMEM; - eventlistener_t listener; - EVENT_INITLISTENER(&listener); - EVENT_ATTACH(&listener, &page_ready_event); + page = mm_get_page(phys); - // wait for page to be ready - while ((page->flags & (PAGE_FLAGS_READY | PAGE_FLAGS_ERROR)) == 0) - EVENT_WAIT(&listener, 0); + trie_preallocation_t *trie_preallocation = trie_preallocate(); + if (trie_preallocation == NULL) { + mm_release_page(phys); + return ENOMEM; + } - EVENT_DETACHALL(&listener); + pushlock_acquire_exclusive(&vnode->pages_lock); + page_t *tmp = lookup_page(vnode, offset); + if (tmp) { + mm_hold_page(mm_get_page_address(tmp)); + pushlock_release_exclusive(&vnode->pages_lock); + trie_free_preallocation(trie_preallocation); + mm_release_page(phys); + page = tmp; + goto retry_page; + } - if (page->flags & PAGE_FLAGS_ERROR) { - // the thread handling the page in failed to read it, we should retry it and see whats up - mm_release_page(mm_get_page_address((page_t *)page)); - goto retry_err; - } + __assert(insert_page(vnode, offset, page, trie_preallocation) == 0); + pushlock_release_exclusive(&vnode->pages_lock); - *res = (page_t *)page; + VOP_LOCK(vnode); + int error = VOP_GETPAGE(vnode, offset, page); + VOP_UNLOCK(vnode); + if (error) { + pushlock_acquire_exclusive(&vnode->pages_lock); + remove_page(vnode, offset); + pushlock_release_exclusive(&vnode->pages_lock); } else { - // page is not present in the cache, we will have to load it in - RELEASE_LOCK(); - - void *address = mm_alloc_page(MEMORY_SECTION_DEFAULT); - if (address == NULL) - return ENOMEM; - - new_page = mm_get_page(address); - - HOLD_LOCK(); - - // while the lock wasn't being held, the page could have potentially been added to the cache - // check for it again and return from the function as if it was always there in the first place - page = find_page(vnode, offset); - if (page) - goto retry; - - new_page->backing = vnode; - new_page->offset = offset; - - // add it to the page cache - put_page(new_page); - - RELEASE_LOCK(); - - VOP_LOCK(vnode); - int error = VOP_GETPAGE(vnode, offset, new_page); - VOP_UNLOCK(vnode); - - if (error) { - // an error happened with GETPAGE, remove the page from the cache, - // tell the sleeping threads that something happened and free the page - // by setting backing to null so it gets treated as an anonymous page again - HOLD_LOCK(); - remove_page(new_page); - - new_page->flags |= PAGE_FLAGS_ERROR; - new_page->backing = NULL; - new_page->offset = 0; - - RELEASE_LOCK(); - mm_release_page(mm_get_page_address(new_page)); - EVENT_SIGNAL(&page_ready_event); - return error; - } - - HOLD_LOCK(); - new_page->flags |= PAGE_FLAGS_READY; - RELEASE_LOCK(); - - EVENT_SIGNAL(&page_ready_event); - *res = new_page; + page->backing = vnode; + page->offset = offset; } - return 0; + __atomic_or_fetch(&page->flags, PAGE_FLAGS_READY | (error ? PAGE_FLAGS_ERROR : 0), __ATOMIC_RELEASE); + page_notify_waiters(page); + *res = page; + trie_free_preallocation(trie_preallocation); + if (error) { + mm_release_page(phys); + } + + return error; } -// adds a page to the cache in a specific offset if its not already there -int mm_cache_push_page(vnode_t *vnode, uintmax_t offset, page_t *page) { - __assert((offset % PAGE_SIZE) == 0); - HOLD_LOCK(); - - page_t *page_test = find_page(vnode, offset); - if (page_test) { - RELEASE_LOCK(); - return EAGAIN; - } - - page->backing = vnode; - page->offset = offset; - page->flags |= PAGE_FLAGS_READY; - - put_page(page); - - RELEASE_LOCK(); - return 0; -} - -// removes a page from the cache AND turns it into anonymous memory -int mm_cache_evict(page_t *page) { - HOLD_LOCK(); - if (page->refcount > 1) { - RELEASE_LOCK(); - return EAGAIN; - } - __assert(page->refcount == 1); - __assert((page->flags & PAGE_FLAGS_DIRTY) == 0); - - if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) { - // the page needs to be removed from the cache to continue - remove_page(page); - } - - page->flags = 0; - page->backing = NULL; - page->offset = 0; - - RELEASE_LOCK(); - return 0; -} - -// removes a page from the cache *but doesn't do anything to it* +// removes a page from the page cache if only the caller holds a reference +// keeps the page in the same state as if it were in the page cache int mm_cache_take_page(page_t *page) { - HOLD_LOCK(); - // someone called mm_cache_get_page() and got this page while the lock wasn't held - // return an error status to the caller + pushlock_acquire_exclusive(&page->backing->pages_lock); if (page->refcount > 1) { - RELEASE_LOCK(); + pushlock_release_exclusive(&page->backing->pages_lock); return EAGAIN; } - __assert(page->refcount == 1); - __assert((page->flags & PAGE_FLAGS_DIRTY) == 0); + __assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_DIRTY) == 0); + // TODO: remove the need for this check (i.e. transform truncated pages into anonymous pages) + if ((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_TRUNCATED) == 0) + remove_page(page->backing, page->offset); - // XXX maybe just not allow truncated pages to take up space like this? - if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) { - // the page needs to be removed from the cache to continue - remove_page(page); - } - - RELEASE_LOCK(); + pushlock_release_exclusive(&page->backing->pages_lock); return 0; } +static void truncate_iterate(void *p) { + page_t *page = p; + __atomic_sub_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED); + if (__atomic_or_fetch(&page->flags, PAGE_FLAGS_TRUNCATED, __ATOMIC_RELEASE) & PAGE_FLAGS_PINNED) { + mm_release_page(mm_get_page_address(page)); + } +} + int mm_cache_truncate(vnode_t *vnode, uintmax_t offset) { - HOLD_LOCK(); - page_t *page_list = NULL; - page_t *page = vnode->pages; + uintmax_t min_page = ROUND_UP(offset, PAGE_SIZE) / PAGE_SIZE; - while (page) { - page_t *old_page = page; - page = page->vnode_next; + pushlock_acquire_exclusive(&vnode->pages_lock); - // only truncate past a certain offset - if (old_page->offset < offset) - continue; - - old_page->flags |= PAGE_FLAGS_TRUNCATED; - remove_page(old_page); - old_page->vnode_next = page_list; - page_list = old_page; - } - - RELEASE_LOCK(); - - // make sure to unref if they are pinned - while (page_list) { - page_t *page = page_list; - page_list = page_list->vnode_next; - if (page->flags & PAGE_FLAGS_PINNED) - mm_release_page(mm_get_page_address(page)); - } + trie_iterate(&vnode->pages, min_page, UINT64_MAX, truncate_iterate); + trie_truncate(&vnode->pages, min_page); + pushlock_release_exclusive(&vnode->pages_lock); return 0; } -// called with lock held -// returns with lock released -// expects backing lock to be held -static int sync_page(page_t *page, bool backing_lock) { - __assert(page->flags & PAGE_FLAGS_DIRTY); - page->flags &= ~PAGE_FLAGS_DIRTY; - RELEASE_LOCK(); - int e = 0; - if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) { - if (backing_lock) - VOP_LOCK(page->backing); - - e = VOP_PUTPAGE(page->backing, page->offset, (page_t *)page); - - if (backing_lock) - VOP_UNLOCK(page->backing); - VOP_RELEASE(page->backing); - } else { - // page got truncated from the file while waiting to be written to disk - // its still holding a reference to the vnode, so release that - VOP_RELEASE(page->backing); +static void mm_cache_init(void) { + for (size_t i = 0; i < WAIT_BLOCK_COUNT; ++i) { + MUTEX_INIT(&wait_blocks[i].mutex); + list_init(&wait_blocks[i].wait_block_list); } - mm_release_page(mm_get_page_address((page_t *)page)); - return e; + mm_cache_init_writer(); } -static page_t *dirty_list; -static page_t *dirty_list_end; - -// expects vnode to be held -int mm_cache_sync_vnode(vnode_t *vnode, uintmax_t offset, size_t size) { - offset = ROUND_DOWN(offset, PAGE_SIZE); - uintmax_t top = offset + size; - // overflow check - __assert(top > offset); - HOLD_LOCK(); - - // loop through all vnode pages in memory and check which ones are in the range and are dirty - // TODO create a proper vnode dirty list as to not have to loop through the ENTIRE thing in memory - page_t *page = vnode->pages; - page_t *vnode_dirty_list = NULL; - for (; page; page = page->vnode_next) { - if (page->offset < offset || page->offset >= top || (page->flags & PAGE_FLAGS_DIRTY) == 0 || (page->flags & PAGE_FLAGS_VNODE_SYNCING)) - continue; - - // remove from write list and add to an internal list using the write pointers - // in a singly linked list way - if (page->write_next) - page->write_next->write_prev = page->write_prev; - else - dirty_list_end = page->write_prev; - - if (page->write_prev) - page->write_prev->write_next = page->write_next; - else - dirty_list = page->write_next; - - page->write_next = vnode_dirty_list; - page->write_prev = NULL; - page->flags |= PAGE_FLAGS_VNODE_SYNCING; - vnode_dirty_list = page; - } - - RELEASE_LOCK(); - - int e = 0; - while (vnode_dirty_list) { - // in the case of failure, only the first error to occur will be reported and we will not - // retry the write and keep on syncing the pages to disk - HOLD_LOCK(); - page_t *page = vnode_dirty_list; - vnode_dirty_list = vnode_dirty_list->write_next; - page->write_next = NULL; - page->flags &= ~PAGE_FLAGS_VNODE_SYNCING; - - // another thread could already have synced this page, verify if it is still dirty - if (page->flags & PAGE_FLAGS_DIRTY) { - int error = sync_page(page, false); - - if (e == 0) - e = error; - } else { - RELEASE_LOCK(); - } - // sync_page returns with lock released - } - - return e; -} - -int mm_cache_sync(void) { - eventlistener_t event_listener; - EVENT_INITLISTENER(&event_listener); - HOLD_LOCK(); - if (dirty_list == NULL) { - // no dirty pages - RELEASE_LOCK(); - return 0; - } - - EVENT_ATTACH(&event_listener, &sync_event); - semaphore_signal(&sync); - RELEASE_LOCK(); - - EVENT_WAIT(&event_listener, 0); - - EVENT_DETACHALL(&event_listener); - return 0; -} - -// backing expected locked -int mm_cache_make_dirty(page_t *page) { - bool made_dirty = false; - HOLD_LOCK(); - - if ((page->flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_TRUNCATED)) == 0) { - made_dirty = true; - // page is neither dirty nor truncated, add to dirty list and hold the page and vnode - page->flags |= PAGE_FLAGS_DIRTY; - - if ((page->flags & PAGE_FLAGS_VNODE_SYNCING) == 0) { - page->write_prev = NULL; - page->write_next = dirty_list; - if (dirty_list) - dirty_list->write_prev = page; - else - dirty_list_end = page; - - dirty_list = page; - } - - mm_hold_page(mm_get_page_address(page)); - __assert(page->backing); - VOP_HOLD(page->backing); - } - - RELEASE_LOCK(); - if (made_dirty) { - vattr_t attr; - attr.mtime = timekeeper_time(); - VOP_SETATTR(page->backing, &attr, V_ATTR_MTIME, NULL); - } - return 0; -} - -static void tick(context_t *, dpcarg_t arg) { - semaphore_signal(&sync); -} - -static void writer() { - timerentry_t timer_entry; - // this will be inserted on some random cpu's timer, but it will always work after that - interrupt_set(false); - timer_insert(current_cpu()->timer, &timer_entry, tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true); - interrupt_set(true); - for (;;) { - HOLD_LOCK(); - volatile page_t *page = dirty_list_end; - if (page == NULL) { - EVENT_SIGNAL(&sync_event); - RELEASE_LOCK(); - semaphore_wait(&sync, false); - continue; - } - - dirty_list_end = page->write_prev; - if (dirty_list_end) - dirty_list_end->write_next = NULL; - else - dirty_list = NULL; - - page->write_prev = NULL; - // TODO notify error on mm_cache_sync_vnode - sync_page((page_t *)page, true); - } -} - -void mm_cache_init(void) { - MUTEX_INIT(&mutex); - table = mm_map(NULL, TABLE_SIZE * sizeof(page_t *), MM_RANGE_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); - __assert(table); - memset(table, 0, TABLE_SIZE * sizeof(page_t *)); - - SEMAPHORE_INIT(&sync, 0); - writer_thread = sched_newthread(writer, PAGE_SIZE * 16, 1, NULL, NULL); - __assert(writer_thread); - sched_queue(writer_thread); - mm_cache_sync(); - EVENT_INITHEADER(&sync_event); - EVENT_INITHEADER(&page_ready_event); -} - -INIT_ROUTINE_DEFINE(mm_cache, INIT_ROUTINE_FLAGS_NONE, mm_cache_init, scheduler); +INIT_ROUTINE_DEFINE(mm_cache, INIT_ROUTINE_FLAGS_NONE, mm_cache_init, scheduler, trie); diff --git a/kernel-src/mm/mm.c b/kernel-src/mm/mm.c index 158382f1..1275a076 100644 --- a/kernel-src/mm/mm.c +++ b/kernel-src/mm/mm.c @@ -163,11 +163,8 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) { if ((!private || (!file_mapping && current_page->refcount == 1)) && mapped_address != zero_page) { arch_mmu_remap(current_mm_context()->pagetable, mapped_address, vaddr, range->mmuflags); - if (file_mapping && cacheable) { - VOP_LOCK(range->vnode); + if (file_mapping && cacheable) mm_cache_make_dirty(current_page); - VOP_UNLOCK(range->vnode); - } *resulting_page = current_page; return 0; @@ -228,11 +225,8 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) { return ENOMEM; } - if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE)) { - VOP_LOCK(range->vnode); + if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE)) mm_cache_make_dirty(vn_page); - VOP_UNLOCK(range->vnode); - } *resulting_page = vn_page; return 0; diff --git a/kernel-src/mm/page.c b/kernel-src/mm/page.c index 71559ffe..ff1d0328 100644 --- a/kernel-src/mm/page.c +++ b/kernel-src/mm/page.c @@ -119,7 +119,7 @@ static void remove_from_free_list(page_t *page) { static void internal_hold(page_t *page) { if (++page->refcount == 1) { // this is only valid on standby pages, in case of free pages its an use after free - __assert((page->flags & PAGE_FLAGS_FREE) == 0); + __assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_FREE) == 0); remove_from_free_list(page); } } @@ -147,10 +147,10 @@ void mm_release_page(void *addr) { __assert(page->refcount != 0); if (--page->refcount == 0) { __assert(!mm_is_page_locked(page)); - __assert((page->flags & PAGE_FLAGS_DIRTY) == 0); + __assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_DIRTY) == 0); insert_in_free_list(page); if (page->backing == NULL) - page->flags |= PAGE_FLAGS_FREE; + __atomic_or_fetch(&page->flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED); } MUTEX_RELEASE(&free_list_mutex); } @@ -236,7 +236,7 @@ void mm_force_free_page(void *address, size_t count) { uintmax_t page_id = base_id + i; PAGE_BOUNDARY_CHECK(page_id); page_t *page = &pages[page_id]; - page->flags |= PAGE_FLAGS_FREE; + __atomic_or_fetch(&page->flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED); insert_in_free_list(page); } MUTEX_RELEASE(&free_list_mutex); @@ -277,7 +277,7 @@ void mm_page_init() { if (e->type == LIMINE_MEMMAP_USABLE) { int first_usable_page = e == biggest ? ROUND_UP(e->base + page_count * sizeof(page_t), PAGE_SIZE) / PAGE_SIZE : e->base / PAGE_SIZE; for (int i = first_usable_page; i < (e->base + e->length) / PAGE_SIZE; ++i) { - pages[i].flags |= PAGE_FLAGS_FREE; + __atomic_or_fetch(&pages[i].flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED); insert_in_free_list(&pages[i]); } } else if (e->type == LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE) { @@ -311,7 +311,7 @@ void *mm_alloc_pages(size_t size, int section) { if (page >= page_count) break; - if (pages[page].flags & PAGE_FLAGS_FREE) { + if (__atomic_load_n(&pages[page].flags, __ATOMIC_RELAXED) & PAGE_FLAGS_FREE) { ++found; } else found = 0; diff --git a/kernel-src/mm/range.c b/kernel-src/mm/range.c index 93414034..a99576df 100644 --- a/kernel-src/mm/range.c +++ b/kernel-src/mm/range.c @@ -165,9 +165,7 @@ void mm_destroy_range(mm_range_t *range, uintmax_t _offset, size_t size, int fla } else if (arch_mmu_iswritable(current_mm_context()->pagetable, vaddr)) { // dirty page cache mapping arch_mmu_unmap(current_mm_context()->pagetable, vaddr); - VOP_LOCK(range->vnode); mm_cache_make_dirty(mm_get_page(physical)); - VOP_UNLOCK(range->vnode); mm_release_page(physical); } else { // non dirty page mapping @@ -212,9 +210,7 @@ static void change_mmu_range(mm_range_t *range, void *base, size_t size, mmuflag // removing write permissions from a writeable dirty shared mapped page, mark it as dirty, as // it won't be marked dirty upon a mm_unmap after this page_t *page = mm_get_page(physical); - VOP_LOCK(range->vnode); mm_cache_make_dirty(page); - VOP_UNLOCK(range->vnode); } // we will only change the mapping if the permissions decreased diff --git a/kernel-src/mm/writeback.c b/kernel-src/mm/writeback.c new file mode 100644 index 00000000..e3e9b7f7 --- /dev/null +++ b/kernel-src/mm/writeback.c @@ -0,0 +1,206 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define WRITER_TICK_SECONDS 15 + +static mutex_t vnode_dirty_list_mutex; +static list_t vnode_dirty_list; +static bool writer_syncing; +static eventheader_t writer_wakeup_event; +static eventheader_t sync_complete_event; + +// OK +void mm_cache_make_dirty(page_t *page) { + vnode_t *vnode = page->backing; + MUTEX_ACQUIRE(&vnode->dirty_list_mutex); + + int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED); + for (;;) { + if (flags & PAGE_FLAGS_TRUNCATED) { + MUTEX_RELEASE(&vnode->dirty_list_mutex); + return; + } + + int new_flags = flags | PAGE_FLAGS_DIRTY; + if (__atomic_compare_exchange_n(&page->flags, &flags, new_flags, true, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) + break; + } + + if (flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_SYNCING)) { + MUTEX_RELEASE(&vnode->dirty_list_mutex); + return; + } + + mm_hold_page(mm_get_page_address(page)); + list_push_back(&vnode->dirty_pages, &page->dirty_list_node); + + if (!vnode->dirty) { + vnode->dirty = true; + VOP_HOLD(vnode); + + MUTEX_ACQUIRE(&vnode_dirty_list_mutex); + list_push_back(&vnode_dirty_list, &vnode->vnode_dirty_list_node); + MUTEX_RELEASE(&vnode_dirty_list_mutex); + } + + MUTEX_RELEASE(&vnode->dirty_list_mutex); + + vattr_t attr; + attr.mtime = timekeeper_time(); + VOP_LOCK(vnode); + VOP_SETATTR(vnode, &attr, V_ATTR_MTIME, NULL); + VOP_UNLOCK(vnode); +} + +// OK +static void mark_page_syncing(page_t *page) { + int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED); + for (;;) { + __assert(flags & PAGE_FLAGS_DIRTY); + __assert((flags & PAGE_FLAGS_SYNCING) == 0); + + int new_flags = (flags & ~PAGE_FLAGS_DIRTY) | PAGE_FLAGS_SYNCING; + if (__atomic_compare_exchange_n(&page->flags, &flags, new_flags, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) + break; + } +} + +// OK +// expects the per-vnode dirty list mutex to be held. returns true if the caller must release a vnode reference. +static bool try_undirty_vnode(vnode_t *vnode) { + if (vnode->dirty == false) + return false; + + MUTEX_ACQUIRE(&vnode_dirty_list_mutex); + list_remove(&vnode_dirty_list, &vnode->vnode_dirty_list_node); + MUTEX_RELEASE(&vnode_dirty_list_mutex); + vnode->dirty = false; + return true; +} + +// OK +int mm_cache_sync_vnode(vnode_t *vnode) { + int first_error = 0; + MUTEX_ACQUIRE(&vnode->writeback_mutex); + + for (;;) { + MUTEX_ACQUIRE(&vnode->dirty_list_mutex); + list_node_t *page_node = list_pop_front(&vnode->dirty_pages); + if (page_node == NULL) { + bool was_dirty = try_undirty_vnode(vnode); + MUTEX_RELEASE(&vnode->dirty_list_mutex); + MUTEX_RELEASE(&vnode->writeback_mutex); + + if (was_dirty) { + VOP_RELEASE(vnode); + } + return first_error; + } + + page_t *page = container_of(page_node, page_t, dirty_list_node); + mark_page_syncing(page); + MUTEX_RELEASE(&vnode->dirty_list_mutex); + + VOP_LOCK(vnode); + + // TODO: remark as dirty on error + int error = 0; + if ((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_TRUNCATED) == 0) + error = VOP_PUTPAGE(vnode, page->offset, page); + + VOP_UNLOCK(vnode); + if (first_error == 0) + first_error = error; + + int flags = __atomic_and_fetch(&page->flags, ~PAGE_FLAGS_SYNCING, __ATOMIC_ACQUIRE); + if (flags & PAGE_FLAGS_DIRTY) { + MUTEX_ACQUIRE(&vnode->dirty_list_mutex); + list_push_back(&vnode->dirty_pages, &page->dirty_list_node); + MUTEX_RELEASE(&vnode->dirty_list_mutex); + } else { + mm_release_page(mm_get_page_address(page)); + } + } +} + +// OK +int mm_cache_sync(void) { + MUTEX_ACQUIRE(&vnode_dirty_list_mutex); + + if (vnode_dirty_list.head == NULL && !writer_syncing) { + MUTEX_RELEASE(&vnode_dirty_list_mutex); + return 0; + } + + eventlistener_t listener; + EVENT_INITLISTENER(&listener); + EVENT_ATTACH(&listener, &sync_complete_event); + + EVENT_SIGNAL(&writer_wakeup_event); + MUTEX_RELEASE(&vnode_dirty_list_mutex); + + EVENT_WAIT(&listener, 0); + EVENT_DETACHALL(&listener); + return 0; +} + +// OK +static void writer_tick(context_t *, dpcarg_t) { + EVENT_SIGNAL(&writer_wakeup_event); +} + +// OK +static void writer(void) { + long ipl = interrupt_raiseipl(IPL_DPC); + timerentry_t timer_entry; + timer_insert(current_cpu()->timer, &timer_entry, writer_tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true); + interrupt_loweripl(ipl); + + for (;;) { + eventlistener_t listener; + EVENT_INITLISTENER(&listener); + + MUTEX_ACQUIRE(&vnode_dirty_list_mutex); + list_node_t *list_node = vnode_dirty_list.head; + if (list_node == NULL) { + writer_syncing = false; + EVENT_ATTACH(&listener, &writer_wakeup_event); + EVENT_SIGNAL(&sync_complete_event); + MUTEX_RELEASE(&vnode_dirty_list_mutex); + + EVENT_WAIT(&listener, 0); + EVENT_DETACHALL(&listener); + continue; + } + + vnode_t *vnode = container_of(list_node, vnode_t, vnode_dirty_list_node); + VOP_HOLD(vnode); + + writer_syncing = true; + MUTEX_RELEASE(&vnode_dirty_list_mutex); + + mm_cache_sync_vnode(vnode); + + VOP_RELEASE(vnode); + } +} + +// OK +void mm_cache_init_writer(void) { + MUTEX_INIT(&vnode_dirty_list_mutex); + list_init(&vnode_dirty_list); + writer_syncing = false; + EVENT_INITHEADER(&writer_wakeup_event); + EVENT_INITHEADER(&sync_complete_event); + + thread_t *writer_thread = sched_newthread(writer, PAGE_SIZE * 2, 0, NULL, NULL); + __assert(writer_thread); + sched_queue(writer_thread); +} diff --git a/kernel-src/sys/syscalls/sync.c b/kernel-src/sys/syscalls/sync.c index 7af46501..4f8be42d 100644 --- a/kernel-src/sys/syscalls/sync.c +++ b/kernel-src/sys/syscalls/sync.c @@ -19,9 +19,7 @@ syscallret_t syscall_fsync(context_t *context, int fd) { return ret; } - VOP_LOCK(file->vnode); ret.errno = VOP_SYNC(file->vnode); - VOP_UNLOCK(file->vnode); fd_release(file); ret.ret = ret.errno ? -1 : 0;