From 7217a1e258204cf5b4ea1ddbaed3c1d1f5a86b3f Mon Sep 17 00:00:00 2001 From: Matheus Date: Thu, 20 Aug 2026 22:36:38 -0300 Subject: [PATCH] page cache: proper truncation --- kernel-src/fs/vfs.c | 4 +- kernel-src/include/kernel/mm.h | 5 +- kernel-src/include/kernel/page.h | 2 + kernel-src/include/kernel/vfs.h | 4 + kernel-src/mm/cache.c | 139 +++++++++++++++++++++++-------- kernel-src/mm/mm.c | 4 +- kernel-src/mm/page.c | 55 +++++++++--- kernel-src/mm/range.c | 4 +- kernel-src/mm/writeback.c | 11 ++- 9 files changed, 170 insertions(+), 58 deletions(-) diff --git a/kernel-src/fs/vfs.c b/kernel-src/fs/vfs.c index 810e6558..0f423bc4 100644 --- a/kernel-src/fs/vfs.c +++ b/kernel-src/fs/vfs.c @@ -508,7 +508,7 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size goto leave; } - mm_cache_make_dirty(page); + mm_cache_make_dirty(node, page); *written += writesize; pageoffset += 1; @@ -532,7 +532,7 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size goto leave; } - mm_cache_make_dirty(page); + mm_cache_make_dirty(node, page); *written += writesize; mm_release_page(FROM_HHDM(address)); diff --git a/kernel-src/include/kernel/mm.h b/kernel-src/include/kernel/mm.h index 6d80bdd3..e876a0e7 100644 --- a/kernel-src/include/kernel/mm.h +++ b/kernel-src/include/kernel/mm.h @@ -99,10 +99,11 @@ void mm_init(); extern size_t mm_cache_cached_pages; +// the mm_cache_* functions that take a vnode NEED it to be taken from a source which is not from the page_t itself #define MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE 1 int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res); -int mm_cache_take_page(page_t *page); -void mm_cache_make_dirty(page_t *page); +void mm_cache_take_page(page_t *page); +void mm_cache_make_dirty(vnode_t *vnode, page_t *page); int mm_cache_truncate(vnode_t *vnode, uintmax_t offset); int mm_cache_sync_vnode(vnode_t *vnode); int mm_cache_sync(void); diff --git a/kernel-src/include/kernel/page.h b/kernel-src/include/kernel/page.h index 1c6f4589..263bf95e 100644 --- a/kernel-src/include/kernel/page.h +++ b/kernel-src/include/kernel/page.h @@ -18,6 +18,7 @@ #define PAGE_FLAGS_READY 16 #define PAGE_FLAGS_ERROR 32 #define PAGE_FLAGS_SYNCING 64 +#define PAGE_FLAGS_RECLAIMING 128 typedef struct page_t { struct vnode_t *backing; @@ -47,6 +48,7 @@ void mm_page_init(); void mm_get_page_statistics(size_t *total_pages, size_t *free_pages); void mm_unlock_page(page_t *page); bool mm_is_page_locked(page_t *page); +void mm_make_page_anonymous(page_t *page); extern uintptr_t hhdm_base; diff --git a/kernel-src/include/kernel/vfs.h b/kernel-src/include/kernel/vfs.h index 9ae8ab77..415c29a1 100644 --- a/kernel-src/include/kernel/vfs.h +++ b/kernel-src/include/kernel/vfs.h @@ -147,6 +147,8 @@ typedef struct vnode_t { pushlock_t pages_lock; trie_t pages; + size_t page_reclaim_handoffs; + eventheader_t zero_reclaim_event; mutex_t adv_mutex; advlock_t *advlock; @@ -222,6 +224,8 @@ typedef struct vops_t { (vn)->vfsmounted = NULL; \ trie_init(&(vn)->pages); \ (vn)->pages_lock = 0; \ + (vn)->page_reclaim_handoffs = 0; \ + EVENT_INITHEADER(&(vn)->zero_reclaim_event); \ MUTEX_INIT(&(vn)->writeback_mutex); \ MUTEX_INIT(&(vn)->dirty_list_mutex); \ (vn)->dirty = false; \ diff --git a/kernel-src/mm/cache.c b/kernel-src/mm/cache.c index 8101315e..714eba04 100644 --- a/kernel-src/mm/cache.c +++ b/kernel-src/mm/cache.c @@ -19,6 +19,8 @@ typedef struct { static page_wait_block_header_t wait_blocks[WAIT_BLOCK_COUNT]; size_t mm_cache_cached_pages; +static page_t *lookup_page(vnode_t *vnode, uintmax_t offset); + static page_wait_block_header_t *page_wait_pick_header(page_t *page) { return &wait_blocks[(uintptr_t)mm_get_page_address(page) / PAGE_SIZE % WAIT_BLOCK_COUNT]; // TODO: maybe use a hash } @@ -61,6 +63,34 @@ static void page_notify_waiters(page_t *page) { MUTEX_RELEASE(&header->mutex); } +static page_t *lookup_and_hold_page(vnode_t *vnode, uintmax_t offset) { + for (;;) { + pushlock_acquire_shared(&vnode->pages_lock); + + page_t *page = lookup_page(vnode, offset); + if (page == NULL) { + pushlock_release_shared(&vnode->pages_lock); + return NULL; + } + + mm_hold_page(mm_get_page_address(page)); + if ((__atomic_load_n(&page->flags, __ATOMIC_ACQUIRE) & PAGE_FLAGS_RECLAIMING) == 0) { + pushlock_release_shared(&vnode->pages_lock); + return page; + } + + page_wait_block_t page_wait_block; + page_wait_attach(page, &page_wait_block); + mm_release_page(mm_get_page_address(page)); + pushlock_release_shared(&vnode->pages_lock); + + if (__atomic_load_n(&page->flags, __ATOMIC_ACQUIRE) & PAGE_FLAGS_RECLAIMING) + page_wait(&page_wait_block); + + page_wait_detach(&page_wait_block); + } +} + // expects vnode pushlock with shared+ acquire static page_t *lookup_page(vnode_t *vnode, uintmax_t offset) { void *v; @@ -106,20 +136,14 @@ static int wait_for_ready(page_t *page) { // TODO: clustered page in int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res) { - pushlock_acquire_shared(&vnode->pages_lock); - - page_t *page = lookup_page(vnode, offset); - if (page) - mm_hold_page(mm_get_page_address(page)); - - pushlock_release_shared(&vnode->pages_lock); - -retry_page: + retry_lookup: + page_t *page = lookup_and_hold_page(vnode, offset); if (page) { *res = page; return wait_for_ready(page); - } else if (flags & MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE) + } else if (flags & MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE) { return ENOENT; + } void *phys = mm_alloc_page(MEMORY_SECTION_DEFAULT); if (phys == NULL) @@ -136,12 +160,10 @@ retry_page: pushlock_acquire_exclusive(&vnode->pages_lock); page_t *tmp = lookup_page(vnode, offset); if (tmp) { - mm_hold_page(mm_get_page_address(tmp)); pushlock_release_exclusive(&vnode->pages_lock); trie_free_preallocation(trie_preallocation); mm_release_page(phys); - page = tmp; - goto retry_page; + goto retry_lookup; } __assert(insert_page(vnode, offset, page, trie_preallocation) == 0); @@ -150,50 +172,81 @@ retry_page: VOP_LOCK(vnode); int error = VOP_GETPAGE(vnode, offset, page); VOP_UNLOCK(vnode); - if (error) { - pushlock_acquire_exclusive(&vnode->pages_lock); + + bool release_pin = false; + pushlock_acquire_exclusive(&vnode->pages_lock); + int page_flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE); + if (page_flags & PAGE_FLAGS_TRUNCATED) { + release_pin = __atomic_fetch_and(&page->flags, ~PAGE_FLAGS_PINNED, __ATOMIC_ACQ_REL) & PAGE_FLAGS_PINNED; + } else if (error) { remove_page(vnode, offset); - pushlock_release_exclusive(&vnode->pages_lock); + mm_make_page_anonymous(page); } else { page->backing = vnode; page->offset = offset; } + pushlock_release_exclusive(&vnode->pages_lock); + + if (release_pin) + mm_release_page(phys); __atomic_or_fetch(&page->flags, PAGE_FLAGS_READY | (error ? PAGE_FLAGS_ERROR : 0), __ATOMIC_RELEASE); page_notify_waiters(page); *res = page; trie_free_preallocation(trie_preallocation); - if (error) { + if (error) mm_release_page(phys); - } return error; } -// removes a page from the page cache if only the caller holds a reference -// keeps the page in the same state as if it were in the page cache -int mm_cache_take_page(page_t *page) { - pushlock_acquire_exclusive(&page->backing->pages_lock); - if (page->refcount > 1) { - pushlock_release_exclusive(&page->backing->pages_lock); - return EAGAIN; +void mm_cache_take_page(page_t *page) { + int flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE); + __assert(flags & PAGE_FLAGS_RECLAIMING); + __assert((flags & PAGE_FLAGS_DIRTY) == 0); + + vnode_t *vnode = page->backing; + __assert(vnode); + + bool signal = false; + pushlock_acquire_exclusive(&vnode->pages_lock); + flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE); + if ((flags & PAGE_FLAGS_TRUNCATED) == 0) + remove_page(vnode, page->offset); + + mm_make_page_anonymous(page); + if (flags & PAGE_FLAGS_TRUNCATED) { + __assert(vnode->page_reclaim_handoffs != 0); + if (--vnode->page_reclaim_handoffs == 0) + signal = true; } + pushlock_release_exclusive(&vnode->pages_lock); - __assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_DIRTY) == 0); - // TODO: remove the need for this check (i.e. transform truncated pages into anonymous pages) - if ((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_TRUNCATED) == 0) - remove_page(page->backing, page->offset); + if (signal) + EVENT_SIGNAL(&vnode->zero_reclaim_event); - pushlock_release_exclusive(&page->backing->pages_lock); - return 0; + __atomic_and_fetch(&page->flags, ~PAGE_FLAGS_RECLAIMING, __ATOMIC_RELEASE); + page_notify_waiters(page); } static void truncate_iterate(void *p) { page_t *page = p; __atomic_sub_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED); - if (__atomic_or_fetch(&page->flags, PAGE_FLAGS_TRUNCATED, __ATOMIC_RELEASE) & PAGE_FLAGS_PINNED) { - mm_release_page(mm_get_page_address(page)); + + int old_flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED); + for (;;) { + int new_flags = (old_flags | PAGE_FLAGS_TRUNCATED) & ~PAGE_FLAGS_PINNED; + if (__atomic_compare_exchange_n(&page->flags, &old_flags, new_flags, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) + break; } + + if (old_flags & PAGE_FLAGS_RECLAIMING) + ++page->backing->page_reclaim_handoffs; + else + mm_make_page_anonymous(page); + + if (old_flags & PAGE_FLAGS_PINNED) + mm_release_page(mm_get_page_address(page)); } int mm_cache_truncate(vnode_t *vnode, uintmax_t offset) { @@ -204,7 +257,27 @@ int mm_cache_truncate(vnode_t *vnode, uintmax_t offset) { trie_iterate(&vnode->pages, min_page, UINT64_MAX, truncate_iterate); trie_truncate(&vnode->pages, min_page); + bool wait = vnode->page_reclaim_handoffs != 0; + eventlistener_t listener; + if (wait) { + EVENT_INITLISTENER(&listener); + EVENT_ATTACH(&listener, &vnode->zero_reclaim_event); + } pushlock_release_exclusive(&vnode->pages_lock); + + // TODO use another abstraction for waiting here OR make interruptible sleep optional + while (wait) { + EVENT_WAIT(&listener, 0); + EVENT_DETACHALL(&listener); + + pushlock_acquire_exclusive(&vnode->pages_lock); + wait = vnode->page_reclaim_handoffs != 0; + if (wait) { + EVENT_INITLISTENER(&listener); + EVENT_ATTACH(&listener, &vnode->zero_reclaim_event); + } + pushlock_release_exclusive(&vnode->pages_lock); + } return 0; } diff --git a/kernel-src/mm/mm.c b/kernel-src/mm/mm.c index 1275a076..40c45461 100644 --- a/kernel-src/mm/mm.c +++ b/kernel-src/mm/mm.c @@ -164,7 +164,7 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) { arch_mmu_remap(current_mm_context()->pagetable, mapped_address, vaddr, range->mmuflags); if (file_mapping && cacheable) - mm_cache_make_dirty(current_page); + mm_cache_make_dirty(range->vnode, current_page); *resulting_page = current_page; return 0; @@ -226,7 +226,7 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) { } if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE)) - mm_cache_make_dirty(vn_page); + mm_cache_make_dirty(range->vnode, vn_page); *resulting_page = vn_page; return 0; diff --git a/kernel-src/mm/page.c b/kernel-src/mm/page.c index ff1d0328..21b52acc 100644 --- a/kernel-src/mm/page.c +++ b/kernel-src/mm/page.c @@ -124,6 +124,18 @@ static void internal_hold(page_t *page) { } } +static bool try_page_reclaim(page_t *page) { + int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED); + + for (;;) { + if (flags & PAGE_FLAGS_TRUNCATED) + return false; + + if (__atomic_compare_exchange_n(&page->flags, &flags, flags | PAGE_FLAGS_RECLAIMING, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) + return true; + } +} + page_t *mm_get_page(void *address) { return &pages[((uintptr_t)address / PAGE_SIZE)]; } @@ -155,6 +167,25 @@ void mm_release_page(void *addr) { MUTEX_RELEASE(&free_list_mutex); } +void mm_make_page_anonymous(page_t *page) { + MUTEX_ACQUIRE(&free_list_mutex); + + __assert(page->backing); + + bool on_standby = page->refcount == 0; + if (on_standby) + remove_from_free_list(page); + + page->backing = NULL; + + if (on_standby) { + __atomic_or_fetch(&page->flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED); + insert_in_free_list(page); + } + + MUTEX_RELEASE(&free_list_mutex); +} + void mm_unlock_and_release_page(void *addr) { page_t *page = &pages[(uintptr_t)addr / PAGE_SIZE]; mm_unlock_page(page); @@ -176,7 +207,6 @@ static void do_alloc(page_t *page) { } void *mm_alloc_page(int section) { - retry: MUTEX_ACQUIRE(&free_list_mutex); page_t *page = NULL; @@ -196,23 +226,22 @@ void *mm_alloc_page(int section) { if (page == NULL) { for (int i = section; i >= 0; --i) { page = standby_tails[i]; - if (page) { - cache_page = true; - internal_hold(page); - break; - } + while (page && try_page_reclaim(page) == false) + page = page->free_prev; + + if (page == NULL) + continue; + + cache_page = true; + internal_hold(page); + break; } } MUTEX_RELEASE(&free_list_mutex); - if (cache_page && mm_cache_take_page(page) == EAGAIN) { - // someone already got the page from the cache between us holding it and taking it - mm_release_page(mm_get_page_address(page)); - page = NULL; - // retry it from the start, as an anonymous page could have been released while the lock was not held - goto retry; - } else if (cache_page) { + if (cache_page) { + mm_cache_take_page(page); // we got the page from the cache, all is good and we hold the only reference to it. // set the refcount to 0, as expected by the do_alloc call page->refcount = 0; diff --git a/kernel-src/mm/range.c b/kernel-src/mm/range.c index a99576df..83787f63 100644 --- a/kernel-src/mm/range.c +++ b/kernel-src/mm/range.c @@ -165,7 +165,7 @@ void mm_destroy_range(mm_range_t *range, uintmax_t _offset, size_t size, int fla } else if (arch_mmu_iswritable(current_mm_context()->pagetable, vaddr)) { // dirty page cache mapping arch_mmu_unmap(current_mm_context()->pagetable, vaddr); - mm_cache_make_dirty(mm_get_page(physical)); + mm_cache_make_dirty(range->vnode, mm_get_page(physical)); mm_release_page(physical); } else { // non dirty page mapping @@ -210,7 +210,7 @@ static void change_mmu_range(mm_range_t *range, void *base, size_t size, mmuflag // removing write permissions from a writeable dirty shared mapped page, mark it as dirty, as // it won't be marked dirty upon a mm_unmap after this page_t *page = mm_get_page(physical); - mm_cache_make_dirty(page); + mm_cache_make_dirty(range->vnode, page); } // we will only change the mapping if the permissions decreased diff --git a/kernel-src/mm/writeback.c b/kernel-src/mm/writeback.c index e3e9b7f7..4cd81f31 100644 --- a/kernel-src/mm/writeback.c +++ b/kernel-src/mm/writeback.c @@ -16,12 +16,15 @@ static bool writer_syncing; static eventheader_t writer_wakeup_event; static eventheader_t sync_complete_event; -// OK -void mm_cache_make_dirty(page_t *page) { - vnode_t *vnode = page->backing; +void mm_cache_make_dirty(vnode_t *vnode, page_t *page) { + // check this here as a fast-path to reduce the mutex contention + int flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE); + if (flags & PAGE_FLAGS_TRUNCATED) + return; + MUTEX_ACQUIRE(&vnode->dirty_list_mutex); - int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED); + flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED); for (;;) { if (flags & PAGE_FLAGS_TRUNCATED) { MUTEX_RELEASE(&vnode->dirty_list_mutex);