page cache: proper truncation

kernel-src/fs/vfs.c

@@ -508,7 +508,7 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size
 				goto leave;
 			}
 
-			mm_cache_make_dirty(page);
+			mm_cache_make_dirty(node, page);
 			*written += writesize;
 
 			pageoffset += 1;
@@ -532,7 +532,7 @@ int vfs_write_iovec(vnode_t *node, iovec_iterator_t *iovec_iterator, size_t size
 				goto leave;
 			}
 
-			mm_cache_make_dirty(page);
+			mm_cache_make_dirty(node, page);
 			*written += writesize;
 
 			mm_release_page(FROM_HHDM(address));

kernel-src/include/kernel/mm.h

@@ -99,10 +99,11 @@ void mm_init();
 
 extern size_t mm_cache_cached_pages;
 
+// the mm_cache_* functions that take a vnode NEED it to be taken from a source which is not from the page_t itself
 #define MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE 1
 int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res);
-int mm_cache_take_page(page_t *page);
-void mm_cache_make_dirty(page_t *page);
+void mm_cache_take_page(page_t *page);
+void mm_cache_make_dirty(vnode_t *vnode, page_t *page);
 int mm_cache_truncate(vnode_t *vnode, uintmax_t offset);
 int mm_cache_sync_vnode(vnode_t *vnode);
 int mm_cache_sync(void);

kernel-src/include/kernel/page.h

@@ -18,6 +18,7 @@
 #define PAGE_FLAGS_READY 16
 #define PAGE_FLAGS_ERROR 32
 #define PAGE_FLAGS_SYNCING 64
+#define PAGE_FLAGS_RECLAIMING 128
 
 typedef struct page_t {
 	struct vnode_t *backing;
@@ -47,6 +48,7 @@ void mm_page_init();
 void mm_get_page_statistics(size_t *total_pages, size_t *free_pages);
 void mm_unlock_page(page_t *page);
 bool mm_is_page_locked(page_t *page);
+void mm_make_page_anonymous(page_t *page);
 
 extern uintptr_t hhdm_base;
 

kernel-src/include/kernel/vfs.h

@@ -147,6 +147,8 @@ typedef struct vnode_t {
 
 	pushlock_t pages_lock;
 	trie_t pages;
+	size_t page_reclaim_handoffs;
+	eventheader_t zero_reclaim_event;
 
 	mutex_t adv_mutex;
 	advlock_t *advlock;
@@ -222,6 +224,8 @@ typedef struct vops_t {
 	(vn)->vfsmounted = NULL; \
 	trie_init(&(vn)->pages); \
 	(vn)->pages_lock = 0; \
+	(vn)->page_reclaim_handoffs = 0; \
+	EVENT_INITHEADER(&(vn)->zero_reclaim_event); \
 	MUTEX_INIT(&(vn)->writeback_mutex); \
 	MUTEX_INIT(&(vn)->dirty_list_mutex); \
 	(vn)->dirty = false; \

kernel-src/mm/cache.c

@@ -19,6 +19,8 @@ typedef struct {
 static page_wait_block_header_t wait_blocks[WAIT_BLOCK_COUNT];
 size_t mm_cache_cached_pages;
 
+static page_t *lookup_page(vnode_t *vnode, uintmax_t offset);
+
 static page_wait_block_header_t *page_wait_pick_header(page_t *page) {
 	return &wait_blocks[(uintptr_t)mm_get_page_address(page) / PAGE_SIZE % WAIT_BLOCK_COUNT]; // TODO: maybe use a hash
 }
@@ -61,6 +63,34 @@ static void page_notify_waiters(page_t *page) {
 	MUTEX_RELEASE(&header->mutex);
 }
 
+static page_t *lookup_and_hold_page(vnode_t *vnode, uintmax_t offset) {
+	for (;;) {
+		pushlock_acquire_shared(&vnode->pages_lock);
+
+		page_t *page = lookup_page(vnode, offset);
+		if (page == NULL) {
+			pushlock_release_shared(&vnode->pages_lock);
+			return NULL;
+		}
+
+		mm_hold_page(mm_get_page_address(page));
+		if ((__atomic_load_n(&page->flags, __ATOMIC_ACQUIRE) & PAGE_FLAGS_RECLAIMING) == 0) {
+			pushlock_release_shared(&vnode->pages_lock);
+			return page;
+		}
+
+		page_wait_block_t page_wait_block;
+		page_wait_attach(page, &page_wait_block);
+		mm_release_page(mm_get_page_address(page));
+		pushlock_release_shared(&vnode->pages_lock);
+
+		if (__atomic_load_n(&page->flags, __ATOMIC_ACQUIRE) & PAGE_FLAGS_RECLAIMING)
+			page_wait(&page_wait_block);
+
+		page_wait_detach(&page_wait_block);
+	}
+}
+
 // expects vnode pushlock with shared+ acquire
 static page_t *lookup_page(vnode_t *vnode, uintmax_t offset) {
 	void *v;
@@ -106,20 +136,14 @@ static int wait_for_ready(page_t *page) {
 
 // TODO: clustered page in
 int mm_cache_get_page(vnode_t *vnode, uintmax_t offset, int flags, page_t **res) {
-	pushlock_acquire_shared(&vnode->pages_lock);
-
-	page_t *page = lookup_page(vnode, offset);
-	if (page)
-		mm_hold_page(mm_get_page_address(page));
-
-	pushlock_release_shared(&vnode->pages_lock);
-
-retry_page:
+	retry_lookup:
+	page_t *page = lookup_and_hold_page(vnode, offset);
 	if (page) {
 		*res = page;
 		return wait_for_ready(page);
-	} else if (flags & MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE)
+	} else if (flags & MM_CACHE_GET_PAGE_FLAGS_NO_POPULATE) {
 		return ENOENT;
+	}
 
 	void *phys = mm_alloc_page(MEMORY_SECTION_DEFAULT);
 	if (phys == NULL)
@@ -136,12 +160,10 @@ retry_page:
 	pushlock_acquire_exclusive(&vnode->pages_lock);
 	page_t *tmp = lookup_page(vnode, offset);
 	if (tmp) {
-		mm_hold_page(mm_get_page_address(tmp));
 		pushlock_release_exclusive(&vnode->pages_lock);
 		trie_free_preallocation(trie_preallocation);
 		mm_release_page(phys);
-		page = tmp;
-		goto retry_page;
+		goto retry_lookup;
 	}
 
 	__assert(insert_page(vnode, offset, page, trie_preallocation) == 0);
@@ -150,50 +172,81 @@ retry_page:
 	VOP_LOCK(vnode);
 	int error = VOP_GETPAGE(vnode, offset, page);
 	VOP_UNLOCK(vnode);
-	if (error) {
-		pushlock_acquire_exclusive(&vnode->pages_lock);
+
+	bool release_pin = false;
+	pushlock_acquire_exclusive(&vnode->pages_lock);
+	int page_flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE);
+	if (page_flags & PAGE_FLAGS_TRUNCATED) {
+		release_pin = __atomic_fetch_and(&page->flags, ~PAGE_FLAGS_PINNED, __ATOMIC_ACQ_REL) & PAGE_FLAGS_PINNED;
+	} else if (error) {
 		remove_page(vnode, offset);
-		pushlock_release_exclusive(&vnode->pages_lock);
+		mm_make_page_anonymous(page);
 	} else {
 		page->backing = vnode;
 		page->offset = offset;
 	}
+	pushlock_release_exclusive(&vnode->pages_lock);
+
+	if (release_pin)
+		mm_release_page(phys);
 
 	__atomic_or_fetch(&page->flags, PAGE_FLAGS_READY | (error ? PAGE_FLAGS_ERROR : 0), __ATOMIC_RELEASE);
 	page_notify_waiters(page);
 	*res = page;
 	trie_free_preallocation(trie_preallocation);
-	if (error) {
+	if (error)
 		mm_release_page(phys);
-	}
 
 	return error;
 }
 
-// removes a page from the page cache if only the caller holds a reference
-// keeps the page in the same state as if it were in the page cache
-int mm_cache_take_page(page_t *page) {
-	pushlock_acquire_exclusive(&page->backing->pages_lock);
-	if (page->refcount > 1) {
-		pushlock_release_exclusive(&page->backing->pages_lock);
-		return EAGAIN;
+void mm_cache_take_page(page_t *page) {
+	int flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE);
+	__assert(flags & PAGE_FLAGS_RECLAIMING);
+	__assert((flags & PAGE_FLAGS_DIRTY) == 0);
+
+	vnode_t *vnode = page->backing;
+	__assert(vnode);
+
+	bool signal = false;
+	pushlock_acquire_exclusive(&vnode->pages_lock);
+	flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE);
+	if ((flags & PAGE_FLAGS_TRUNCATED) == 0)
+		remove_page(vnode, page->offset);
+
+	mm_make_page_anonymous(page);
+	if (flags & PAGE_FLAGS_TRUNCATED) {
+		__assert(vnode->page_reclaim_handoffs != 0);
+		if (--vnode->page_reclaim_handoffs == 0)
+			signal = true;
 	}
+	pushlock_release_exclusive(&vnode->pages_lock);
 
-	__assert((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_DIRTY) == 0);
-	// TODO: remove the need for this check (i.e. transform truncated pages into anonymous pages)
-	if ((__atomic_load_n(&page->flags, __ATOMIC_RELAXED) & PAGE_FLAGS_TRUNCATED) == 0)
-		remove_page(page->backing, page->offset);
+	if (signal)
+		EVENT_SIGNAL(&vnode->zero_reclaim_event);
 
-	pushlock_release_exclusive(&page->backing->pages_lock);
-	return 0;
+	__atomic_and_fetch(&page->flags, ~PAGE_FLAGS_RECLAIMING, __ATOMIC_RELEASE);
+	page_notify_waiters(page);
 }
 
 static void truncate_iterate(void *p) {
 	page_t *page = p;
 	__atomic_sub_fetch(&mm_cache_cached_pages, 1, __ATOMIC_RELAXED);
-	if (__atomic_or_fetch(&page->flags, PAGE_FLAGS_TRUNCATED, __ATOMIC_RELEASE) & PAGE_FLAGS_PINNED) {
-		mm_release_page(mm_get_page_address(page));
+
+	int old_flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED);
+	for (;;) {
+		int new_flags = (old_flags | PAGE_FLAGS_TRUNCATED) & ~PAGE_FLAGS_PINNED;
+		if (__atomic_compare_exchange_n(&page->flags, &old_flags, new_flags, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
+			break;
 	}
+
+	if (old_flags & PAGE_FLAGS_RECLAIMING)
+		++page->backing->page_reclaim_handoffs;
+	else
+		mm_make_page_anonymous(page);
+
+	if (old_flags & PAGE_FLAGS_PINNED)
+		mm_release_page(mm_get_page_address(page));
 }
 
 int mm_cache_truncate(vnode_t *vnode, uintmax_t offset) {
@@ -204,7 +257,27 @@ int mm_cache_truncate(vnode_t *vnode, uintmax_t offset) {
 	trie_iterate(&vnode->pages, min_page, UINT64_MAX, truncate_iterate);
 	trie_truncate(&vnode->pages, min_page);
 
+	bool wait = vnode->page_reclaim_handoffs != 0;
+	eventlistener_t listener;
+	if (wait) {
+		EVENT_INITLISTENER(&listener);
+		EVENT_ATTACH(&listener, &vnode->zero_reclaim_event);
+	}
 	pushlock_release_exclusive(&vnode->pages_lock);
+
+	// TODO use another abstraction for waiting here OR make interruptible sleep optional
+	while (wait) {
+		EVENT_WAIT(&listener, 0);
+		EVENT_DETACHALL(&listener);
+
+		pushlock_acquire_exclusive(&vnode->pages_lock);
+		wait = vnode->page_reclaim_handoffs != 0;
+		if (wait) {
+			EVENT_INITLISTENER(&listener);
+			EVENT_ATTACH(&listener, &vnode->zero_reclaim_event);
+		}
+		pushlock_release_exclusive(&vnode->pages_lock);
+	}
 	return 0;
 }
 

kernel-src/mm/mm.c

@@ -164,7 +164,7 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) {
 			arch_mmu_remap(current_mm_context()->pagetable, mapped_address, vaddr, range->mmuflags);
 
 			if (file_mapping && cacheable)
-				mm_cache_make_dirty(current_page);
+				mm_cache_make_dirty(range->vnode, current_page);
 
 			*resulting_page = current_page;
 			return 0;
@@ -226,7 +226,7 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) {
 		}
 
 		if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE))
-			mm_cache_make_dirty(vn_page);
+			mm_cache_make_dirty(range->vnode, vn_page);
 
 		*resulting_page = vn_page;
 		return 0;

kernel-src/mm/page.c

@@ -124,6 +124,18 @@ static void internal_hold(page_t *page) {
 	}
 }
 
+static bool try_page_reclaim(page_t *page) {
+	int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED);
+
+	for (;;) {
+		if (flags & PAGE_FLAGS_TRUNCATED)
+			return false;
+
+		if (__atomic_compare_exchange_n(&page->flags, &flags, flags | PAGE_FLAGS_RECLAIMING, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
+			return true;
+	}
+}
+
 page_t *mm_get_page(void *address) {
 	return &pages[((uintptr_t)address / PAGE_SIZE)];
 }
@@ -155,6 +167,25 @@ void mm_release_page(void *addr) {
 	MUTEX_RELEASE(&free_list_mutex);
 }
 
+void mm_make_page_anonymous(page_t *page) {
+	MUTEX_ACQUIRE(&free_list_mutex);
+
+	__assert(page->backing);
+
+	bool on_standby = page->refcount == 0;
+	if (on_standby)
+		remove_from_free_list(page);
+
+	page->backing = NULL;
+
+	if (on_standby) {
+		__atomic_or_fetch(&page->flags, PAGE_FLAGS_FREE, __ATOMIC_RELAXED);
+		insert_in_free_list(page);
+	}
+
+	MUTEX_RELEASE(&free_list_mutex);
+}
+
 void mm_unlock_and_release_page(void *addr) {
 	page_t *page = &pages[(uintptr_t)addr / PAGE_SIZE];
 	mm_unlock_page(page);
@@ -176,7 +207,6 @@ static void do_alloc(page_t *page) {
 }
 
 void *mm_alloc_page(int section) {
-	retry:
 	MUTEX_ACQUIRE(&free_list_mutex);
 	page_t *page = NULL;
 
@@ -196,23 +226,22 @@ void *mm_alloc_page(int section) {
 	if (page == NULL) {
 		for (int i = section; i >= 0; --i) {
 			page = standby_tails[i];
-			if (page) {
-				cache_page = true;
-				internal_hold(page);
-				break;
-			}
+			while (page && try_page_reclaim(page) == false)
+				page = page->free_prev;
+
+			if (page == NULL)
+				continue;
+
+			cache_page = true;
+			internal_hold(page);
+			break;
 		}
 	}
 
 	MUTEX_RELEASE(&free_list_mutex);
 
-	if (cache_page && mm_cache_take_page(page) == EAGAIN) {
-		// someone already got the page from the cache between us holding it and taking it
-		mm_release_page(mm_get_page_address(page));
-		page = NULL;
-		// retry it from the start, as an anonymous page could have been released while the lock was not held
-		goto retry;
-	} else if (cache_page) {
+	if (cache_page) {
+		mm_cache_take_page(page);
 		// we got the page from the cache, all is good and we hold the only reference to it.
 		// set the refcount to 0, as expected by the do_alloc call
 		page->refcount = 0;

kernel-src/mm/range.c

@@ -165,7 +165,7 @@ void mm_destroy_range(mm_range_t *range, uintmax_t _offset, size_t size, int fla
 			} else if (arch_mmu_iswritable(current_mm_context()->pagetable, vaddr)) {
 				// dirty page cache mapping
 				arch_mmu_unmap(current_mm_context()->pagetable, vaddr);
-				mm_cache_make_dirty(mm_get_page(physical));
+				mm_cache_make_dirty(range->vnode, mm_get_page(physical));
 				mm_release_page(physical);
 			} else {
 				// non dirty page mapping
@@ -210,7 +210,7 @@ static void change_mmu_range(mm_range_t *range, void *base, size_t size, mmuflag
 			// removing write permissions from a writeable dirty shared mapped page, mark it as dirty, as
 			// it won't be marked dirty upon a mm_unmap after this
 			page_t *page = mm_get_page(physical);
-			mm_cache_make_dirty(page);
+			mm_cache_make_dirty(range->vnode, page);
 		}
 
 		// we will only change the mapping if the permissions decreased

kernel-src/mm/writeback.c

@@ -16,12 +16,15 @@ static bool writer_syncing;
 static eventheader_t writer_wakeup_event;
 static eventheader_t sync_complete_event;
 
-// OK
-void mm_cache_make_dirty(page_t *page) {
-	vnode_t *vnode = page->backing;
+void mm_cache_make_dirty(vnode_t *vnode, page_t *page) {
+	// check this here as a fast-path to reduce the mutex contention
+	int flags = __atomic_load_n(&page->flags, __ATOMIC_ACQUIRE);
+	if (flags & PAGE_FLAGS_TRUNCATED)
+		return;
+
 	MUTEX_ACQUIRE(&vnode->dirty_list_mutex);
 
-	int flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED);
+	flags = __atomic_load_n(&page->flags, __ATOMIC_RELAXED);
 	for (;;) {
 		if (flags & PAGE_FLAGS_TRUNCATED) {
 			MUTEX_RELEASE(&vnode->dirty_list_mutex);