From 967ca194360edc67857bb0c5cdddb261c254be5f Mon Sep 17 00:00:00 2001 From: Matheus Date: Mon, 13 Jul 2026 12:06:32 -0300 Subject: [PATCH] kernel: advanced buffer cache --- kernel-src/fs/abc.c | 304 ++++++++++++++++++++++++++++++++ kernel-src/fs/vfs.c | 2 +- kernel-src/include/kernel/abc.h | 43 +++++ 3 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 kernel-src/fs/abc.c create mode 100644 kernel-src/include/kernel/abc.h diff --git a/kernel-src/fs/abc.c b/kernel-src/fs/abc.c new file mode 100644 index 00000000..cfe5d777 --- /dev/null +++ b/kernel-src/fs/abc.c @@ -0,0 +1,304 @@ +#include +#include +#include +#include +#include +#include + +#define WRITEBACK_MAX 32 +#define WRITER_TICK_SECONDS 15 + +static scache_t *abc_block_cache; + +void abc_release_block(abc_t *abc, abc_block_t *abc_block) { + MUTEX_ACQUIRE(&abc->mutex); // TODO exclusive lock + if (__atomic_sub_fetch(&abc_block->refcount, 1, __ATOMIC_RELEASE) == 0) { + __atomic_thread_fence(__ATOMIC_ACQUIRE); + + // another thread might have referenced the block while we were busy + if (__atomic_load_n(&abc_block->refcount, __ATOMIC_RELAXED)) + goto leave; + + __assert(trie_remove(&abc->blocks, abc_block->block) == 0); + + mm_release_page(FROM_HHDM(abc_block->data)); + abc_block->block = 0; + abc_block->refcount = 1; + slab_free(abc_block_cache, abc_block); + } +leave: + MUTEX_RELEASE(&abc->mutex); +} + +static list_node_t *try_stealing_from_dirty(abc_t *abc, uint64_t block) { + void *v; + if (trie_lookup(&abc->blocks, block, &v)) + return NULL; + + abc_block_t *blk = v; + if ((__atomic_load_n(&blk->flags, __ATOMIC_RELAXED) & ABC_BLOCK_FLAGS_DIRTY) == 0) + return NULL; + + list_remove(&abc->dirty_list, &blk->dirty_list_node); + __atomic_exchange_n(&blk->flags, ABC_BLOCK_FLAGS_BUSY, __ATOMIC_ACQUIRE); + return &blk->dirty_list_node; +} + +void abc_sync(abc_t *abc) { + eventlistener_t listener; + EVENT_INITLISTENER(&listener); + EVENT_ATTACH(&listener, &abc->dirty_list_empty_event); + + MUTEX_ACQUIRE(&abc->dirty_list_mutex); + if (abc->dirty_list.head == NULL && !abc->syncing) { + MUTEX_RELEASE(&abc->dirty_list_mutex); + EVENT_DETACHALL(&listener); + return; + } + + EVENT_SIGNAL(&abc->dirty_list_sync_event); + MUTEX_RELEASE(&abc->dirty_list_mutex); + + EVENT_WAIT(&listener, 0); + EVENT_DETACHALL(&listener); +} + +static void tick(context_t *, dpcarg_t arg) { + abc_t *abc = arg; + EVENT_SIGNAL(&abc->dirty_list_sync_event); +} + +static void writer_thread(void) { + abc_t *abc = current_thread()->kernelarg; + + long ipl = interrupt_raiseipl(IPL_DPC); + timerentry_t timer_entry; + timer_insert(current_cpu()->timer, &timer_entry, tick, abc, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true); + interrupt_loweripl(ipl); + + for (;;) { + list_t internal_list; + list_init(&internal_list); + + eventlistener_t listener; + EVENT_INITLISTENER(&listener); + EVENT_ATTACH(&listener, &abc->dirty_list_sync_event); + + MUTEX_ACQUIRE(&abc->dirty_list_mutex); + abc_block_t *abc_block = (abc_block_t *)list_pop_front(&abc->dirty_list); + if (abc_block == NULL) { + abc->syncing = false; + EVENT_SIGNAL(&abc->dirty_list_empty_event); + MUTEX_RELEASE(&abc->dirty_list_mutex); + EVENT_WAIT(&listener, 0); + EVENT_DETACHALL(&listener); + continue; + } + + abc->syncing = true; + EVENT_DETACHALL(&listener); + + list_push_front(&internal_list, &abc_block->dirty_list_node); + __atomic_exchange_n(&abc_block->flags, ABC_BLOCK_FLAGS_BUSY, __ATOMIC_ACQUIRE); + + MUTEX_ACQUIRE(&abc->mutex); // TODO shared lock + size_t block_count = 1; + if (abc_block->block) { + uint64_t iterator = abc_block->block - 1; + while (block_count < WRITEBACK_MAX) { + list_node_t *node = try_stealing_from_dirty(abc, iterator); + if (node == NULL) + break; + + list_push_front(&internal_list, node); + ++block_count; + if (iterator == 0) + break; + --iterator; + } + } + + uint64_t iterator = abc_block->block + 1; + while (block_count < WRITEBACK_MAX) { + list_node_t *node = try_stealing_from_dirty(abc, iterator); + if (node == NULL) + break; + + list_push_back(&internal_list, node); + + ++block_count; + ++iterator; + } + + MUTEX_RELEASE(&abc->mutex); + MUTEX_RELEASE(&abc->dirty_list_mutex); + + abc_block_t *front_block = (abc_block_t *)internal_list.head; + size_t blocks_per_page = PAGE_SIZE / abc->block_size; + size_t iovec_count = (block_count + front_block->block % blocks_per_page + blocks_per_page - 1) / blocks_per_page; + iovec_t iovec[iovec_count]; + abc_block_t *blk_it = front_block; + + size_t inserted = 0; + size_t remaining_blocks = block_count; + while (remaining_blocks) { + size_t blocks_in_page = min(remaining_blocks, blocks_per_page - blk_it->block % blocks_per_page); + iovec[inserted].addr = blk_it->data; + iovec[inserted].len = blocks_in_page * abc->block_size; + ++inserted; + for (size_t i = 0; i < blocks_in_page; ++i) + blk_it = (abc_block_t *)blk_it->dirty_list_node.next; + remaining_blocks -= blocks_in_page; + } + + iovec_iterator_t iovec_iterator; + iovec_iterator_init(&iovec_iterator, iovec, inserted); + + size_t written; + int error = vfs_write_iovec(abc->backing, &iovec_iterator, block_count * abc->block_size, front_block->block * abc->block_size, &written, V_FFLAGS_NOCACHE); + if (error) { + // TODO: this should be propagated in some way for the filesystem + // TODO: it should also set the blocks as dirty to retry + printf("abc: writing back %lu blocks of filesystem metadata starting at block %lu failed. you should probably run fsck on your filesystems.\n", block_count, front_block->block); + } else { + __assert(written == block_count * abc->block_size); + } + + list_for_each_safe(&internal_list, list_node) { + abc_block_t *blk = (abc_block_t *)list_node; + if (__atomic_and_fetch(&blk->flags, ~ABC_BLOCK_FLAGS_BUSY, __ATOMIC_ACQUIRE) & ABC_BLOCK_FLAGS_DIRTY) { + MUTEX_ACQUIRE(&abc->dirty_list_mutex); + list_push_back(&abc->dirty_list, &blk->dirty_list_node); + MUTEX_RELEASE(&abc->dirty_list_mutex); + } else { + abc_release_block(abc, blk); + } + } + } +} + +// returns true if the thread should insert it into the dirty list +static bool try_to_set_dirty_flag(abc_block_t *abc_block) { + int old_flags = __atomic_fetch_or(&abc_block->flags, ABC_BLOCK_FLAGS_DIRTY, __ATOMIC_RELEASE); + return !(old_flags & ABC_BLOCK_FLAGS_DIRTY) && !(old_flags & ABC_BLOCK_FLAGS_BUSY); +} + +void abc_make_dirty(abc_t *abc, abc_block_t *abc_block) { + // TODO: fix this race + MUTEX_ACQUIRE(&abc->dirty_list_mutex); + + if (!try_to_set_dirty_flag(abc_block)) { + MUTEX_RELEASE(&abc->dirty_list_mutex); + return; + } + + __atomic_fetch_add(&abc_block->refcount, 1, __ATOMIC_RELAXED); + + list_push_back(&abc->dirty_list, &abc_block->dirty_list_node); + MUTEX_RELEASE(&abc->dirty_list_mutex); +} + +static abc_block_t *get_block(abc_t *abc, uint64_t block) { + void *val; + int error = trie_lookup(&abc->blocks, block, &val); + __assert(!error || error == ENOENT); + if (error) + return NULL; + + abc_block_t *abc_block = val; + __atomic_fetch_add(&abc_block->refcount, 1, __ATOMIC_RELAXED); + return abc_block; +} + +int abc_get_block(abc_t *abc, uint64_t block, abc_block_t **ret) { + MUTEX_ACQUIRE(&abc->mutex); // TODO: shared lock + abc_block_t *abc_block = get_block(abc, block); + MUTEX_RELEASE(&abc->mutex); + + if (abc_block) { + *ret = abc_block; + return 0; + } + + abc_block = slab_allocate(abc_block_cache); + if (abc_block == NULL) + return ENOMEM; + + trie_preallocation_t *trie_preallocation = trie_preallocate(); + if (trie_preallocation == NULL) { + slab_free(abc_block_cache, abc_block); + return ENOMEM; + } + + page_t *page; + int error = mm_cache_get_page(abc->backing, ROUND_DOWN(block * abc->block_size, PAGE_SIZE), &page); + if (error) { + slab_free(abc_block_cache, abc_block); + trie_free_preallocation(trie_preallocation); + return error; + } + size_t page_offset = block * abc->block_size - ROUND_DOWN(block * abc->block_size, PAGE_SIZE); + + MUTEX_ACQUIRE(&abc->mutex); // TODO: exclusive lock + + abc_block_t *abc_block_check = get_block(abc, block); + if (abc_block_check) { + MUTEX_RELEASE(&abc->mutex); + mm_release_page(mm_get_page_address(page)); + slab_free(abc_block_cache, abc_block); + trie_free_preallocation(trie_preallocation); + *ret = abc_block_check; + return 0; + } + + abc_block->block = block; + abc_block->data = MAKE_HHDM((void *)((uintptr_t)mm_get_page_address(page) + page_offset)); + __assert(trie_insert_preallocated(&abc->blocks, block, abc_block, trie_preallocation) == 0); + + MUTEX_RELEASE(&abc->mutex); + + trie_free_preallocation(trie_preallocation); + + *ret = abc_block; + return 0; +} + +int abc_init(abc_t *abc, vnode_t *vnode, size_t block_size) { + __assert(block_size <= PAGE_SIZE); + __assert(PAGE_SIZE % block_size == 0); + abc->writer = sched_newthread(writer_thread, PAGE_SIZE * 2, 0, NULL, NULL); + if (abc->writer == NULL) + return ENOMEM; + + MUTEX_INIT(&abc->mutex); + trie_init(&abc->blocks); + abc->block_size = block_size; + + MUTEX_INIT(&abc->dirty_list_mutex); + list_init(&abc->dirty_list); + abc->syncing = false; + EVENT_INITHEADER(&abc->dirty_list_sync_event); + EVENT_INITHEADER(&abc->dirty_list_empty_event); + + VOP_HOLD(vnode); + abc->backing = vnode; + + abc->writer->kernelarg = abc; + sched_queue(abc->writer); + + return 0; +} + +static bool abc_block_constructor(scache_t *, void *ptr) { + abc_block_t *block = ptr; + __atomic_store_n(&block->refcount, 1, __ATOMIC_RELAXED); + __atomic_store_n(&block->flags, 0, __ATOMIC_RELAXED); + return true; +} + +static void abc_init_routine(void) { + abc_block_cache = slab_newcache(sizeof(abc_block_t), 0, abc_block_constructor, NULL); + __assert(abc_block_cache); +} + +INIT_ROUTINE_DEFINE(abc, INIT_ROUTINE_FLAGS_NONE, abc_init_routine, scheduler, trie); diff --git a/kernel-src/fs/vfs.c b/kernel-src/fs/vfs.c index 4f994de4..9364abd5 100644 --- a/kernel-src/fs/vfs.c +++ b/kernel-src/fs/vfs.c @@ -73,7 +73,7 @@ void vfs_init() { __assert(advlock_cache); } -INIT_ROUTINE_DEFINE(vfs, INIT_ROUTINE_FLAGS_NONE, vfs_init, bsp_early); +INIT_ROUTINE_DEFINE(vfs, INIT_ROUTINE_FLAGS_NONE, vfs_init, abc); advlock_t *advlock_allocate(void) { return slab_allocate(advlock_cache); diff --git a/kernel-src/include/kernel/abc.h b/kernel-src/include/kernel/abc.h new file mode 100644 index 00000000..4799cc59 --- /dev/null +++ b/kernel-src/include/kernel/abc.h @@ -0,0 +1,43 @@ +#ifndef _ABC_H +#define _ABC_H + +#include +#include +#include +#include +#include +#include +#include + +#define ABC_BLOCK_FLAGS_DIRTY 1 +#define ABC_BLOCK_FLAGS_BUSY 2 +typedef struct { + list_node_t dirty_list_node; + uint64_t block; + void *data; + size_t refcount; + int flags; +} abc_block_t; + +typedef struct { + mutex_t mutex; // TODO: use pushlock properly here once pushlock rewrite is known good + trie_t blocks; + size_t block_size; + + mutex_t dirty_list_mutex; + list_t dirty_list; + bool syncing; + eventheader_t dirty_list_sync_event; + eventheader_t dirty_list_empty_event; + + vnode_t *backing; + thread_t *writer; +} abc_t; + +int abc_init(abc_t *abc, vnode_t *vnode, size_t block_size); +int abc_get_block(abc_t *abc, uint64_t block, abc_block_t **ret); +void abc_make_dirty(abc_t *abc, abc_block_t *block); +void abc_sync(abc_t *abc); +void abc_release_block(abc_t *abc, abc_block_t *abc_block); + +#endif