#include #include #include #include #include #include #include #define TABLE_SIZE 4096 #define WRITER_TICK_SECONDS 15 static mutex_t *mutex_table; static page_t **table; static thread_t *writerthread; static semaphore_t sync; static eventheader_t syncevent; static eventheader_t pagereadyevent; size_t vmmcache_cachedpages; #define HOLD_LOCK(vnode, offset) \ MUTEX_ACQUIRE(&mutex_table[getentry(vnode, offset)]); #define RELEASE_LOCK(vnode, offset) \ MUTEX_RELEASE(&mutex_table[getentry(vnode, offset)]); static inline uint64_t fnv1ahash(void *buffer, size_t size); static uintmax_t getentry(vnode_t *vnode, uintmax_t offset) { struct { vnode_t *vnode; uintmax_t offset; } tmp; tmp.vnode = vnode; tmp.offset = offset; return fnv1ahash(&tmp, sizeof(tmp)) % TABLE_SIZE; } // assumes lock is held static page_t *findpage(vnode_t *vnode, uintmax_t offset) { uintmax_t entry = getentry(vnode, offset); page_t *page = table[entry]; while (page) { if (page->backing == vnode && page->offset == offset) break; page = page->hashnext; } return page; } // assumes lock is held static void putpage(page_t *page) { uintmax_t entry = getentry(page->backing, page->offset); // add to table list page->hashprev = NULL; page->hashnext = table[entry]; if (table[entry]) table[entry]->hashprev = page; table[entry] = page; // add to vnode list // XXX this could be a bit slow if a large portion of the file is in memory and it gets purged page->vnodeprev = NULL; page->vnodenext = page->backing->pages; if (page->backing->pages) page->backing->pages->vnodeprev = page; page->backing->pages = page; __atomic_fetch_add(&vmmcache_cachedpages, 1, __ATOMIC_SEQ_CST); } // assumes lock is held static void removepage(page_t *page) { uintmax_t entry = getentry(page->backing, page->offset); // remove from table list if (page->hashnext) page->hashnext->hashprev = page->hashprev; if (page->hashprev) page->hashprev->hashnext = page->hashnext; else table[entry] = page->hashnext; page->hashnext = NULL; page->hashprev = NULL; // remove from vnode list if (page->vnodenext) page->vnodenext->vnodeprev = page->vnodeprev; if (page->vnodeprev) page->vnodeprev->vnodenext = page->vnodenext; else page->backing->pages = page->vnodenext; page->vnodenext = NULL; page->vnodeprev = NULL; __atomic_fetch_sub(&vmmcache_cachedpages, 1, __ATOMIC_SEQ_CST); } int vmmcache_getpage(vnode_t *vnode, uintmax_t offset, page_t **res) { __assert(vnode->type == V_TYPE_REGULAR || vnode->type == V_TYPE_BLKDEV); __assert((offset % PAGE_SIZE) == 0); retry_err: HOLD_LOCK(vnode, offset); page_t *newpage = NULL; volatile page_t *page = findpage(vnode, offset); retry: if (page) { // page is present in the page cache pmm_hold(pmm_getpageaddress((page_t *)page)); RELEASE_LOCK(vnode, offset); // in the case of a retry, release the allocated page here if (newpage) pmm_release(pmm_getpageaddress(newpage)); eventlistener_t listener; EVENT_INITLISTENER(&listener); EVENT_ATTACH(&listener, &pagereadyevent); // wait for page to be ready while ((page->flags & (PAGE_FLAGS_READY | PAGE_FLAGS_ERROR)) == 0) EVENT_WAIT(&listener, 0); EVENT_DETACHALL(&listener); if (page->flags & PAGE_FLAGS_ERROR) { // the thread handling the page in failed to read it, we should retry it and see whats up pmm_release(pmm_getpageaddress((page_t *)page)); goto retry_err; } *res = (page_t *)page; } else { // page is not present in the cache, we will have to load it in RELEASE_LOCK(vnode, offset); void *address = pmm_allocpage(PMM_SECTION_DEFAULT); if (address == NULL) return ENOMEM; newpage = pmm_getpage(address); MUTEX_ACQUIRE(&vnode->pages_mutex); HOLD_LOCK(vnode, offset); // while the lock wasn't being held, the page could have potentially been added to the cache // check for it again and return from the function as if it was always there in the first place page = findpage(vnode, offset); if (page) { MUTEX_RELEASE(&vnode->pages_mutex); goto retry; } newpage->backing = vnode; newpage->offset = offset; // add it to the page cache putpage(newpage); MUTEX_RELEASE(&vnode->pages_mutex); RELEASE_LOCK(vnode, offset); VOP_LOCK(vnode); int error = VOP_GETPAGE(vnode, offset, newpage); VOP_UNLOCK(vnode); if (error) { // an error happened with GETPAGE, remove the page from the cache, // tell the sleeping threads that something happened and free the page // by setting backing to null so it gets treated as an anonymous page again MUTEX_ACQUIRE(&vnode->pages_mutex); HOLD_LOCK(vnode, offset); removepage(newpage); MUTEX_RELEASE(&vnode->pages_mutex); newpage->flags |= PAGE_FLAGS_ERROR; newpage->backing = NULL; newpage->offset = 0; RELEASE_LOCK(vnode, offset); pmm_release(pmm_getpageaddress(newpage)); EVENT_SIGNAL(&pagereadyevent); return error; } newpage->flags |= PAGE_FLAGS_READY; EVENT_SIGNAL(&pagereadyevent); *res = newpage; } return 0; } // adds a page to the cache in a specific offset if its not already there int vmmcache_pushpage(vnode_t *vnode, uintmax_t offset, page_t *page) { __assert((offset % PAGE_SIZE) == 0); MUTEX_ACQUIRE(&vnode->pages_mutex); HOLD_LOCK(vnode, offset); page_t *pagetest = findpage(vnode, offset); if (pagetest) { MUTEX_RELEASE(&vnode->pages_mutex); RELEASE_LOCK(vnode, offset); return EAGAIN; } page->backing = vnode; page->offset = offset; page->flags |= PAGE_FLAGS_READY; putpage(page); MUTEX_RELEASE(&vnode->pages_mutex); RELEASE_LOCK(vnode, offset); return 0; } // removes a page from the cache AND turns it into anonymous memory int vmmcache_evict(page_t *page) { MUTEX_ACQUIRE(&page->backing->pages_mutex); HOLD_LOCK(page->backing, page->offset); if (page->refcount > 1) { MUTEX_RELEASE(&page->backing->pages_mutex); RELEASE_LOCK(page->backing, page->offset); return EAGAIN; } __assert(page->refcount == 1); __assert((page->flags & PAGE_FLAGS_DIRTY) == 0); if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) { // the page needs to be removed from the cache to continue removepage(page); } MUTEX_RELEASE(&page->backing->pages_mutex); RELEASE_LOCK(page->backing, page->offset); page->flags = 0; page->backing = NULL; page->offset = 0; return 0; } // removes a page from the cache *but doesn't do anything to it* int vmmcache_takepage(page_t *page) { MUTEX_ACQUIRE(&page->backing->pages_mutex); HOLD_LOCK(page->backing, page->offset); // someone called vmmcache_getpage() and got this page while the lock wasn't held // return an error status to the caller if (page->refcount > 1) { MUTEX_RELEASE(&page->backing->pages_mutex); RELEASE_LOCK(page->backing, page->offset); return EAGAIN; } __assert(page->refcount == 1); __assert((page->flags & PAGE_FLAGS_DIRTY) == 0); // XXX maybe just not allow truncated pages to take up space like this? if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) { // the page needs to be removed from the cache to continue removepage(page); } MUTEX_RELEASE(&page->backing->pages_mutex); RELEASE_LOCK(page->backing, page->offset); return 0; } int vmmcache_truncate(vnode_t *vnode, uintmax_t offset) { MUTEX_ACQUIRE(&vnode->pages_mutex); page_t *pagelist = NULL; page_t *page = vnode->pages; while (page) { page_t *oldpage = page; page = page->vnodenext; // only truncate past a certain offset if (oldpage->offset < offset) continue; __atomic_or_fetch(&oldpage->flags, PAGE_FLAGS_TRUNCATED, __ATOMIC_SEQ_CST); HOLD_LOCK(oldpage->backing, oldpage->offset); removepage(oldpage); RELEASE_LOCK(oldpage->backing, oldpage->offset); oldpage->vnodenext = pagelist; pagelist = oldpage; } MUTEX_RELEASE(&vnode->pages_mutex); // make sure to unref if they are pinned while (pagelist) { page_t *page = pagelist; pagelist = pagelist->vnodenext; if (page->flags & PAGE_FLAGS_PINNED) pmm_release(pmm_getpageaddress(page)); } return 0; } static int syncpage(page_t *page, bool backinglock) { int flags = __atomic_fetch_and(&page->flags, ~PAGE_FLAGS_DIRTY, __ATOMIC_SEQ_CST); __assert(flags & PAGE_FLAGS_DIRTY); int e = 0; if ((flags & PAGE_FLAGS_TRUNCATED) == 0) { if (backinglock) VOP_LOCK(page->backing); e = VOP_PUTPAGE(page->backing, page->offset, (page_t *)page); if (backinglock) VOP_UNLOCK(page->backing); VOP_RELEASE(page->backing); } else { // page got truncated from the file while waiting to be written to disk // its still holding a reference to the vnode, so release that VOP_RELEASE(page->backing); } pmm_release(pmm_getpageaddress((page_t *)page)); return e; } MUTEX_DEFINE(dirty_list_mutex); static page_t *dirtylist; static page_t *dirtylistend; // expects vnode to be held int vmmcache_syncvnode(vnode_t *vnode, uintmax_t offset, size_t size) { offset = ROUND_DOWN(offset, PAGE_SIZE); uintmax_t top = offset + size; // overflow check __assert(top > offset); MUTEX_ACQUIRE(&vnode->pages_mutex); MUTEX_ACQUIRE(&dirty_list_mutex); // loop through all vnode pages in memory and check which ones are in the range and are dirty // TODO create a proper vnode dirty list as to not have to loop through the ENTIRE thing in memory page_t *page = vnode->pages; page_t *vnodedirtylist = NULL; for (; page; page = page->vnodenext) { if (page->offset < offset || page->offset >= top || (page->offset & PAGE_FLAGS_DIRTY) == 0) continue; // remove from write list and add to an internal list using the write pointers // in a singly linked list way if (page->writenext) page->writenext->writeprev = page->writeprev; if (page->writeprev) page->writeprev->writenext = page->writenext; else dirtylist = page->writenext; if (dirtylist == NULL) dirtylistend = NULL; page->writenext = vnodedirtylist; page->writeprev = NULL; vnodedirtylist = page; } MUTEX_RELEASE(&dirty_list_mutex); MUTEX_RELEASE(&vnode->pages_mutex); int e = 0; while (vnodedirtylist) { // in the case of failure, only the first error to occur will be reported and we will not // retry the write and keep on syncing the pages to disk page_t *page = vnodedirtylist; vnodedirtylist = vnodedirtylist->writenext; page->writenext = NULL; int error = syncpage(page, false); if (e == 0) e = error; // syncpage returns with lock released } return e; } int vmmcache_sync() { eventlistener_t eventlistener; EVENT_INITLISTENER(&eventlistener); if (dirtylist == NULL) { // no dirty pages return 0; } EVENT_ATTACH(&eventlistener, &syncevent); semaphore_signal(&sync); EVENT_WAIT(&eventlistener, 0); EVENT_DETACHALL(&eventlistener); return 0; } // backing expected locked int vmmcache_makedirty(page_t *page) { bool madedirty = false; MUTEX_ACQUIRE(&dirty_list_mutex); if ((page->flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_TRUNCATED)) == 0) { madedirty = true; // page is neither dirty nor truncated, add to dirty list and hold the page and vnode page->flags |= PAGE_FLAGS_DIRTY; page->writeprev = NULL; page->writenext = dirtylist; if (dirtylist) dirtylist->writeprev = page; else dirtylistend = page; dirtylist = page; pmm_hold(pmm_getpageaddress(page)); __assert(page->backing); VOP_HOLD(page->backing); } MUTEX_RELEASE(&dirty_list_mutex); if (madedirty) { vattr_t attr; attr.mtime = timekeeper_time(); VOP_SETATTR(page->backing, &attr, V_ATTR_MTIME, NULL); } return 0; } static void tick(context_t *, dpcarg_t arg) { semaphore_signal(&sync); } static void writer() { timerentry_t timerentry; // this will be inserted on some random cpu's timer, but it will always work after that interrupt_set(false); timer_insert(current_cpu()->timer, &timerentry, tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true); interrupt_set(true); for (;;) { MUTEX_ACQUIRE(&dirty_list_mutex); volatile page_t *page = dirtylistend; if (page == NULL) { MUTEX_RELEASE(&dirty_list_mutex); EVENT_SIGNAL(&syncevent); semaphore_wait(&sync, false); continue; } dirtylistend = page->writeprev; if (dirtylistend) dirtylistend->writenext = NULL; else dirtylist = NULL; page->writeprev = NULL; MUTEX_RELEASE(&dirty_list_mutex); // TODO notify error on vmmcache_syncvnode syncpage((page_t *)page, true); } } void vmmcache_init() { table = vmm_map(NULL, TABLE_SIZE * sizeof(page_t *), VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); __assert(table); mutex_table = vmm_map(NULL, TABLE_SIZE * sizeof(mutex_t), VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); __assert(mutex_table); memset(table, 0, TABLE_SIZE * sizeof(page_t *)); for (int i = 0; i < TABLE_SIZE; ++i) MUTEX_INIT(&mutex_table[i]); SEMAPHORE_INIT(&sync, 0); writerthread = sched_newthread(writer, PAGE_SIZE * 16, 1, NULL, NULL); __assert(writerthread); sched_queue(writerthread); vmmcache_sync(); EVENT_INITHEADER(&syncevent); EVENT_INITHEADER(&pagereadyevent); } INIT_ROUTINE_DEFINE(vmm_cache, INIT_ROUTINE_FLAGS_NONE, vmmcache_init, scheduler);