vmmcache: initial implementation

This commit is contained in:
Matheus
2024-05-24 18:03:24 -03:00
parent 7673a365e8
commit 48b7fb4324
21 changed files with 772 additions and 134 deletions
+4
View File
@@ -37,6 +37,7 @@
#include <kernel/mouse.h>
#include <kernel/tty.h>
#include <kernel/pty.h>
#include <kernel/vmmcache.h>
static cpu_t bsp_cpu;
@@ -64,6 +65,9 @@ void kernel_entry() {
arch_apic_timerinit();
sched_init();
arch_smp_wakeup();
vmmcache_init();
vfs_init();
tmpfs_init();
devfs_init();
+7
View File
@@ -175,6 +175,13 @@ bool arch_mmu_iswritable(pagetableptr_t table, void *vaddr) {
return entry == NULL ? false : *entry & ARCH_MMU_FLAGS_WRITE;
}
#define ARCH_MMU_FLAGS_DIRTY (1 << 6)
bool arch_mmu_isdirty(pagetableptr_t table, void *vaddr) {
uint64_t *entry = get_page(table, vaddr);
return entry == NULL ? false : *entry & ARCH_MMU_FLAGS_DIRTY;
}
void arch_mmu_switch(pagetableptr_t table) {
asm volatile("mov %%rax, %%cr3" : : "a"(table));
}
+24 -1
View File
@@ -6,6 +6,7 @@
#include <errno.h>
#include <kernel/alloc.h>
#include <kernel/timekeeper.h>
#include <kernel/pmm.h>
static devnode_t *devfsroot;
@@ -332,6 +333,26 @@ static int devfs_getdents(vnode_t *node, dent_t *buffer, size_t count, uintmax_t
return 0;
}
static int devfs_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
// only block devices will have this called
__assert(node->type == V_TYPE_BLKDEV);
size_t readc;
int error = VOP_READ(node, MAKE_HHDM(pmm_getpageaddress(page)), PAGE_SIZE, offset, 0, &readc, NULL);
if (readc == 0)
return ENXIO;
return error;
}
static int devfs_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
// only block devices will have this called
__assert(node->type == V_TYPE_BLKDEV);
size_t writec;
int error = VOP_WRITE(node, MAKE_HHDM(pmm_getpageaddress(page)), PAGE_SIZE, offset, 0, &writec, NULL);
__assert(writec != 0);
return error;
}
static int devfs_enodev() {
return ENODEV;
@@ -365,7 +386,9 @@ static vops_t vnops = {
.ioctl = devfs_ioctl,
.maxseek = devfs_maxseek,
.resize = devfs_enodev,
.rename = devfs_enodev
.rename = devfs_enodev,
.putpage = devfs_putpage,
.getpage = devfs_getpage
};
static void ctor(scache_t *cache, void *obj) {
+39 -4
View File
@@ -9,6 +9,7 @@
#include <mutex.h>
#include <logging.h>
#include <util.h>
#include <kernel/vmmcache.h>
#define INODE_ROOT 2
@@ -1262,9 +1263,12 @@ int ext2_write(vnode_t *vnode, void *buffer, size_t size, uintmax_t offset, int
endoffset = -1ll;
if (endoffset > inodesize) {
err = resizeinode(fs, node, endoffset);
if (err)
if (offset > inodesize) {
*writec = 0;
goto cleanup;
}
size = min(endoffset, inodesize) - offset;
}
err = rwbytes(fs, node, buffer, size, offset, true);
@@ -1281,10 +1285,17 @@ int ext2_write(vnode_t *vnode, void *buffer, size_t size, uintmax_t offset, int
}
int ext2_resize(vnode_t *vnode, size_t newsize, cred_t *cred) {
__assert(vnode->type == V_TYPE_REGULAR);
ext2node_t *node = (ext2node_t *)vnode;
ext2fs_t *fs = (ext2fs_t *)vnode->vfs;
VOP_LOCK(vnode);
int e = resizeinode(fs, node, newsize);
size_t size = INODE_SIZE(&node->inode);
int e = 0;
if (size != newsize) {
e = resizeinode(fs, node, newsize);
if (size > newsize)
vmmcache_truncate(vnode, newsize);
}
VOP_UNLOCK(vnode);
return e;
}
@@ -1546,6 +1557,27 @@ static int ext2_unlink(vnode_t *vnode, char *name, cred_t *cred) {
return err;
}
static int ext2_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
// only regular files get cached
__assert(node->type == V_TYPE_REGULAR);
size_t readc;
int error = VOP_READ(node, MAKE_HHDM(pmm_getpageaddress(page)), PAGE_SIZE, offset, 0, &readc, NULL);
if (readc == 0)
return ENXIO;
return error;
}
static int ext2_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
// only regular files get cached
__assert(node->type == V_TYPE_REGULAR);
size_t writec;
int error = VOP_WRITE(node, MAKE_HHDM(pmm_getpageaddress(page)), PAGE_SIZE, offset, 0, &writec, NULL);
__assert(writec != 0);
return error;
}
static int ext2_inactive(vnode_t *vnode) {
ext2node_t *node = (ext2node_t *)vnode;
ext2fs_t *fs = (ext2fs_t *)vnode->vfs;
@@ -1564,6 +1596,7 @@ static int ext2_inactive(vnode_t *vnode) {
__assert(hashtable_remove(&fs->inodetable, &node->id, sizeof(node->id)) == 0);
MUTEX_RELEASE(&fs->inodetablelock);
vmmcache_truncate(vnode, 0);
freeinode(fs, &node->inode, node->id);
slab_free(nodecache, node);
@@ -1722,7 +1755,9 @@ static vops_t vnops = {
.munmap = ext2_munmap,
.unlink = ext2_unlink,
.inactive = ext2_inactive,
.rename = ext2_rename
.rename = ext2_rename,
.getpage = ext2_getpage,
.putpage = ext2_putpage
};
void ext2_init() {
+3 -1
View File
@@ -258,7 +258,9 @@ static vops_t vnops = {
.munmap = pipefs_enodev,
.getdents = pipefs_enodev,
.resize = pipefs_enodev,
.rename = pipefs_enodev
.rename = pipefs_enodev,
.putpage = pipefs_enodev,
.getpage = pipefs_enodev
};
static void ctor(scache_t *cache, void *obj) {
+3 -1
View File
@@ -184,7 +184,9 @@ static vops_t vnops = {
.getdents = sockfs_enodev,
.resize = sockfs_enodev,
.rename = sockfs_enodev,
.ioctl = sockfs_ioctl
.ioctl = sockfs_ioctl,
.putpage = sockfs_enodev,
.getpage = sockfs_enodev
};
static void ctor(scache_t *cache, void *obj) {
+46 -96
View File
@@ -12,6 +12,7 @@
#include <kernel/vmm.h>
#include <arch/cpu.h>
#include <string.h>
#include <kernel/vmmcache.h>
static scache_t *nodecache;
static tmpfsnode_t *newnode(vfs_t *vfs, int type);
@@ -31,60 +32,11 @@ static int tmpfs_mount(vfs_t **vfs, vnode_t *mountpoint, vnode_t *backing, void
#define DATAMMUFLAGS (ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC)
// expects to be called with node locked
static int resize(tmpfsnode_t *node, size_t size) {
if (size == node->attr.size || node->vnode.type != V_TYPE_REGULAR)
return 0;
// if smaller, don't deallocate any pages
if (size < node->attr.size) {
node->attr.size = size;
return 0;
}
// no mapping yet
if (node->pagecount == 0) {
size_t mapsize = ROUND_UP(size, PAGE_SIZE);
node->data = vmm_map(NULL, mapsize, VMM_FLAGS_ALLOCATE, DATAMMUFLAGS, NULL);
if (node->data == NULL)
return ENOMEM;
memset(node->data, 0, size);
node->pagecount = mapsize / PAGE_SIZE;
node->attr.size = size;
return 0;
}
size_t diff = size - node->attr.size;
// no need for new pages
if (size < node->pagecount * PAGE_SIZE) {
memset((void *)((uintptr_t)node->data + node->attr.size), 0, diff);
node->attr.size = size;
return 0;
}
// double the capacity or, if new size larger than double, resize to fit
size_t newcount = size > node->pagecount * 2 * PAGE_SIZE ? ROUND_UP(size, PAGE_SIZE) / PAGE_SIZE : node->pagecount * 2;
void *newdata = vmm_map(NULL, newcount, VMM_FLAGS_PAGESIZE | VMM_FLAGS_ALLOCATE, DATAMMUFLAGS, NULL);
if (newdata == NULL)
return ENOMEM;
memcpy(newdata, node->data, node->attr.size);
memset(((void *)(uintptr_t)newdata + node->attr.size), 0, diff);
vmm_unmap(node->data, node->pagecount, VMM_FLAGS_PAGESIZE);
node->pagecount = newcount;
node->data = newdata;
node->attr.size = size;
return 0;
}
static int tmpfs_getattr(vnode_t *node, vattr_t *attr, cred_t *cred) {
VOP_LOCK(node);
tmpfsnode_t *tmpnode = (tmpfsnode_t *)node;
*attr = tmpnode->attr;
attr->blocksused = node->type == V_TYPE_DIR ? 1 : tmpnode->pagecount;
attr->blocksused = ROUND_UP(attr->size, PAGE_SIZE) / PAGE_SIZE;
VOP_UNLOCK(node);
return 0;
}
@@ -233,50 +185,11 @@ static int tmpfs_root(vfs_t *vfs, vnode_t **vnode) {
}
static int tmpfs_read(vnode_t *node, void *buffer, size_t size, uintmax_t offset, int flags, size_t *readc, cred_t *cred) {
tmpfsnode_t *tmpnode = (tmpfsnode_t *)node;
if (node->type != V_TYPE_REGULAR)
return EINVAL;
VOP_LOCK(node);
// offset past end of file
if (offset >= tmpnode->attr.size) {
*readc = 0;
VOP_UNLOCK(node);
return 0;
}
// top of read goes past end of file
if (offset + size > tmpnode->attr.size)
size = tmpnode->attr.size - offset;
*readc = size;
memcpy(buffer, (void *)((uintptr_t)tmpnode->data + offset), size);
VOP_UNLOCK(node);
return 0;
__assert(!"tmpfs_read is handled by the page cache");
}
static int tmpfs_write(vnode_t *node, void *buffer, size_t size, uintmax_t offset, int flags, size_t *writec, cred_t *cred) {
tmpfsnode_t *tmpnode = (tmpfsnode_t *)node;
if (node->type != V_TYPE_REGULAR)
return EINVAL;
size_t top = offset + size;
VOP_LOCK(node);
if (top > tmpnode->attr.size) {
int err = resize(tmpnode, top);
if (err) {
VOP_UNLOCK(node);
return err;
}
}
*writec = size;
memcpy((void *)((uintptr_t)tmpnode->data + offset), buffer, size);
VOP_UNLOCK(node);
return 0;
__assert(!"tmpfs_read is handled by the page cache");
}
static int tmpfs_access(vnode_t *node, mode_t mode, cred_t *cred) {
@@ -530,6 +443,9 @@ static int tmpfs_getdents(vnode_t *node, dent_t *buffer, size_t count, uintmax_t
}
static int tmpfs_inactive(vnode_t *node) {
if (node->type == V_TYPE_REGULAR) {
vmmcache_truncate(node, 0);
}
freenode((tmpfsnode_t *)node);
return 0;
}
@@ -537,10 +453,44 @@ static int tmpfs_inactive(vnode_t *node) {
static int tmpfs_resize(vnode_t *node, size_t size, cred_t *) {
VOP_LOCK(node);
int err = resize((tmpfsnode_t *)node, size);
tmpfsnode_t *tmpfsnode = (tmpfsnode_t *)node;
if (size != tmpfsnode->attr.size && tmpfsnode->vnode.type == V_TYPE_REGULAR) {
if (tmpfsnode->attr.size > size)
vmmcache_truncate(node, size);
tmpfsnode->attr.size = size;
}
VOP_UNLOCK(node);
return err;
return 0;
}
static int tmpfs_getpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
int error = 0;
VOP_LOCK(node);
tmpfsnode_t *tmpfsnode = (tmpfsnode_t *)node;
if (offset >= tmpfsnode->attr.size)
error = ENXIO;
VOP_UNLOCK(node);
if (error)
return error;
// since tmpfs files now store their data on the vmmcache,
// all getpage will do will be set the page to 0 and pin it in memory
void *phy = pmm_getpageaddress(page);
void *phyhhdm = MAKE_HHDM(phy);
pmm_hold(phy);
page->flags |= PAGE_FLAGS_PINNED;
memset(phyhhdm, 0, PAGE_SIZE);
return 0;
}
static int tmpfs_putpage(vnode_t *node, uintmax_t offset, struct page_t *page) {
// putpage is a no-op on tmpfs
return 0;
}
static vfsops_t vfsops = {
@@ -568,7 +518,9 @@ static vops_t vnops = {
.munmap = tmpfs_munmap,
.getdents = tmpfs_getdents,
.resize = tmpfs_resize,
.rename = tmpfs_rename
.rename = tmpfs_rename,
.getpage = tmpfs_getpage,
.putpage = tmpfs_putpage
};
static tmpfsnode_t *newnode(vfs_t *vfs, int type) {
@@ -613,8 +565,6 @@ static void freenode(tmpfsnode_t *node) {
VOP_RELEASE(pnode);
__assert(hashtable_destroy(&node->children) == 0);
break;
case V_TYPE_REGULAR:
vmm_unmap(node->data, node->pagecount, VMM_FLAGS_PAGESIZE);
default:
break;
}
+170 -2
View File
@@ -7,6 +7,8 @@
#include <arch/cpu.h>
#include <kernel/poll.h>
#include <kernel/sock.h>
#include <kernel/vmmcache.h>
#include <kernel/block.h>
#define PATHNAME_MAX 512
#define MAXLINKDEPTH 64
@@ -144,13 +146,179 @@ int vfs_create(vnode_t *ref, char *path, vattr_t *attr, int type, vnode_t **node
return err;
}
static void bytestopages(uintmax_t offset, size_t size, uintmax_t *pageoffset, size_t *pagecount, uintmax_t *startoffset) {
*pageoffset = offset / PAGE_SIZE;
*startoffset = offset % PAGE_SIZE;
size_t end = offset + size;
size_t toppage = ROUND_UP(end, PAGE_SIZE) / PAGE_SIZE;
*pagecount = toppage - *pageoffset;
}
int vfs_write(vnode_t *node, void *buffer, size_t size, uintmax_t offset, size_t *written, int flags) {
int err = VOP_WRITE(node, buffer, size, offset, flags, written, getcred());
int err;
// TODO skip caching flag, cred check
if (node->type == V_TYPE_REGULAR || node->type == V_TYPE_BLKDEV) {
*written = 0;
// can't write a size 0 buffer
if (size == 0)
return 0;
// overflow
if (size + offset < offset) {
err = EINVAL;
goto leave;
}
MUTEX_ACQUIRE(&node->sizelock, false);
vattr_t attr;
err = VOP_GETATTR(node, &attr, getcred());
if (err)
goto leave;
size_t newsize = size + offset > attr.size ? size + offset : 0;
if (node->type == V_TYPE_REGULAR && newsize) {
// do resize stuff if regular and applicable
err = VOP_RESIZE(node, newsize, &_cpu()->thread->proc->cred);
if (err)
goto leave;
} else if (node->type == V_TYPE_BLKDEV) {
// else just get the disk size and limit the read size
blockdesc_t blockdesc;
int r;
err = VOP_IOCTL(node, BLOCK_IOCTL_GETDESC, &blockdesc, &r);
__assert(err == 0);
size_t bytesize = blockdesc.blockcapacity * blockdesc.blocksize;
if (offset >= bytesize)
goto leave;
size = min(size + offset, bytesize) - offset;
}
uintmax_t pageoffset, pagecount, startoffset;
bytestopages(offset, size, &pageoffset, &pagecount, &startoffset);
page_t *page = NULL;
if (startoffset) {
// unaligned first page
err = vmmcache_getpage(node, pageoffset * PAGE_SIZE, &page);
if (err)
goto leave;
size_t writesize = min(PAGE_SIZE - startoffset, size);
void *address = MAKE_HHDM(pmm_getpageaddress(page));
memcpy((void *)((uintptr_t)address + startoffset), buffer, writesize);
vmmcache_makedirty(page);
pmm_release(FROM_HHDM(address));
*written += writesize;
pageoffset += 1;
pagecount -= 1;
}
for (uintmax_t offset = 0; offset < pagecount * PAGE_SIZE; offset += PAGE_SIZE) {
// the other pages
err = vmmcache_getpage(node, pageoffset * PAGE_SIZE + offset, &page);
if (err)
goto leave;
size_t writesize = min(PAGE_SIZE, size - *written);
void *address = MAKE_HHDM(pmm_getpageaddress(page));
memcpy(address, (void *)((uintptr_t)buffer + *written), writesize);
vmmcache_makedirty(page);
pmm_release(FROM_HHDM(address));
*written += writesize;
}
leave:
MUTEX_RELEASE(&node->sizelock);
} else {
// special file, just write as its not being cached
err = VOP_WRITE(node, buffer, size, offset, flags, written, getcred());
}
return err;
}
int vfs_read(vnode_t *node, void *buffer, size_t size, uintmax_t offset, size_t *bytesread, int flags) {
int err = VOP_READ(node, buffer, size, offset, flags, bytesread, getcred());
// TODO skip caching flag, cred check
int err;
if (node->type == V_TYPE_REGULAR || node->type == V_TYPE_BLKDEV) {
*bytesread = 0;
// can't read 0 bytes from the cache
if (size == 0)
return 0;
// overflow
if (size + offset < offset) {
err = EINVAL;
goto leave;
}
MUTEX_ACQUIRE(&node->sizelock, false);
size_t nodesize = 0;
if (node->type == V_TYPE_REGULAR) {
vattr_t attr;
err = VOP_GETATTR(node, &attr, getcred());
if (err)
goto leave;
nodesize = attr.size;
} else {
blockdesc_t blockdesc;
int r;
err = VOP_IOCTL(node, BLOCK_IOCTL_GETDESC, &blockdesc, &r);
__assert(err == 0);
nodesize = blockdesc.blockcapacity * blockdesc.blocksize;
}
// read past end of file?
if (offset >= nodesize)
goto leave;
size = min(size + offset, nodesize) - offset;
uintmax_t pageoffset, pagecount, startoffset;
bytestopages(offset, size, &pageoffset, &pagecount, &startoffset);
page_t *page = NULL;
if (startoffset) {
// unaligned first page
err = vmmcache_getpage(node, pageoffset * PAGE_SIZE, &page);
if (err)
goto leave;
size_t readsize = min(PAGE_SIZE - startoffset, size);
void *address = MAKE_HHDM(pmm_getpageaddress(page));
memcpy(buffer, (void *)((uintptr_t)address + startoffset), readsize);
pmm_release(FROM_HHDM(address));
*bytesread += readsize;
pageoffset += 1;
pagecount -= 1;
}
for (uintmax_t offset = 0; offset < pagecount * PAGE_SIZE; offset += PAGE_SIZE) {
// the other pages
err = vmmcache_getpage(node, pageoffset * PAGE_SIZE + offset, &page);
if (err)
goto leave;
size_t readsize = min(PAGE_SIZE, size - *bytesread);
void *address = MAKE_HHDM(pmm_getpageaddress(page));
memcpy((void *)((uintptr_t)buffer + *bytesread), address, readsize);
pmm_release(FROM_HHDM(address));
*bytesread += readsize;
}
leave:
MUTEX_RELEASE(&node->sizelock);
} else {
// special file, just read as size doesn't matter
err = VOP_READ(node, buffer, size, offset, flags, bytesread, getcred());
}
return err;
}
+6
View File
@@ -10,6 +10,11 @@
#define PMM_SECTION_DEFAULT 2
#define PAGE_FLAGS_FREE 1
#define PAGE_FLAGS_TRUNCATED 2
#define PAGE_FLAGS_PINNED 4
#define PAGE_FLAGS_DIRTY 8
#define PAGE_FLAGS_READY 16
#define PAGE_FLAGS_ERROR 32
typedef struct page_t {
struct vnode_t *backing;
@@ -34,6 +39,7 @@ typedef struct page_t {
void *pmm_allocpage(int section);
page_t *pmm_getpage(void *addr);
void *pmm_getpageaddress(page_t *);
void pmm_hold(void *addr);
void pmm_release(void *addr);
void pmm_makefree(void *address, size_t count);
-4
View File
@@ -16,10 +16,6 @@ typedef struct tmpfsnode_t {
vattr_t attr;
union {
hashtable_t children;
struct {
void *data;
size_t pagecount;
};
char *link;
};
} tmpfsnode_t;
+7
View File
@@ -60,12 +60,14 @@ typedef struct vfs_t {
typedef struct vnode_t {
struct vops_t *ops;
mutex_t lock;
mutex_t sizelock;
int refcount;
int flags;
int type;
vfs_t *vfs;
vfs_t *vfsmounted;
void *socketbinding;
struct page_t *pages;
} vnode_t;
typedef struct vfsops_t {
@@ -101,6 +103,8 @@ typedef struct vops_t {
int (*maxseek)(vnode_t *node, size_t *max);
int (*resize)(vnode_t *node, size_t newsize, cred_t *cred);
int (*rename)(vnode_t *source, char *oldname, vnode_t *target, char *newname, int flags);
int (*getpage)(vnode_t *node, uintmax_t offset, struct page_t *page);
int (*putpage)(vnode_t *node, uintmax_t offset, struct page_t *page);
} vops_t;
#define VFS_INIT(v, o, f) \
@@ -118,6 +122,7 @@ typedef struct vops_t {
#define VOP_INIT(vn, o, f, t, v) \
(vn)->ops = o; \
MUTEX_INIT(&(vn)->lock); \
MUTEX_INIT(&(vn)->sizelock); \
(vn)->refcount = 1; \
(vn)->flags = f; \
(vn)->type = t; \
@@ -149,6 +154,8 @@ typedef struct vops_t {
#define VOP_MAXSEEK(v, rp) ((v)->ops->maxseek ? (v)->ops->maxseek(v, rp) : ENOTTY)
#define VOP_RESIZE(v, s, c) (v)->ops->resize(v, s, c)
#define VOP_RENAME(s, o, t, n, f) (s)->ops->rename(s, o, t, n, f)
#define VOP_GETPAGE(v, o, p) (v)->ops->getpage(v, o, p)
#define VOP_PUTPAGE(v, o, p) (v)->ops->putpage(v, o, p)
#define VOP_HOLD(v) __atomic_add_fetch(&(v)->refcount, 1, __ATOMIC_SEQ_CST)
#define VOP_RELEASE(v) {\
if (__atomic_sub_fetch(&(v)->refcount, 1, __ATOMIC_SEQ_CST) == 0) {\
+15
View File
@@ -0,0 +1,15 @@
#ifndef _VMMCACHE_H
#define _VMMCACHE_H
#include <kernel/pmm.h>
#include <kernel/vfs.h>
void vmmcache_init();
int vmmcache_getpage(vnode_t *vnode, uintmax_t offset, page_t **res);
int vmmcache_takepage(page_t *page);
int vmmcache_makedirty(page_t *page);
int vmmcache_truncate(vnode_t *vnode, uintmax_t offset);
int vmmcache_syncvnode(vnode_t *vnode);
int vmmcache_sync();
#endif
+1
View File
@@ -32,6 +32,7 @@ void arch_mmu_switch(pagetableptr_t table);
void *arch_mmu_getphysical(pagetableptr_t table, void *vaddr);
bool arch_mmu_ispresent(pagetableptr_t table, void *vaddr);
bool arch_mmu_iswritable(pagetableptr_t table, void *vaddr);
bool arch_mmu_isdirty(pagetableptr_t table, void *vaddr);
pagetableptr_t arch_mmu_newtable();
void arch_mmu_init();
void arch_mmu_apswitch();
-3
View File
@@ -1,3 +0,0 @@
void pgcache_init() {
}
+52 -3
View File
@@ -5,6 +5,7 @@
#include <arch/mmu.h>
#include <mutex.h>
#include <util.h>
#include <kernel/vmmcache.h>
uintptr_t hhdmbase;
static size_t memorysize;
@@ -101,9 +102,29 @@ static void removefromfreelist(page_t *page) {
--freepagecount;
}
static void internalhold(page_t *page) {
__atomic_add_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST);
if (page->refcount == 1) {
// this is only valid on standby pages, in case of free pages its an use after free
__assert((page->flags & PAGE_FLAGS_FREE) == 0);
removefromfreelist(page);
}
}
page_t *pmm_getpage(void *address) {
return &pages[((uintptr_t)address / PAGE_SIZE)];
}
void *pmm_getpageaddress(page_t *page) {
return (void *)(PAGE_GETID(page) * PAGE_SIZE);
}
void pmm_hold(void *addr) {
page_t *page = &pages[((uintptr_t)addr / PAGE_SIZE)];
__atomic_add_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST);
MUTEX_ACQUIRE(&freelistmutex, false);
internalhold(page);
MUTEX_RELEASE(&freelistmutex);
}
void pmm_release(void *addr) {
@@ -124,13 +145,14 @@ static void doalloc(page_t *page) {
__assert(page->refcount == 0);
memset(page, 0, sizeof(page_t));
page->refcount = 1;
page->flags &= ~PAGE_FLAGS_FREE;
}
void *pmm_allocpage(int section) {
retry:
MUTEX_ACQUIRE(&freelistmutex, false);
page_t *page = NULL;
// try to take a free anonymous page
for (int i = section; i >= 0; --i) {
page = freelists[i];
if (page) {
@@ -139,9 +161,34 @@ void *pmm_allocpage(int section) {
}
}
// TODO get page from standby list and call to remove from cache
bool cachepage = false;
// if that wasn't possible, try to take from the cache standby list
// TODO do this FIFO rather than LIFO
if (page == NULL) {
for (int i = section; i >= 0; --i) {
page = standbylists[i];
if (page) {
cachepage = true;
internalhold(page);
break;
}
}
}
MUTEX_RELEASE(&freelistmutex);
if (cachepage && vmmcache_takepage(page) == EAGAIN) {
// someone already got the page from the cache between us holding it and taking it
page = NULL;
// retry it from the start, as an anonymous page could have been released while the lock was not held
goto retry;
} else if (cachepage) {
// we got the page from the cache, all is good and we hold the only reference to it.
// set the refcount to 0, as expected by the doalloc call
page->refcount = 0;
}
void *address = NULL;
if (page) {
address = (void *)(PAGE_GETID(page) * PAGE_SIZE);
@@ -207,6 +254,8 @@ void pmm_init() {
MUTEX_INIT(&freelistmutex);
}
// XXX pmm_alloc won't be able to take pages from the page cache when the allocation size is over 1 page
void *pmm_alloc(size_t size, int section) {
__assert(size);
// pmm_allocpage is more suited for single page allocations, so use that instead
+54 -17
View File
@@ -6,6 +6,7 @@
#include <limine.h>
#include <string.h>
#include <kernel/slab.h>
#include <kernel/vmmcache.h>
#define RANGE_TOP(x) (void *)((uintptr_t)x->start + x->size)
@@ -217,9 +218,18 @@ static void destroyrange(vmmrange_t *range, uintmax_t _offset, size_t size, int
proc_t *proc = thread ? thread->proc : NULL;
cred_t *cred = proc ? &proc->cred : NULL;
if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED)) {
__assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
if ((range->flags & VMM_FLAGS_FILE) && ((range->flags & VMM_FLAGS_SHARED) || range->vnode->type == V_TYPE_CHDEV)) {
// shared file mapping or character device mapping
if (range->vnode->type == V_TYPE_CHDEV) {
// character device mapping
__assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
} else if (arch_mmu_iswritable(_cpu()->vmmctx->pagetable, vaddr) == false || arch_mmu_isdirty(_cpu()->vmmctx->pagetable, vaddr) == false) {
// dirty page cache mapping
vmmcache_makedirty(pmm_getpage(physical));
pmm_release(physical);
}
} else {
// anonymous or private non character device mapping
pmm_release(physical);
arch_mmu_unmap(_cpu()->vmmctx->pagetable, vaddr);
}
@@ -289,6 +299,15 @@ static void unmap(vmmspace_t *space, void *address, size_t size) {
}
static void printspace(vmmspace_t *space) {
printf("vmm: ranges:\n");
vmmrange_t *range = space->ranges;
while (range) {
printf("vmm: address %p size %lx flags %x\n", range->start, range->size, range->flags);
range = range->next;
}
}
static void *zeropage;
bool vmm_pagefault(void *addr, bool user, int actions) {
@@ -335,8 +354,28 @@ bool vmm_pagefault(void *addr, bool user, int actions) {
// page not present in the page tables
if (range->flags & VMM_FLAGS_FILE) {
uintmax_t mapoffset = (uintptr_t)addr - (uintptr_t)range->start;
__assert(VOP_MMAP(range->vnode, addr, range->offset + mapoffset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
status = true;
if (range->vnode->type != V_TYPE_BLKDEV && range->vnode->type != V_TYPE_REGULAR) {
// map non cacheable vnodes
__assert(VOP_MMAP(range->vnode, addr, range->offset + mapoffset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0);
status = true;
} else {
// cacheable vnode
page_t *res = NULL;
int error = vmmcache_getpage(range->vnode, range->offset + mapoffset, &res);
if (error == ENXIO) {
// TODO if error returned == ENXIO, raise an urgent SIGBUS on the calling thread
__assert(!"TODO: sigbus");
} else if (error) {
printf("vmm: error on vmmcache_getpage(): %d\n", error);
status = false;
} else {
status = arch_mmu_map(_cpu()->vmmctx->pagetable, pmm_getpageaddress(res), addr, range->mmuflags & ~ARCH_MMU_FLAGS_WRITE);
if (!status) {
printf("vmm: out of memory to map file into address space\n");
pmm_release(pmm_getpageaddress(res));
}
}
}
} else {
// anonymous memory. map the zero'd page
status = arch_mmu_map(_cpu()->vmmctx->pagetable, zeropage, addr, range->mmuflags & ~ARCH_MMU_FLAGS_WRITE);
@@ -346,12 +385,16 @@ bool vmm_pagefault(void *addr, bool user, int actions) {
pmm_hold(zeropage);
}
}
} else if (arch_mmu_iswritable(_cpu()->vmmctx->pagetable, addr) == false){
} else if (arch_mmu_iswritable(_cpu()->vmmctx->pagetable, addr) == false) {
// page present but not writeable in the page tables
void *oldphys = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr);
if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED)) {
// TODO shared file mappings (set as dirty and set as writable on PTE)
arch_mmu_remap(_cpu()->vmmctx->pagetable, oldphys, addr, range->mmuflags);
if (range->vnode->type != V_TYPE_CHDEV)
vmmcache_makedirty(pmm_getpage(oldphys));
status = true;
} else {
void *oldphys = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr);
// do copy on write
void *newphys = pmm_allocpage(PMM_SECTION_DEFAULT);
if (newphys == NULL) {
@@ -360,7 +403,9 @@ bool vmm_pagefault(void *addr, bool user, int actions) {
} else {
memcpy(MAKE_HHDM(newphys), MAKE_HHDM(oldphys), PAGE_SIZE);
arch_mmu_remap(_cpu()->vmmctx->pagetable, newphys, addr, range->mmuflags);
pmm_release(oldphys);
if ((range->flags & VMM_FLAGS_FILE) == 0 || range->vnode->type != V_TYPE_CHDEV)
pmm_release(oldphys);
status = true;
}
}
@@ -429,6 +474,7 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags,
if (flags & VMM_FLAGS_FILE) {
// XXX make sure that writes can't happen to executable memory mapped files and check file permissions
vmmfiledesc_t *desc = private;
__assert((desc->offset % PAGE_SIZE) == 0);
range->vnode = desc->node;
range->offset = desc->offset;
VOP_HOLD(desc->node);
@@ -602,15 +648,6 @@ void vmm_switchcontext(vmmcontext_t *ctx) {
arch_mmu_switch(ctx->pagetable);
}
static void printspace(vmmspace_t *space) {
printf("vmm: ranges:\n");
vmmrange_t *range = space->ranges;
while (range) {
printf("vmm: address %p size %lx flags %x\n", range->start, range->size, range->flags);
range = range->next;
}
}
extern void *_text_start;
extern void *_data_start;
extern void *_rodata_start;
+316
View File
@@ -0,0 +1,316 @@
#include <kernel/vmmcache.h>
#include <kernel/vmm.h>
#include <util.h>
#include <logging.h>
#define TABLE_SIZE 4096
#define WRITER_TICK_SECONDS 1
static mutex_t mutex;
static page_t **table;
static thread_t *writerthread;
static semaphore_t sync;
#define HOLD_LOCK() \
MUTEX_ACQUIRE(&mutex, false);
#define RELEASE_LOCK() \
MUTEX_RELEASE(&mutex);
static inline uint64_t fnv1ahash(void *buffer, size_t size);
static uintmax_t getentry(vnode_t *vnode, uintmax_t offset) {
struct {
vnode_t *vnode;
uintmax_t offset;
} tmp;
tmp.vnode = vnode;
tmp.offset = offset;
return fnv1ahash(&tmp, sizeof(tmp)) % TABLE_SIZE;
}
// assumes lock is held
static page_t *findpage(vnode_t *vnode, uintmax_t offset) {
uintmax_t entry = getentry(vnode, offset);
page_t *page = table[entry];
while (page) {
if (page->backing == vnode && page->offset == offset)
break;
page = page->hashnext;
}
return page;
}
// assumes lock is held
static void putpage(page_t *page) {
uintmax_t entry = getentry(page->backing, page->offset);
// add to table list
page->hashprev = NULL;
page->hashnext = table[entry];
if (table[entry])
table[entry]->hashprev = page;
table[entry] = page;
// add to vnode list
// XXX this could be a bit slow if a large portion of the file is in memory and it gets purged
page->vnodeprev = NULL;
page->vnodenext = page->backing->pages;
if (page->backing->pages)
page->backing->pages->vnodeprev = page;
page->backing->pages = page;
}
// assumes lock is held
static void removepage(page_t *page) {
uintmax_t entry = getentry(page->backing, page->offset);
// remove from table list
if (page->hashnext)
page->hashnext->hashprev = page->hashprev;
if (page->hashprev)
page->hashprev->hashnext = page->hashnext;
else
table[entry] = page->hashnext;
page->hashnext = NULL;
page->hashprev = NULL;
// remove from vnode list
if (page->vnodenext)
page->vnodenext->vnodeprev = page->vnodeprev;
if (page->vnodeprev)
page->vnodeprev->vnodenext = page->vnodenext;
else
page->backing->pages = page->vnodenext;
page->vnodenext = NULL;
page->vnodeprev = NULL;
}
int vmmcache_getpage(vnode_t *vnode, uintmax_t offset, page_t **res) {
__assert(vnode->type == V_TYPE_REGULAR || vnode->type == V_TYPE_BLKDEV);
__assert((offset % PAGE_SIZE) == 0);
retry_err:
HOLD_LOCK();
page_t *newpage = NULL;
volatile page_t *page = findpage(vnode, offset);
retry:
if (page) {
// page is present in the page cache
pmm_hold(pmm_getpageaddress((page_t *)page));
RELEASE_LOCK();
// in the case of a retry, release the allocated page here
if (newpage)
pmm_release(newpage);
// wait for page to be ready
while ((page->flags & (PAGE_FLAGS_READY | PAGE_FLAGS_ERROR)) == 0)
sched_yield(); // TODO proper sleeping mechanism here
if (page->flags & PAGE_FLAGS_ERROR) {
// the thread handling the page in failed to read it, we should retry it and see whats up
pmm_release((page_t *)page);
goto retry_err;
}
*res = (page_t *)page;
} else {
// page is not present in the cache, we will have to load it in
RELEASE_LOCK();
void *address = pmm_allocpage(PMM_SECTION_DEFAULT);
if (address == NULL)
return ENOMEM;
newpage = pmm_getpage(address);
HOLD_LOCK();
// while the lock wasn't being held, the page could have potentially been added to the cache
// check for it again and return from the function as if it was always there in the first place
page = findpage(vnode, offset);
if (page)
goto retry;
newpage->backing = vnode;
newpage->offset = offset;
// add it to the page cache
putpage(newpage);
RELEASE_LOCK();
int error = VOP_GETPAGE(vnode, offset, newpage);
// TODO wake up the sleeping threads when implemented
if (error) {
// an error happened with GETPAGE, remove the page from the cache,
// tell the sleeping threads that something happened and free the page
// by setting backing to null so it gets treated as an anonymous page again
HOLD_LOCK();
removepage(newpage);
newpage->flags |= PAGE_FLAGS_ERROR;
newpage->backing = NULL;
newpage->offset = 0;
RELEASE_LOCK();
pmm_release(newpage);
return error;
}
newpage->flags |= PAGE_FLAGS_READY;
*res = newpage;
}
RELEASE_LOCK();
return 0;
}
int vmmcache_takepage(page_t *page) {
HOLD_LOCK();
// someone called vmmcache_getpage() and got this page while the lock wasn't held
// return an error status to the caller
if (page->refcount > 1) {
RELEASE_LOCK();
return EAGAIN;
}
__assert(page->refcount == 1);
__assert((page->flags & PAGE_FLAGS_DIRTY) == 0);
// XXX maybe just not allow truncated pages to take up space like this?
if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
// the page needs to be removed from the cache to continue
removepage(page);
}
RELEASE_LOCK();
return 0;
}
int vmmcache_truncate(vnode_t *vnode, uintmax_t offset) {
HOLD_LOCK();
page_t *pagelist = NULL;
page_t *page = vnode->pages;
while (page) {
page_t *oldpage = page;
page = page->vnodenext;
// only truncate past a certain offset
if (oldpage->offset < offset)
continue;
oldpage->flags |= PAGE_FLAGS_TRUNCATED;
removepage(oldpage);
oldpage->vnodenext = pagelist;
pagelist = oldpage;
}
RELEASE_LOCK();
// make sure to unref if they are pinned
while (pagelist) {
page_t *page = pagelist;
pagelist = pagelist->vnodenext;
if (page->flags & PAGE_FLAGS_PINNED)
pmm_release(pmm_getpageaddress(page));
}
return 0;
}
int vmmcache_syncvnode(vnode_t *vnode) {
__assert(!"unimplemented");
}
int vmmcache_sync() {
semaphore_signal(&sync);
return 0;
}
static page_t *dirtylist;
int vmmcache_makedirty(page_t *page) {
HOLD_LOCK();
if ((page->flags & (PAGE_FLAGS_DIRTY | PAGE_FLAGS_TRUNCATED)) == 0) {
// page is neither dirty nor truncated, add to dirty list and hold the page and vnode
page->flags |= PAGE_FLAGS_DIRTY;
page->writeprev = NULL;
page->writenext = dirtylist;
if (dirtylist)
dirtylist->writeprev = page;
dirtylist = page;
pmm_hold(pmm_getpageaddress(page));
__assert(page->backing);
VOP_HOLD(page->backing);
}
RELEASE_LOCK();
return 0;
}
static void tick(context_t *, dpcarg_t arg) {
vmmcache_sync();
}
static void writer() {
timerentry_t timerentry;
// this will be inserted on some random cpu's timer, but it will always work after that
interrupt_set(false);
timer_insert(_cpu()->timer, &timerentry, tick, NULL, (uintmax_t)WRITER_TICK_SECONDS * 1000000, true);
interrupt_set(true);
for (;;) {
HOLD_LOCK();
// TODO make FIFO rather than LIFO
volatile page_t *page = dirtylist;
if (page == NULL) {
RELEASE_LOCK();
semaphore_wait(&sync, false);
continue;
}
__assert(page->flags & PAGE_FLAGS_DIRTY);
page->flags &= ~PAGE_FLAGS_DIRTY;
dirtylist = page->writenext;
if (dirtylist)
dirtylist->writeprev = NULL;
page->writenext = NULL;
RELEASE_LOCK();
if ((page->flags & PAGE_FLAGS_TRUNCATED) == 0) {
VOP_PUTPAGE(page->backing, page->offset, (page_t *)page);
VOP_RELEASE(page->backing);
}
pmm_release(pmm_getpageaddress((page_t *)page));
}
}
void vmmcache_init() {
MUTEX_INIT(&mutex);
table = vmm_map(NULL, TABLE_SIZE * sizeof(page_t *), VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
__assert(table);
memset(table, 0, TABLE_SIZE * sizeof(page_t *));
SEMAPHORE_INIT(&sync, 0);
writerthread = sched_newthread(writer, PAGE_SIZE * 4, 1, NULL, NULL);
__assert(writerthread);
sched_queue(writerthread);
vmmcache_sync();
}
+7
View File
@@ -57,11 +57,18 @@ syscallret_t syscall_mmap(context_t *context, void *hint, size_t len, int prot,
vmmfiledesc_t vfd;
file_t *file = NULL;
if (isfile) {
// make sure offset is page aligned
if (offset % PAGE_SIZE) {
ret.errno = EINVAL;
return ret;
}
file = fd_get(fd);
if (file == NULL) {
ret.errno = EBADF;
return ret;
}
vfd.node = file->vnode;
vfd.offset = offset;
}
+2
View File
@@ -61,7 +61,9 @@ syscallret_t syscall_openat(context_t *context, int dirfd, const char *path, int
goto cleanup;
if (vnode->type == V_TYPE_REGULAR && (flags & O_TRUNC)) {
MUTEX_ACQUIRE(&vnode->sizelock, false);
ret.errno = VOP_RESIZE(vnode, 0, &_cpu()->thread->proc->cred);
MUTEX_RELEASE(&vnode->sizelock);
if (ret.errno)
goto cleanup;
}
+3
View File
@@ -1,4 +1,5 @@
#include <kernel/syscalls.h>
#include <mutex.h>
syscallret_t syscall_ftruncate(int fd, size_t size) {
syscallret_t ret = {
@@ -16,7 +17,9 @@ syscallret_t syscall_ftruncate(int fd, size_t size) {
goto cleanup;
}
MUTEX_ACQUIRE(&file->vnode->sizelock, false);
ret.errno = VOP_RESIZE(file->vnode, size, &_cpu()->thread->proc->cred);
MUTEX_RELEASE(&file->vnode->sizelock);
ret.ret = ret.errno ? -1 : 0;
cleanup: