diff --git a/kernel-src/arch/x86-64/smp.c b/kernel-src/arch/x86-64/smp.c index d2a2e499..1a034d58 100644 --- a/kernel-src/arch/x86-64/smp.c +++ b/kernel-src/arch/x86-64/smp.c @@ -58,8 +58,8 @@ void arch_smp_haltallothers(void) { } size_t arch_smp_get_cpu_count(void) { - if (cmdline_get("nosmp")) - return 1; + //if (cmdline_get("nosmp")) + // return 1; struct limine_smp_response *response = smprequest.response; if (response == NULL) @@ -95,7 +95,7 @@ void arch_smp_wakeup(void) { smp_cpus = alloc(response->cpu_count * sizeof(cpu_t *)); __assert(smp_cpus); - void (*wakeupfn)(struct limine_smp_info *) = cmdline_get("nosmp") ? cpuwakeuphalt : cpuwakeup; + void (*wakeupfn)(struct limine_smp_info *) = cpuwakeup;//cmdline_get("nosmp") ? cpuwakeuphalt : cpuwakeup; // make the other processors jump to cpuwakeup() for (int i = 0; i < response->cpu_count; ++i) { diff --git a/kernel-src/fs/devfs.c b/kernel-src/fs/devfs.c index ef1b9496..4374a4bd 100644 --- a/kernel-src/fs/devfs.c +++ b/kernel-src/fs/devfs.c @@ -254,6 +254,13 @@ int devfs_maxseek(vnode_t *node, size_t *max) { return devnode->devops->maxseek ? devnode->devops->maxseek(devnode->attr.rdevminor, max) : ENOTTY; } +static void destroy_node(devnode_t *node) { + memset(node, 0, sizeof(devnode_t)); + VOP_INIT(&node->vnode, &vnops, 0, 0, NULL) + node->attr.inode = ++currentinode; + slab_free(nodecache, node); +} + int devfs_inactive(vnode_t *node) { INTERNAL_LOCK(node); devnode_t *devnode = (devnode_t *)node; @@ -267,7 +274,7 @@ int devfs_inactive(vnode_t *node) { if (devnode->devops && devnode->devops->inactive) devnode->devops->inactive(devnode->attr.rdevminor); - slab_free(nodecache, node); + destroy_node(devnode); return 0; } @@ -311,13 +318,13 @@ int devfs_create(vnode_t *parent, char *name, vattr_t *attr, int type, vnode_t * if (type == V_TYPE_DIR) { int error = hashtable_init(&node->children, 100); if (error) { - slab_free(nodecache, node); + destroy_node(node); INTERNAL_UNLOCK(parent); return error; } if (hashtable_set(&node->children, node, ".", 1, true) || hashtable_set(&node->children, parent, "..", 2, true)) { hashtable_destroy(&node->children); - slab_free(nodecache, node); + destroy_node(node); INTERNAL_UNLOCK(parent); return error; } @@ -328,7 +335,7 @@ int devfs_create(vnode_t *parent, char *name, vattr_t *attr, int type, vnode_t * if (type == V_TYPE_DIR) hashtable_destroy(&node->children); INTERNAL_UNLOCK(parent); - slab_free(nodecache, node); + destroy_node(node); return error; } @@ -476,16 +483,17 @@ static vops_t vnops = { .unlock = devfs_unlock }; -static void ctor(scache_t *cache, void *obj) { +static bool ctor(scache_t *cache, void *obj) { devnode_t *node = obj; memset(node, 0, sizeof(devnode_t)); VOP_INIT(&node->vnode, &vnops, 0, 0, NULL) node->attr.inode = ++currentinode; + return true; } void devfs_init() { __assert(hashtable_init(&devtable, 50) == 0); - nodecache = slab_newcache(sizeof(devnode_t), 0, ctor, ctor); + nodecache = slab_newcache(sizeof(devnode_t), 0, ctor, NULL); MUTEX_INIT(&tablelock); __assert(nodecache); @@ -525,7 +533,7 @@ int devfs_register(devops_t *devops, char *name, int type, int major, int minor, int error = hashtable_set(&devtable, master, key, sizeof(key), true); MUTEX_RELEASE(&tablelock); if (error) { - slab_free(nodecache, master); + destroy_node(master); return error; } @@ -540,7 +548,7 @@ int devfs_register(devops_t *devops, char *name, int type, int major, int minor, char lastcomp[strlen(name) + 1]; error = vfs_lookup(&dir, (vnode_t *)devfsroot, name, lastcomp, VFS_LOOKUP_PARENT); if (error) { - slab_free(nodecache, master); + destroy_node(master); return error; } @@ -556,7 +564,7 @@ int devfs_register(devops_t *devops, char *name, int type, int major, int minor, MUTEX_ACQUIRE(&tablelock); __assert(hashtable_remove(&devtable, key, sizeof(key) == 0)); MUTEX_RELEASE(&tablelock); - slab_free(nodecache, master); + destroy_node(master); return error; } diff --git a/kernel-src/fs/file.c b/kernel-src/fs/file.c index c6af1fb4..f23966e0 100644 --- a/kernel-src/fs/file.c +++ b/kernel-src/fs/file.c @@ -13,19 +13,20 @@ static scache_t *filecache; cleanfile(f); \ } -static void ctor(scache_t *cache, void *obj) { +static bool ctor(scache_t *cache, void *obj) { file_t *file = obj; file->vnode = NULL; MUTEX_INIT(&file->mutex); - file->refcount = 1; file->offset = 0; file->flags = 0; file->mode = 0; + file->refcount = 1; + return true; } static file_t* newfile() { if (filecache == NULL) { - filecache = slab_newcache(sizeof(file_t), 0, ctor, ctor); + filecache = slab_newcache(sizeof(file_t), 0, ctor, NULL); __assert(filecache); } @@ -37,6 +38,13 @@ static void cleanfile(file_t *file) { vfs_close(file->vnode, fileflagstovnodeflags(file->flags)); VOP_RELEASE(file->vnode); } + + file->vnode = NULL; + file->refcount = 1; + file->offset = 0; + file->flags = 0; + file->mode = 0; + MUTEX_INIT(&file->mutex); slab_free(filecache, file); } diff --git a/kernel-src/fs/pipefs.c b/kernel-src/fs/pipefs.c index e073cc6d..3aa2ccdb 100644 --- a/kernel-src/fs/pipefs.c +++ b/kernel-src/fs/pipefs.c @@ -342,6 +342,13 @@ int pipefs_inactive(vnode_t *node) { pipenode_t *pipenode = (pipenode_t *)node; INTERNAL_LOCK(node); ringbuffer_destroy(&pipenode->data); + // TODO write this properly using cache ctor and dtor + memset(node, 0, sizeof(pipenode_t)); + VOP_INIT(node, &vnops, 0, V_TYPE_FIFO, NULL); + pipenode->attr.inode = ++currentinode; + POLL_INITHEADER(&pipenode->pollheader); + EVENT_INITHEADER(&pipenode->writeopenevent); + EVENT_INITHEADER(&pipenode->readopenevent); slab_free(nodecache, node); return 0; } @@ -389,7 +396,7 @@ static vops_t vnops = { .unlock = pipefs_unlock }; -static void ctor(scache_t *cache, void *obj) { +static bool ctor(scache_t *cache, void *obj) { pipenode_t *node = obj; memset(node, 0, sizeof(pipenode_t)); VOP_INIT(&node->vnode, &vnops, 0, V_TYPE_FIFO, NULL); @@ -397,10 +404,11 @@ static void ctor(scache_t *cache, void *obj) { POLL_INITHEADER(&node->pollheader); EVENT_INITHEADER(&node->writeopenevent); EVENT_INITHEADER(&node->readopenevent); + return true; } void pipefs_init() { - nodecache = slab_newcache(sizeof(pipenode_t), 0, ctor, ctor); + nodecache = slab_newcache(sizeof(pipenode_t), 0, ctor, NULL); __assert(nodecache); } diff --git a/kernel-src/fs/sockfs.c b/kernel-src/fs/sockfs.c index 2064e93b..a3af5902 100644 --- a/kernel-src/fs/sockfs.c +++ b/kernel-src/fs/sockfs.c @@ -204,10 +204,16 @@ int sockfs_ioctl(vnode_t *node, unsigned long request, void *arg, int *result, c return 0; } +static vops_t vnops; + int sockfs_inactive(vnode_t *node) { socket_t *socket = SOCKFS_SOCKET_FROM_NODE(node); + socketnode_t *socketnode = (socketnode_t *)node; INTERNAL_LOCK(node); socket->ops->destroy(socket); + memset(node, 0, sizeof(socketnode_t)); + VOP_INIT(&socketnode->vnode, &vnops, 0, V_TYPE_SOCKET, NULL); + //socketnode->attr.inode = ++currentinode; slab_free(nodecache, node); return 0; } @@ -256,15 +262,16 @@ static vops_t vnops = { .unlock = sockfs_unlock }; -static void ctor(scache_t *cache, void *obj) { +static bool ctor(scache_t *cache, void *obj) { socketnode_t *node = obj; memset(node, 0, sizeof(socketnode_t)); VOP_INIT(&node->vnode, &vnops, 0, V_TYPE_SOCKET, NULL); node->attr.inode = ++currentinode; + return true; } void sockfs_init() { - nodecache = slab_newcache(sizeof(socketnode_t), 0, ctor, ctor); + nodecache = slab_newcache(sizeof(socketnode_t), 0, ctor, NULL); __assert(nodecache); } diff --git a/kernel-src/include/kernel/slab.h b/kernel-src/include/kernel/slab.h index f8997a8a..6a0d7a09 100644 --- a/kernel-src/include/kernel/slab.h +++ b/kernel-src/include/kernel/slab.h @@ -4,31 +4,61 @@ #include #include #include +#include typedef struct slab_t { struct slab_t *next; struct slab_t *prev; - size_t used; + size_t reference_count; void **free; void *base; } slab_t; +typedef struct magazine { + struct magazine *next; + size_t round_count; + void *rounds[]; +} magazine_t; + +typedef struct { + magazine_t *loaded_magazine; + magazine_t *previous_magazine; + mutex_t mutex; +} cache_per_cpu_t; + +typedef struct slab_indirect { + struct slab_indirect *next; + slab_t *slab; + void *object; +} slab_indirect_t; + typedef struct scache_t { mutex_t mutex; - void (*ctor)(struct scache_t *cache, void *obj); + bool (*ctor)(struct scache_t *cache, void *obj); void (*dtor)(struct scache_t *cache, void *obj); - slab_t *full; - slab_t *partial; - slab_t *empty; + slab_t *list; size_t size; - size_t truesize; + size_t true_size; size_t alignment; - size_t slabobjcount; + size_t slab_object_count; + slab_indirect_t **indirect_table; + + // magazine data + size_t magazine_size; + size_t magazine_minimum_size; + size_t magazine_maximum_size; + + mutex_t depot_mutex; + int contention_count; + magazine_t *empty_depot; + magazine_t *full_depot; + + cache_per_cpu_t per_cpu[]; } scache_t; void *slab_allocate(scache_t *cache); void slab_free(scache_t *cache, void *addr); -scache_t *slab_newcache(size_t size, size_t alignment, void (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *)); +scache_t *slab_newcache(size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *)); void slab_freecache(scache_t *cache); #endif diff --git a/kernel-src/include/x86-64/arch/cpu.h b/kernel-src/include/x86-64/arch/cpu.h index 5ba07c43..78661fd1 100644 --- a/kernel-src/include/x86-64/arch/cpu.h +++ b/kernel-src/include/x86-64/arch/cpu.h @@ -25,7 +25,7 @@ typedef struct cpu_t { vmmcontext_t *vmmctx; // expected to be here by other code long hardware_id; // expected to be here by other code - long internal_id; + long internal_id; // expected to be here by other code timekeeper_source_t *timekeeper_source; timekeeper_source_info_t *timekeeper_source_info; @@ -125,6 +125,12 @@ static inline void set_current_vmm_context(vmmcontext_t *context) { asm volatile ("mov %%rax, %%gs:16" : : "a"(context) : "memory"); } +static inline long current_cpu_internal_id(void) { + long id; + asm volatile ("mov %%gs:32, %%rax" : "=a"(id) : : "memory"); + return id; +} + static inline long current_cpu_id(void) { long id; asm volatile ("mov %%gs:24, %%rax" : "=a"(id) : : "memory"); diff --git a/kernel-src/mm/alloc.c b/kernel-src/mm/alloc.c index c12be0b7..e213c726 100644 --- a/kernel-src/mm/alloc.c +++ b/kernel-src/mm/alloc.c @@ -98,4 +98,4 @@ void alloc_init() { caches[i] = caches[5]; } -INIT_ROUTINE_DEFINE(alloc, INIT_ROUTINE_FLAGS_NONE, alloc_init, vmm); +INIT_ROUTINE_DEFINE(alloc, INIT_ROUTINE_FLAGS_NONE, alloc_init, slab); diff --git a/kernel-src/mm/slab.c b/kernel-src/mm/slab.c index e5265042..7d17f5c2 100644 --- a/kernel-src/mm/slab.c +++ b/kernel-src/mm/slab.c @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include #define SLAB_INDIRECT_CUTOFF 512 #define SLAB_INDIRECT_COUNT 16 @@ -15,76 +18,111 @@ #define SLAB_DEBUG 0 -// the cache responsible for allocating all others -static bool selfcacheinit = false; -static scache_t selfcache = { - .size = sizeof(scache_t), - .alignment = 8, - .truesize = ROUND_UP(sizeof(scache_t) + sizeof(void **), 8), - .slabobjcount = SLAB_DATA_SIZE / ROUND_UP(sizeof(scache_t) + sizeof(void **), 8) -}; +static slab_indirect_t *self_cache_indirect_table[32]; +static scache_t *self_cache; +static scache_t *magazine_cache; +static scache_t *slab_cache; +static scache_t *indirect_cache; -static void initdirect(scache_t *cache, slab_t *slab, void *base) { - slab->free = NULL; - slab->used = 0; - - for (uintmax_t offset = 0; offset < cache->truesize * cache->slabobjcount; offset += cache->truesize) { - if (cache->ctor) - cache->ctor(cache, (void *)((uintptr_t)base + offset)); - - void **freenext = (void **)((uintptr_t)base + offset + cache->size); - *freenext = slab->free; - slab->free = freenext; - } -} - -static void initindirect(scache_t *cache, slab_t *slab, void **base, void *objbase) { - slab->free = NULL; - slab->used = 0; - slab->base = objbase; - - for (uintmax_t offset = 0; offset < cache->truesize * cache->slabobjcount; offset += cache->truesize) { - if (cache->ctor) - cache->ctor(cache, (void *)((uintptr_t)objbase + offset)); - - void **freenext = &base[offset / cache->truesize]; - *freenext = slab->free; - slab->free = freenext; - } -} - -static bool growcache(scache_t *cache) { - void *_slab = pmm_allocpage(PMM_SECTION_DEFAULT); - if (_slab == NULL) - return false; - - slab_t *slab = MAKE_HHDM(GET_SLAB(_slab)); +static inline bool grow_cache(scache_t *cache) { + slab_t *slab; if (cache->size < SLAB_INDIRECT_CUTOFF) { - initdirect(cache, slab, MAKE_HHDM(_slab)); + void *slab_hhdm = pmm_allocpage(PMM_SECTION_DEFAULT); + if (unlikely(slab_hhdm == NULL)) + return false; + + slab_hhdm = MAKE_HHDM(slab_hhdm); + slab = GET_SLAB(slab_hhdm); + slab->free = NULL; + + for (uintmax_t offset = 0; offset < cache->true_size * cache->slab_object_count; offset += cache->true_size) { + void **freenext = (void **)((uintptr_t)slab_hhdm + offset + cache->size); + *freenext = slab->free; + slab->free = freenext; + } } else { - void *base = vmm_map(NULL, cache->slabobjcount * cache->truesize, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); - if (base == NULL) { - pmm_release(_slab); + slab = slab_allocate(slab_cache); + if (unlikely(slab == NULL)) + return false; + + size_t byte_size = cache->slab_object_count * cache->true_size; + slab->base = vmm_map(NULL, byte_size, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); + if (unlikely(slab->base == NULL)) { + slab_free(slab_cache, slab); return false; } - initindirect(cache, slab, MAKE_HHDM(_slab), base); + + slab_indirect_t *indirect[SLAB_INDIRECT_COUNT]; + int i; + for (i = 0; i < SLAB_INDIRECT_COUNT; ++i) { + indirect[i] = slab_allocate(indirect_cache); + if (indirect[i] == NULL) + break; + + indirect[i]->next = i == 0 ? NULL : indirect[i - 1]; + indirect[i]->slab = slab; + } + + if (unlikely(i < SLAB_INDIRECT_COUNT)) { + for (int j = 0; j < i; ++j) + slab_free(indirect_cache, indirect[i]); + + vmm_unmap(slab->base, byte_size, 0); + + slab_free(slab_cache, slab); + return false; + } + + i = 0; + for (uintmax_t offset = 0; offset < byte_size; offset += cache->true_size) { + void *object = (void *)((uintptr_t)slab->base + offset); + indirect[i++]->object = object; + } + + slab->free = (void **)indirect[SLAB_INDIRECT_COUNT - 1]; } + slab->reference_count = 0; + + // it can be inserted directly into cache->list because if this is called there is nothing to allocate + slab_t *list = cache->list; + if (likely(cache->list)) { + slab->next = list; + slab->prev = list->prev; + list->prev->next = slab; + list->prev = slab; + } else { + slab->next = slab; + slab->prev = slab; + } + + cache->list = slab; - slab->next = cache->empty; - slab->prev = NULL; - if (slab->next) - slab->next->prev = slab; - cache->empty = slab; return true; } -static void *takeobject(scache_t *cache, slab_t *slab) { - if (slab->free == NULL) +static inline int get_index(void *p) { + uintptr_t key = (uintptr_t)p; + return fnv1ahash(&key, sizeof(key)) % 32; +} + +static void *allocate_from_slab(scache_t *cache) { + MUTEX_ACQUIRE(&cache->mutex); + // if there is no slab or only an empty one, grow the cache + if ((cache->list == NULL || cache->list->reference_count == cache->slab_object_count) && unlikely(grow_cache(cache) == false)) return NULL; - void **objend = slab->free; + slab_t *slab = cache->list; + +#if SLAB_DEBUG != 0 + __assert(slab != NULL && slab->reference_count != cache->slab_object_count); +#endif + + // take an object from the cache + void **free_list = slab->free; + slab->free = *free_list; + ++slab->reference_count; #if SLAB_DEBUG != 0 if (cache->size < SLAB_INDIRECT_CUTOFF) { @@ -92,200 +130,348 @@ static void *takeobject(scache_t *cache, slab_t *slab) { void *base = (void *)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE); __assert(addr >= base && addr < (void *)slab); } else { - void *addr = slab->base + ((uintptr_t)objend - ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)) / sizeof(void **) * cache->truesize; - __assert(addr >= slab->base && (uintptr_t)addr < (uintptr_t)slab->base + cache->slabobjcount * cache->truesize); + void *addr = slab->base + ((uintptr_t)objend - ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)) / sizeof(void **) * cache->true_size; + __assert(addr >= slab->base && (uintptr_t)addr < (uintptr_t)slab->base + cache->slab_object_count * cache->true_size); } #endif - slab->free = *slab->free; - slab->used += 1; - *objend = NULL; - if (cache->size < SLAB_INDIRECT_CUTOFF) - return (void *)((uintptr_t)objend - cache->size); - else - return (void *)((uintptr_t)slab->base + ((uintptr_t)objend - ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)) / sizeof(void **) * cache->truesize); -} - -// frees an object and returns the slab the object belongs to -static slab_t *returnobject(scache_t *cache, void *obj) { - slab_t *slab = NULL; - void **freeptr = NULL; + void *ret; if (cache->size < SLAB_INDIRECT_CUTOFF) { - slab = (slab_t *)(ROUND_DOWN((uintptr_t)obj, PAGE_SIZE) + SLAB_PAGE_OFFSET); - freeptr = (void **)((uintptr_t)obj + cache->size); - __assert(*freeptr == NULL); - } else { - bool partial = cache->full == NULL; - slab = partial ? cache->partial : cache->full; - while (slab) { - void *top = (void *)((uintptr_t)slab->base + cache->slabobjcount * cache->truesize); - if (obj >= slab->base && obj < top) - break; - if (slab->next == NULL && partial == false) { - slab = cache->partial; - partial = true; - } else { - slab = slab->next; - } - } - __assert(slab); - uintmax_t objn = ((uintptr_t)obj - (uintptr_t)slab->base) / cache->truesize; - void **base = (void **)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE); - freeptr = &base[objn]; - } - - if (cache->dtor) - cache->dtor(cache, obj); - - *freeptr = slab->free; - slab->free = freeptr; - --slab->used; - - return slab; -} - -void *slab_allocate(scache_t *cache) { - MUTEX_ACQUIRE(&cache->mutex); - slab_t *slab = cache->partial; - if (slab == NULL) - slab = cache->empty; - - if (slab == NULL) { - if (growcache(cache)) { - slab = cache->empty; - } else { - MUTEX_RELEASE(&cache->mutex); + ret = (void *)((uintptr_t)free_list - cache->size); + if (cache->ctor && cache->ctor(cache, ret) == false) { + slab->free = free_list; return NULL; } + +#if SLAB_DEBUG != 0 + *free_list = NULL; +#endif + } else { + slab_indirect_t *indirect = (slab_indirect_t *)free_list; + if (cache->ctor && cache->ctor(cache, indirect->object) == false) { + slab->free = free_list; + return NULL; + } + // add to hash table as well + int idx = get_index(indirect->object); + indirect->next = cache->indirect_table[idx]; + cache->indirect_table[idx] = indirect; + + ret = indirect->object; } - void *ret = takeobject(cache, slab); - - if (slab == cache->empty) { - cache->empty = slab->next; - if (slab->next) - slab->next->prev = NULL; - - if (cache->partial) - cache->partial->prev = slab; - - slab->next = cache->partial; - slab->prev = NULL; - cache->partial = slab; - } else if (slab->used == cache->slabobjcount) { - cache->partial = slab->next; - - if (slab->next) - slab->next->prev = NULL; - - if (cache->full) - cache->full->prev = slab; - - slab->next = cache->full; - slab->prev = NULL; - cache->full = slab; - } + // if the slab is empty, advance the linked list by one to the next partial or full slab + if (slab->reference_count == cache->slab_object_count) + cache->list = slab->next; MUTEX_RELEASE(&cache->mutex); return ret; } -void slab_free(scache_t *cache, void *addr) { +static void free_to_slab(scache_t *cache, void *addr) { MUTEX_ACQUIRE(&cache->mutex); - slab_t *slab = returnobject(cache, addr); - __assert(slab); + // insert into a slab + slab_t *slab = NULL; + void **free_list = NULL; - if (slab->used == 0) { - if (slab->prev == NULL) - cache->partial = slab->next; + if (cache->size < SLAB_INDIRECT_CUTOFF) { + slab = GET_SLAB(addr); + free_list = (void **)((uintptr_t)addr + cache->size); + +#if SLAB_DEBUG != 0 + __assert(*free_list == NULL); +#endif + } else { + int idx = get_index(addr); + slab_indirect_t *prev = NULL; + slab_indirect_t *indirect = cache->indirect_table[idx]; + + while (indirect->object != addr) { + prev = indirect; + indirect = indirect->next; + } + + if (prev) + prev->next = indirect->next; else - slab->prev->next = slab->next; + cache->indirect_table[idx] = indirect->next; - if (slab->next) - slab->next->prev = slab->prev; + slab = indirect->slab; + free_list = (void **)indirect; - slab->next = cache->empty; - slab->prev = NULL; - if (slab->next) - slab->next->prev = slab; - cache->empty = slab; - } else if (slab->used == cache->slabobjcount - 1) { - if (slab->prev == NULL) - cache->full = slab->next; - else - slab->prev->next = slab->next; - - if (slab->next) - slab->next->prev = slab->prev; - - slab->next = cache->partial; - slab->prev = NULL; - if (slab->next) - slab->next->prev = slab; - cache->partial = slab; +#if SLAB_DEBUG != 0 + __assert(slab); + __assert(obj >= slab->base && addr < top); +#endif } + + if (cache->dtor) + cache->dtor(cache, addr); + + *free_list = slab->free; + slab->free = free_list; + --slab->reference_count; + + // reorder lists if nescessary + if (slab->reference_count == cache->slab_object_count - 1) { + // if it became a partial slab, reorder the list + slab->prev->next = slab->next; + slab->next->prev = slab->prev; + + slab->next = cache->list; + slab->prev = cache->list->prev; + slab->prev->next = slab; + slab->next->prev = slab; + + cache->list = slab; + } else if (slab->reference_count == 0) { + // if it became a full slab, reorder the list + slab_t *search = cache->list; + do { + search = search->next; + } while (search != cache->list && search->reference_count != cache->size && (search->reference_count != 0 || search == slab)); + + // if the list points to this slab and the next one is a partial, set the list to it + // otherwise, the next is full (refcount 0) or empty (refcount == slab_object_count), which means this slab will be still be the next one to + // be allocated out of + if (cache->list == slab && slab->next->reference_count != 0 && slab->next->reference_count != cache->slab_object_count) + cache->list = slab->next; + + slab->prev->next = slab->next; + slab->next->prev = slab->prev; + + slab->next = search; + slab->prev = search->prev; + search->prev = slab; + slab->prev->next = slab; + } + MUTEX_RELEASE(&cache->mutex); } +static inline void acquire_depot_mutex(scache_t *cache) { + // keep track of contention for dynamic resizing of magazines in this cache + if (MUTEX_TRY(&cache->depot_mutex)) { + if (__atomic_fetch_sub(&cache->contention_count, 1, __ATOMIC_SEQ_CST) == 0) + cache->contention_count = 0; -scache_t *slab_newcache(size_t size, size_t alignment, void (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *)) { + } else { + __atomic_add_fetch(&cache->contention_count, 1, __ATOMIC_SEQ_CST); + MUTEX_ACQUIRE(&cache->depot_mutex); + } +} + +void *slab_allocate(scache_t *cache) { + cache_per_cpu_t *cpu_cache = &cache->per_cpu[current_cpu_internal_id()]; + + MUTEX_ACQUIRE(&cpu_cache->mutex); + // are there any rounds in the loaded magazine? + if (cpu_cache->loaded_magazine && cpu_cache->loaded_magazine->round_count) { + void *ret = cpu_cache->loaded_magazine->rounds[--cpu_cache->loaded_magazine->round_count]; + MUTEX_RELEASE(&cpu_cache->mutex); + return ret; + } + + // are there any rounds in the previous magazine? + if (cpu_cache->previous_magazine && cpu_cache->previous_magazine->round_count) { + // swap loaded and previous + magazine_t *swap = cpu_cache->loaded_magazine; + cpu_cache->loaded_magazine = cpu_cache->previous_magazine; + cpu_cache->previous_magazine = swap; + + void *ret = cpu_cache->loaded_magazine->rounds[--cpu_cache->loaded_magazine->round_count]; + MUTEX_RELEASE(&cpu_cache->mutex); + return ret; + } + + // check the depot for full magazines + acquire_depot_mutex(cache); + + if (cache->full_depot) { + // put the previous magazine into the depot + if (cpu_cache->previous_magazine) { + cpu_cache->previous_magazine->next = cache->empty_depot; + cache->empty_depot = cpu_cache->previous_magazine; + } + + // load a new full magazine + cpu_cache->previous_magazine = cpu_cache->loaded_magazine; + cpu_cache->loaded_magazine = cache->full_depot; + cache->full_depot = cache->full_depot->next; + + void *ret = cpu_cache->loaded_magazine->rounds[--cpu_cache->loaded_magazine->round_count]; + MUTEX_RELEASE(&cache->depot_mutex); + MUTEX_RELEASE(&cpu_cache->mutex); + return ret; + } + + MUTEX_RELEASE(&cache->depot_mutex); + + // fall back into allocating from the slab directly + MUTEX_RELEASE(&cpu_cache->mutex); + return allocate_from_slab(cache); +} + +static magazine_t *allocate_magazine_for_cache(scache_t *cache) { + return slab_allocate(magazine_cache); +} + +void slab_free(scache_t *cache, void *addr) { + cache_per_cpu_t *cpu_cache = &cache->per_cpu[current_cpu_internal_id()]; + + MUTEX_ACQUIRE(&cpu_cache->mutex); + // can insert into the loaded magazine? + if (cpu_cache->loaded_magazine && cpu_cache->loaded_magazine->round_count < cache->magazine_size) { + cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr; + MUTEX_RELEASE(&cpu_cache->mutex); + return; + } + + // can insert into previous magazine? + // the check for 0 is valid as the previous magazines will either be full or empty + if (cpu_cache->previous_magazine && cpu_cache->previous_magazine->round_count == 0) { + // swap loaded and previous + magazine_t *swap = cpu_cache->loaded_magazine; + cpu_cache->loaded_magazine = cpu_cache->previous_magazine; + cpu_cache->previous_magazine = swap; + + cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr; + MUTEX_RELEASE(&cpu_cache->mutex); + return; + } + + // can get an empty magazine from the depot? + acquire_depot_mutex(cache); + if (cache->empty_depot) { + // put the previous magazine into the depot + if (cpu_cache->previous_magazine) { + cpu_cache->previous_magazine->next = cache->full_depot; + cache->full_depot = cpu_cache->previous_magazine; + } + + // load a new empty magazine + cpu_cache->previous_magazine = cpu_cache->loaded_magazine; + cpu_cache->loaded_magazine = cache->empty_depot; + cache->empty_depot = cache->empty_depot->next; + + cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr; + MUTEX_RELEASE(&cache->depot_mutex); + MUTEX_RELEASE(&cpu_cache->mutex); + return; + } + + // allocate an empty magazine + magazine_t *new_magazine = allocate_magazine_for_cache(cache); + if (new_magazine) { + // put previous into the depot + if (cpu_cache->previous_magazine) { + cpu_cache->previous_magazine->next = cache->full_depot; + cache->full_depot = cpu_cache->previous_magazine; + } + + // load the new magazine + cpu_cache->previous_magazine = cpu_cache->loaded_magazine; + cpu_cache->loaded_magazine = new_magazine; + + cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr; + MUTEX_RELEASE(&cache->depot_mutex); + MUTEX_RELEASE(&cpu_cache->mutex); + return; + } + MUTEX_RELEASE(&cache->depot_mutex); + + // fall back into freeing to the slab directly + MUTEX_RELEASE(&cpu_cache->mutex); + return free_to_slab(cache, addr); +} + +#define MAGAZINE_SIZE 8 + +static bool slab_initialize(scache_t *cache, size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *), slab_indirect_t **indirect_table) { if (alignment == 0) alignment = 8; - if (selfcacheinit == false) { - selfcacheinit = true; - MUTEX_INIT(&selfcache.mutex); + if (size >= SLAB_INDIRECT_CUTOFF && indirect_table == NULL) { + cache->indirect_table = alloc(sizeof(slab_indirect_t *) * 32); + if (cache->indirect_table == NULL) + return false; + } else if (size >= SLAB_INDIRECT_CUTOFF && indirect_table) { + cache->indirect_table = indirect_table; } - scache_t *cache = slab_allocate(&selfcache); - if (cache == NULL) - return NULL; - cache->size = size; cache->alignment = alignment; size_t freeptrsize = size < SLAB_INDIRECT_CUTOFF ? sizeof(void **) : 0; - cache->truesize = ROUND_UP(size + freeptrsize, alignment); + cache->true_size = ROUND_UP(size + freeptrsize, alignment); cache->ctor = ctor; cache->dtor = dtor; - cache->slabobjcount = size < SLAB_INDIRECT_CUTOFF ? SLAB_DATA_SIZE / cache->truesize : SLAB_INDIRECT_COUNT; - cache->full = NULL; - cache->empty = NULL; - cache->partial = NULL; - MUTEX_INIT(&cache->mutex); + cache->slab_object_count = size < SLAB_INDIRECT_CUTOFF ? SLAB_DATA_SIZE / cache->true_size : SLAB_INDIRECT_COUNT; + cache->list = NULL; + cache->magazine_size = MAGAZINE_SIZE; + cache->magazine_minimum_size = MAGAZINE_SIZE; + cache->magazine_maximum_size = MAGAZINE_SIZE; + cache->contention_count = 0; + cache->empty_depot = NULL; + cache->full_depot = NULL; - printf("slab: new cache: size %lu align %lu truesize %lu objcount %lu\n", cache->size, cache->alignment, cache->truesize, cache->slabobjcount); + MUTEX_INIT(&cache->mutex); + MUTEX_INIT(&cache->depot_mutex); + + for (int i = 0; i < arch_smp_get_cpu_count(); ++i) { + cache_per_cpu_t *cpu_cache = &cache->per_cpu[i]; + cpu_cache->loaded_magazine = NULL; + cpu_cache->previous_magazine = NULL; + MUTEX_INIT(&cpu_cache->mutex); + } + + return true; +} + +scache_t *slab_newcache(size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *)) { + scache_t *cache = slab_allocate(self_cache); + if (cache == NULL) + return NULL; + + if (!slab_initialize(cache, size, alignment, ctor, dtor, NULL)) { + slab_free(self_cache, cache); + return NULL; + } + + printf("slab: new cache: size %lu align %lu true_size %lu objcount %lu\n", cache->size, cache->alignment, cache->true_size, cache->slab_object_count); return cache; } -static size_t purge(scache_t *cache, size_t maxcount){ - slab_t *slab = cache->empty; - for (size_t done = 0; done < maxcount; ++done) { - if (slab == NULL) - return done; - slab_t *next = slab->next; - if (next) - next->prev = NULL; +static bool magazine_ctor(scache_t *, void *obj) { + magazine_t *mag = obj; + mag->next = NULL; + mag->round_count = 0; - if (cache->size >= SLAB_INDIRECT_CUTOFF) - vmm_unmap(slab->base, cache->slabobjcount * cache->truesize, 0); +#if SLAB_DEBUG != 0 + memset(mag->rounds, 0xfe, MAGAZINE_SIZE * sizeof(void *)); +#endif - pmm_release(FROM_HHDM((void *)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE))); - - slab = cache->empty; - cache->empty = next; - } - - return maxcount; + return true; } -void slab_freecache(scache_t *cache) { - MUTEX_ACQUIRE(&cache->mutex); - __assert(cache->partial == NULL); - __assert(cache->full == NULL); - - purge(cache, (size_t)-1); - - slab_free(&selfcache, cache); +static scache_t *create_new_from_vmm(size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *), slab_indirect_t **table) { + size_t cache_size = sizeof(scache_t) + sizeof(cache_per_cpu_t) * arch_smp_get_cpu_count(); + scache_t *cache = vmm_map(NULL, cache_size, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); + __assert(cache); + __assert(slab_initialize(cache, size, alignment, ctor, dtor, table)); + return cache; } + +// TODO dynamically change the size of magazines +void slab_init(void) { + size_t cache_size = sizeof(scache_t) + sizeof(cache_per_cpu_t) * arch_smp_get_cpu_count(); + // the cache of caches needs for the slab and indirect caches to be up + slab_cache = create_new_from_vmm(sizeof(slab_t), 0, NULL, NULL, self_cache_indirect_table); + indirect_cache = create_new_from_vmm(sizeof(slab_indirect_t), 0, NULL, NULL, NULL); + self_cache = create_new_from_vmm(cache_size, 0, NULL, NULL, NULL); + + magazine_cache = slab_newcache(sizeof(magazine_t) + sizeof(void *) * MAGAZINE_SIZE, 0, magazine_ctor, NULL); +} + +INIT_ROUTINE_DEFINE(slab, INIT_ROUTINE_FLAGS_NONE, slab_init, vmm); diff --git a/kernel-src/mm/vmm.c b/kernel-src/mm/vmm.c index b0a9ec6c..c09e785b 100644 --- a/kernel-src/mm/vmm.c +++ b/kernel-src/mm/vmm.c @@ -821,17 +821,18 @@ void vmm_unmap(void *addr, size_t size, int flags) { static scache_t *ctxcache; -static void ctxctor(scache_t *cache, void *obj) { +static bool ctxctor(scache_t *cache, void *obj) { vmmcontext_t *ctx = obj; ctx->space.start = USERSPACE_START; ctx->space.end = USERSPACE_END; MUTEX_INIT(&ctx->space.lock); ctx->space.ranges = NULL; + return true; } vmmcontext_t *vmm_newcontext() { if (ctxcache == NULL) { - ctxcache = slab_newcache(sizeof(vmmcontext_t), 0, ctxctor, ctxctor); + ctxcache = slab_newcache(sizeof(vmmcontext_t), 0, ctxctor, NULL); __assert(ctxcache); } @@ -854,6 +855,8 @@ void vmm_destroycontext(vmmcontext_t *context) { vmm_unmap(context->space.start, context->space.end - context->space.start, 0); vmm_switchcontext(oldctx); arch_mmu_destroytable(context->pagetable); + + context->space.ranges = NULL; slab_free(ctxcache, context); } diff --git a/kernel-src/sys/syscalls/futex.c b/kernel-src/sys/syscalls/futex.c index 08ad3541..1d0dd495 100644 --- a/kernel-src/sys/syscalls/futex.c +++ b/kernel-src/sys/syscalls/futex.c @@ -6,7 +6,6 @@ #include #include -static scache_t *futexcache; static mutex_t futexmutex; static hashtable_t hashtable; static bool hashtableinit; @@ -36,29 +35,13 @@ static void removefutex(void *physical) { hashtable_remove(&hashtable, &physical, sizeof(physical)); } -static void ctor(scache_t *cache, void *obj) { - futex_t *futex = obj; - POLL_INITHEADER(&futex->pollheader); - futex->waiting = 0; - futex->waking = 0; -} - syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value, timespec_t *tm) { syscallret_t ret = { .ret = -1 }; - if (futexcache == NULL) { - futexcache = slab_newcache(sizeof(thread_t), 0, ctor, ctor); - - if (futexcache == NULL) { - ret.errno = ENOMEM; - return ret; - } - } - - if (hashtableinit == false) { - if (hashtable_init(&hashtable, 256)) { + if (unlikely(hashtableinit == false)) { + if (unlikely(hashtable_init(&hashtable, 256))) { ret.errno = ENOMEM; return ret; } @@ -70,7 +53,7 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value timespec_t timespec = tm ? *tm : (timespec_t){0}; polldesc_t desc = {0}; ret.errno = poll_initdesc(&desc, 1); - if (ret.errno) + if (unlikely(ret.errno)) return ret; MUTEX_ACQUIRE(&futexmutex); @@ -78,7 +61,7 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value bool doleave = true; uint32_t word; ret.errno = usercopy_fromuseratomic32(futexp, &word); - if (ret.errno) + if (unlikely(ret.errno)) goto cleanup; uint32_t *physical = vmm_getphysical(futexp, false);