slab: rewrite slab allocator

kernel-src/arch/x86-64/smp.c

@@ -58,8 +58,8 @@ void arch_smp_haltallothers(void) {
 }
 
 size_t arch_smp_get_cpu_count(void) {
-	if (cmdline_get("nosmp"))
-		return 1;
+	//if (cmdline_get("nosmp"))
+	//	return 1;
 
 	struct limine_smp_response *response = smprequest.response;
 	if (response == NULL)
@@ -95,7 +95,7 @@ void arch_smp_wakeup(void) {
 	smp_cpus = alloc(response->cpu_count * sizeof(cpu_t *));
 	__assert(smp_cpus);
 
-	void (*wakeupfn)(struct limine_smp_info *) = cmdline_get("nosmp") ? cpuwakeuphalt : cpuwakeup;
+	void (*wakeupfn)(struct limine_smp_info *) = cpuwakeup;//cmdline_get("nosmp") ? cpuwakeuphalt : cpuwakeup;
 
 	// make the other processors jump to cpuwakeup()
 	for (int i = 0; i < response->cpu_count; ++i) {

kernel-src/fs/devfs.c

@@ -254,6 +254,13 @@ int devfs_maxseek(vnode_t *node, size_t *max) {
 	return devnode->devops->maxseek ? devnode->devops->maxseek(devnode->attr.rdevminor, max) : ENOTTY;
 }
 
+static void destroy_node(devnode_t *node) {
+	memset(node, 0, sizeof(devnode_t));
+	VOP_INIT(&node->vnode, &vnops, 0, 0, NULL)
+	node->attr.inode = ++currentinode;
+	slab_free(nodecache, node);
+}
+
 int devfs_inactive(vnode_t *node) {
 	INTERNAL_LOCK(node);
 	devnode_t *devnode = (devnode_t *)node;
@@ -267,7 +274,7 @@ int devfs_inactive(vnode_t *node) {
 	if (devnode->devops && devnode->devops->inactive)
 		devnode->devops->inactive(devnode->attr.rdevminor);
 
-	slab_free(nodecache, node);
+	destroy_node(devnode);
 	return 0;
 }
 
@@ -311,13 +318,13 @@ int devfs_create(vnode_t *parent, char *name, vattr_t *attr, int type, vnode_t *
 	if (type == V_TYPE_DIR) {
 		int error = hashtable_init(&node->children, 100);
 		if (error) {
-			slab_free(nodecache, node);
+			destroy_node(node);
 			INTERNAL_UNLOCK(parent);
 			return error;
 		}
 		if (hashtable_set(&node->children, node, ".", 1, true) || hashtable_set(&node->children, parent, "..", 2, true)) {
 			hashtable_destroy(&node->children);
-			slab_free(nodecache, node);
+			destroy_node(node);
 			INTERNAL_UNLOCK(parent);
 			return error;
 		}
@@ -328,7 +335,7 @@ int devfs_create(vnode_t *parent, char *name, vattr_t *attr, int type, vnode_t *
 		if (type == V_TYPE_DIR)
 			hashtable_destroy(&node->children);
 		INTERNAL_UNLOCK(parent);
-		slab_free(nodecache, node);
+		destroy_node(node);
 		return error;
 	}
 
@@ -476,16 +483,17 @@ static vops_t vnops = {
 	.unlock = devfs_unlock
 };
 
-static void ctor(scache_t *cache, void *obj) {
+static bool ctor(scache_t *cache, void *obj) {
 	devnode_t *node = obj;
 	memset(node, 0, sizeof(devnode_t));
 	VOP_INIT(&node->vnode, &vnops, 0, 0, NULL)
 	node->attr.inode = ++currentinode;
+	return true;
 }
 
 void devfs_init() {
 	__assert(hashtable_init(&devtable, 50) == 0);
-	nodecache = slab_newcache(sizeof(devnode_t), 0, ctor, ctor);
+	nodecache = slab_newcache(sizeof(devnode_t), 0, ctor, NULL);
 	MUTEX_INIT(&tablelock);
 	__assert(nodecache);
 
@@ -525,7 +533,7 @@ int devfs_register(devops_t *devops, char *name, int type, int major, int minor,
 	int error = hashtable_set(&devtable, master, key, sizeof(key), true);
 	MUTEX_RELEASE(&tablelock);
 	if (error) {
-		slab_free(nodecache, master);
+		destroy_node(master);
 		return error;
 	}
 
@@ -540,7 +548,7 @@ int devfs_register(devops_t *devops, char *name, int type, int major, int minor,
 	char lastcomp[strlen(name) + 1];
 	error = vfs_lookup(&dir, (vnode_t *)devfsroot, name, lastcomp, VFS_LOOKUP_PARENT);
 	if (error) {
-		slab_free(nodecache, master);
+		destroy_node(master);
 		return error;
 	}
 
@@ -556,7 +564,7 @@ int devfs_register(devops_t *devops, char *name, int type, int major, int minor,
 		MUTEX_ACQUIRE(&tablelock);
 		__assert(hashtable_remove(&devtable, key, sizeof(key) == 0));
 		MUTEX_RELEASE(&tablelock);
-		slab_free(nodecache, master);
+		destroy_node(master);
 		return error;
 	}
 

kernel-src/fs/file.c

@@ -13,19 +13,20 @@ static scache_t *filecache;
 			cleanfile(f); \
 		}
 
-static void ctor(scache_t *cache, void *obj) {
+static bool ctor(scache_t *cache, void *obj) {
 	file_t *file = obj;
 	file->vnode = NULL;
 	MUTEX_INIT(&file->mutex);
-	file->refcount = 1;
 	file->offset = 0;
 	file->flags = 0;
 	file->mode = 0;
+	file->refcount = 1;
+	return true;
 }
 
 static file_t* newfile() {
 	if (filecache == NULL) {
-		filecache = slab_newcache(sizeof(file_t), 0, ctor, ctor);
+		filecache = slab_newcache(sizeof(file_t), 0, ctor, NULL);
 		__assert(filecache);
 	}
 
@@ -37,6 +38,13 @@ static void cleanfile(file_t *file) {
 		vfs_close(file->vnode, fileflagstovnodeflags(file->flags));
 		VOP_RELEASE(file->vnode);
 	}
+
+	file->vnode = NULL;
+	file->refcount = 1;
+	file->offset = 0;
+	file->flags = 0;
+	file->mode = 0;
+	MUTEX_INIT(&file->mutex);
 	slab_free(filecache, file);
 }
 

kernel-src/fs/pipefs.c

@@ -342,6 +342,13 @@ int pipefs_inactive(vnode_t *node) {
 	pipenode_t *pipenode = (pipenode_t *)node;
 	INTERNAL_LOCK(node);
 	ringbuffer_destroy(&pipenode->data);
+	// TODO write this properly using cache ctor and dtor
+	memset(node, 0, sizeof(pipenode_t));
+	VOP_INIT(node, &vnops, 0, V_TYPE_FIFO, NULL);
+	pipenode->attr.inode = ++currentinode;
+	POLL_INITHEADER(&pipenode->pollheader);
+	EVENT_INITHEADER(&pipenode->writeopenevent);
+	EVENT_INITHEADER(&pipenode->readopenevent);
 	slab_free(nodecache, node);
 	return 0;
 }
@@ -389,7 +396,7 @@ static vops_t vnops = {
 	.unlock = pipefs_unlock
 };
 
-static void ctor(scache_t *cache, void *obj) {
+static bool ctor(scache_t *cache, void *obj) {
 	pipenode_t *node = obj;
 	memset(node, 0, sizeof(pipenode_t));
 	VOP_INIT(&node->vnode, &vnops, 0, V_TYPE_FIFO, NULL);
@@ -397,10 +404,11 @@ static void ctor(scache_t *cache, void *obj) {
 	POLL_INITHEADER(&node->pollheader);
 	EVENT_INITHEADER(&node->writeopenevent);
 	EVENT_INITHEADER(&node->readopenevent);
+	return true;
 }
 
 void pipefs_init() {
-	nodecache = slab_newcache(sizeof(pipenode_t), 0, ctor, ctor);
+	nodecache = slab_newcache(sizeof(pipenode_t), 0, ctor, NULL);
 	__assert(nodecache);
 }
 

kernel-src/fs/sockfs.c

@@ -204,10 +204,16 @@ int sockfs_ioctl(vnode_t *node, unsigned long request, void *arg, int *result, c
 	return 0;
 }
 
+static vops_t vnops;
+
 int sockfs_inactive(vnode_t *node) {
 	socket_t *socket = SOCKFS_SOCKET_FROM_NODE(node);
+	socketnode_t *socketnode = (socketnode_t *)node;
 	INTERNAL_LOCK(node);
 	socket->ops->destroy(socket);
+	memset(node, 0, sizeof(socketnode_t));
+	VOP_INIT(&socketnode->vnode, &vnops, 0, V_TYPE_SOCKET, NULL);
+	//socketnode->attr.inode = ++currentinode;
 	slab_free(nodecache, node);
 	return 0;
 }
@@ -256,15 +262,16 @@ static vops_t vnops = {
 	.unlock = sockfs_unlock
 };
 
-static void ctor(scache_t *cache, void *obj) {
+static bool ctor(scache_t *cache, void *obj) {
 	socketnode_t *node = obj;
 	memset(node, 0, sizeof(socketnode_t));
 	VOP_INIT(&node->vnode, &vnops, 0, V_TYPE_SOCKET, NULL);
 	node->attr.inode = ++currentinode;
+	return true;
 }
 
 void sockfs_init() {
-	nodecache = slab_newcache(sizeof(socketnode_t), 0, ctor, ctor);
+	nodecache = slab_newcache(sizeof(socketnode_t), 0, ctor, NULL);
 	__assert(nodecache);
 }
 

kernel-src/include/kernel/slab.h

@@ -4,31 +4,61 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <mutex.h>
+#include <hashtable.h>
 
 typedef struct slab_t {
 	struct slab_t *next;
 	struct slab_t *prev;
-	size_t used;
+	size_t reference_count;
 	void **free;
 	void *base;
 } slab_t;
 
+typedef struct magazine {
+	struct magazine *next;
+	size_t round_count;
+	void *rounds[];
+} magazine_t;
+
+typedef struct {
+	magazine_t *loaded_magazine;
+	magazine_t *previous_magazine;
+	mutex_t mutex;
+} cache_per_cpu_t;
+
+typedef struct slab_indirect {
+	struct slab_indirect *next;
+	slab_t *slab;
+	void *object;
+} slab_indirect_t;
+
 typedef struct scache_t {
 	mutex_t mutex;
-	void (*ctor)(struct scache_t *cache, void *obj);
+	bool (*ctor)(struct scache_t *cache, void *obj);
 	void (*dtor)(struct scache_t *cache, void *obj);
-	slab_t *full;
-	slab_t *partial;
-	slab_t *empty;
+	slab_t *list;
 	size_t size;
-	size_t truesize;
+	size_t true_size;
 	size_t alignment;
-	size_t slabobjcount;
+	size_t slab_object_count;
+	slab_indirect_t **indirect_table;
+
+	// magazine data
+	size_t magazine_size;
+	size_t magazine_minimum_size;
+	size_t magazine_maximum_size;
+
+	mutex_t depot_mutex;
+	int contention_count;
+	magazine_t *empty_depot;
+	magazine_t *full_depot;
+
+	cache_per_cpu_t per_cpu[];
 } scache_t;
 
 void *slab_allocate(scache_t *cache);
 void slab_free(scache_t *cache, void *addr);
-scache_t *slab_newcache(size_t size, size_t alignment, void (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *));
+scache_t *slab_newcache(size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *));
 void slab_freecache(scache_t *cache);
 
 #endif

kernel-src/include/x86-64/arch/cpu.h

@@ -25,7 +25,7 @@ typedef struct cpu_t {
 	vmmcontext_t *vmmctx; // expected to be here by other code
 
 	long hardware_id; // expected to be here by other code
-	long internal_id;
+	long internal_id; // expected to be here by other code
 
 	timekeeper_source_t *timekeeper_source;
 	timekeeper_source_info_t *timekeeper_source_info;
@@ -125,6 +125,12 @@ static inline void set_current_vmm_context(vmmcontext_t *context) {
 	asm volatile ("mov %%rax, %%gs:16" : : "a"(context) : "memory");
 }
 
+static inline long current_cpu_internal_id(void) {
+	long id;
+	asm volatile ("mov %%gs:32, %%rax" : "=a"(id) : : "memory");
+	return id;
+}
+
 static inline long current_cpu_id(void) {
 	long id;
 	asm volatile ("mov %%gs:24, %%rax" : "=a"(id) : : "memory");

kernel-src/mm/alloc.c

@@ -98,4 +98,4 @@ void alloc_init() {
 		caches[i] = caches[5];
 }
 
-INIT_ROUTINE_DEFINE(alloc, INIT_ROUTINE_FLAGS_NONE, alloc_init, vmm);
+INIT_ROUTINE_DEFINE(alloc, INIT_ROUTINE_FLAGS_NONE, alloc_init, slab);

kernel-src/mm/slab.c

@@ -3,6 +3,9 @@
 #include <kernel/pmm.h>
 #include <logging.h>
 #include <util.h>
+#include <arch/smp.h>
+#include <kernel/init.h>
+#include <kernel/alloc.h>
 
 #define SLAB_INDIRECT_CUTOFF 512
 #define SLAB_INDIRECT_COUNT 16
@@ -15,76 +18,111 @@
 
 #define SLAB_DEBUG 0
 
-// the cache responsible for allocating all others
-static bool selfcacheinit = false;
-static scache_t selfcache = {
-	.size = sizeof(scache_t),
-	.alignment = 8,
-	.truesize = ROUND_UP(sizeof(scache_t) + sizeof(void **), 8),
-	.slabobjcount = SLAB_DATA_SIZE / ROUND_UP(sizeof(scache_t) + sizeof(void **), 8)
-};
+static slab_indirect_t *self_cache_indirect_table[32];
+static scache_t *self_cache;
+static scache_t *magazine_cache;
+static scache_t *slab_cache;
+static scache_t *indirect_cache;
 
-static void initdirect(scache_t *cache, slab_t *slab, void *base) {
-	slab->free = NULL;
-	slab->used = 0;
-
-	for (uintmax_t offset = 0; offset < cache->truesize * cache->slabobjcount; offset += cache->truesize) {
-		if (cache->ctor)
-			cache->ctor(cache, (void *)((uintptr_t)base + offset));
-
-		void **freenext = (void **)((uintptr_t)base + offset + cache->size);
-		*freenext = slab->free;
-		slab->free = freenext;
-	}
-}
-
-static void initindirect(scache_t *cache, slab_t *slab, void **base, void *objbase) {
-	slab->free = NULL;
-	slab->used = 0;
-	slab->base = objbase;
-
-	for (uintmax_t offset = 0; offset < cache->truesize * cache->slabobjcount; offset += cache->truesize) {
-		if (cache->ctor)
-			cache->ctor(cache, (void *)((uintptr_t)objbase + offset));
-
-		void **freenext = &base[offset / cache->truesize];
-		*freenext =  slab->free;
-		slab->free = freenext;
-	}
-}
-
-static bool growcache(scache_t *cache) {
-	void *_slab = pmm_allocpage(PMM_SECTION_DEFAULT);
-	if (_slab == NULL)
-		return false;
-
-	slab_t *slab = MAKE_HHDM(GET_SLAB(_slab));
+static inline bool grow_cache(scache_t *cache) {
+	slab_t *slab;
 
 	if (cache->size < SLAB_INDIRECT_CUTOFF) {
-		initdirect(cache, slab, MAKE_HHDM(_slab));
+		void *slab_hhdm = pmm_allocpage(PMM_SECTION_DEFAULT);
+		if (unlikely(slab_hhdm == NULL))
+			return false;
+
+		slab_hhdm = MAKE_HHDM(slab_hhdm);
+		slab = GET_SLAB(slab_hhdm);
+		slab->free = NULL;
+		
+		for (uintmax_t offset = 0; offset < cache->true_size * cache->slab_object_count; offset += cache->true_size) {
+			void **freenext = (void **)((uintptr_t)slab_hhdm + offset + cache->size);
+			*freenext = slab->free;
+			slab->free = freenext;
+		}
 	} else {
-		void *base = vmm_map(NULL, cache->slabobjcount * cache->truesize, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
-		if (base == NULL) {
-			pmm_release(_slab);
+		slab = slab_allocate(slab_cache);
+		if (unlikely(slab == NULL))
+			return false;
+
+		size_t byte_size = cache->slab_object_count * cache->true_size;
+		slab->base = vmm_map(NULL, byte_size, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
+		if (unlikely(slab->base == NULL)) {
+			slab_free(slab_cache, slab);
 			return false;
 		}
-		initindirect(cache, slab, MAKE_HHDM(_slab), base);
+
+		slab_indirect_t *indirect[SLAB_INDIRECT_COUNT];
+		int i;
+		for (i = 0; i < SLAB_INDIRECT_COUNT; ++i) {
+			indirect[i] = slab_allocate(indirect_cache);
+			if (indirect[i] == NULL)
+				break;
+
+			indirect[i]->next = i == 0 ? NULL : indirect[i - 1];
+			indirect[i]->slab = slab;
+		}
+
+		if (unlikely(i < SLAB_INDIRECT_COUNT)) {
+			for (int j = 0; j < i; ++j)
+				slab_free(indirect_cache, indirect[i]);
+			
+			vmm_unmap(slab->base, byte_size, 0);
+
+			slab_free(slab_cache, slab);
+			return false;
+		}
+
+		i = 0;
+		for (uintmax_t offset = 0; offset < byte_size; offset += cache->true_size) {
+			void *object = (void *)((uintptr_t)slab->base + offset);
+			indirect[i++]->object = object;
+		}
+
+		slab->free = (void **)indirect[SLAB_INDIRECT_COUNT - 1];
 	}
 
+	slab->reference_count = 0;
+
+	// it can be inserted directly into cache->list because if this is called there is nothing to allocate
+	slab_t *list = cache->list;
+	if (likely(cache->list)) {
+		slab->next = list;
+		slab->prev = list->prev;
+		list->prev->next = slab;
+		list->prev = slab;
+	} else {
+		slab->next = slab;
+		slab->prev = slab;
+	}
+
+	cache->list = slab;
 
-	slab->next = cache->empty;
-	slab->prev = NULL;
-	if (slab->next)
-		slab->next->prev = slab;
-	cache->empty = slab;
 	return true;
 }
 
-static void *takeobject(scache_t *cache, slab_t *slab) {
-	if (slab->free == NULL)
+static inline int get_index(void *p) {
+	uintptr_t key = (uintptr_t)p;
+	return fnv1ahash(&key, sizeof(key)) % 32;
+}
+
+static void *allocate_from_slab(scache_t *cache) {
+	MUTEX_ACQUIRE(&cache->mutex);
+	// if there is no slab or only an empty one, grow the cache
+	if ((cache->list == NULL || cache->list->reference_count == cache->slab_object_count) && unlikely(grow_cache(cache) == false))
 		return NULL;
 
-	void **objend = slab->free;
+	slab_t *slab = cache->list;
+
+#if SLAB_DEBUG != 0
+	__assert(slab != NULL && slab->reference_count != cache->slab_object_count);
+#endif
+
+	// take an object from the cache
+	void **free_list = slab->free;
+	slab->free = *free_list;
+	++slab->reference_count;
 
 #if SLAB_DEBUG != 0
 	if (cache->size < SLAB_INDIRECT_CUTOFF) {
@@ -92,200 +130,348 @@ static void *takeobject(scache_t *cache, slab_t *slab) {
 		void *base = (void *)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE);
 		__assert(addr >= base && addr < (void *)slab);
 	} else {
-		void *addr = slab->base + ((uintptr_t)objend - ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)) / sizeof(void **) * cache->truesize;
-		__assert(addr >= slab->base && (uintptr_t)addr < (uintptr_t)slab->base + cache->slabobjcount * cache->truesize);
+		void *addr = slab->base + ((uintptr_t)objend - ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)) / sizeof(void **) * cache->true_size;
+		__assert(addr >= slab->base && (uintptr_t)addr < (uintptr_t)slab->base + cache->slab_object_count * cache->true_size);
 	}
 #endif
 
-	slab->free = *slab->free;
-	slab->used += 1;
-	*objend = NULL;
-	if (cache->size < SLAB_INDIRECT_CUTOFF)
-		return (void *)((uintptr_t)objend - cache->size);
-	else
-		return (void *)((uintptr_t)slab->base + ((uintptr_t)objend - ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)) / sizeof(void **) * cache->truesize);
-}
-
-// frees an object and returns the slab the object belongs to
-static slab_t *returnobject(scache_t *cache, void *obj) {
-	slab_t *slab = NULL;
-	void **freeptr = NULL;
+	void *ret;
 	if (cache->size < SLAB_INDIRECT_CUTOFF) {
-		slab = (slab_t *)(ROUND_DOWN((uintptr_t)obj, PAGE_SIZE) + SLAB_PAGE_OFFSET);
-		freeptr = (void **)((uintptr_t)obj + cache->size);
-		__assert(*freeptr == NULL);
-	} else {
-		bool partial = cache->full == NULL;
-		slab = partial ? cache->partial : cache->full;
-		while (slab) {
-			void *top = (void *)((uintptr_t)slab->base + cache->slabobjcount * cache->truesize);
-			if (obj >= slab->base && obj < top)
-				break;
-			if (slab->next == NULL && partial == false) {
-				slab = cache->partial;
-				partial = true;
-			} else {
-				slab = slab->next;
-			}
-		}
-		__assert(slab);
-		uintmax_t objn = ((uintptr_t)obj - (uintptr_t)slab->base) / cache->truesize;
-		void **base = (void **)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE);
-		freeptr = &base[objn];
-	}
-
-	if (cache->dtor)
-		cache->dtor(cache, obj);
-
-	*freeptr = slab->free;
-	slab->free = freeptr;
-	--slab->used;
-
-	return slab;
-}
-
-void *slab_allocate(scache_t *cache) {
-	MUTEX_ACQUIRE(&cache->mutex);
-	slab_t *slab = cache->partial;
-	if (slab == NULL)
-		slab = cache->empty;
-
-	if (slab == NULL) {
-		if (growcache(cache)) {
-			slab = cache->empty;
-		} else {
-			MUTEX_RELEASE(&cache->mutex);
+		ret = (void *)((uintptr_t)free_list - cache->size);
+		if (cache->ctor && cache->ctor(cache, ret) == false) {
+			slab->free = free_list;
 			return NULL;
 		}
+
+#if SLAB_DEBUG != 0
+		*free_list = NULL;
+#endif
+	} else {
+		slab_indirect_t *indirect = (slab_indirect_t *)free_list;
+		if (cache->ctor && cache->ctor(cache, indirect->object) == false) {
+			slab->free = free_list;
+			return NULL;
+		}
+		// add to hash table as well
+		int idx = get_index(indirect->object);
+		indirect->next = cache->indirect_table[idx];
+		cache->indirect_table[idx] = indirect;
+
+		ret = indirect->object;
 	}
 
-	void *ret = takeobject(cache, slab);
-
-	if (slab == cache->empty) {
-		cache->empty = slab->next;
-		if (slab->next)
-			slab->next->prev = NULL;
-
-		if (cache->partial)
-			cache->partial->prev = slab;
-
-		slab->next = cache->partial;
-		slab->prev = NULL;
-		cache->partial = slab;
-	} else if (slab->used == cache->slabobjcount) {
-		cache->partial = slab->next;
-
-		if (slab->next)
-			slab->next->prev = NULL;
-
-		if (cache->full)
-			cache->full->prev = slab;
-
-		slab->next = cache->full;
-		slab->prev = NULL;
-		cache->full = slab;
-	}
+	// if the slab is empty, advance the linked list by one to the next partial or full slab
+	if (slab->reference_count == cache->slab_object_count)
+		cache->list = slab->next;
 
 	MUTEX_RELEASE(&cache->mutex);
 	return ret;
 }
 
-void slab_free(scache_t *cache, void *addr) {
+static void free_to_slab(scache_t *cache, void *addr) {
 	MUTEX_ACQUIRE(&cache->mutex);
 
-	slab_t *slab = returnobject(cache, addr);
-	__assert(slab);
+	// insert into a slab
+	slab_t *slab = NULL;
+	void **free_list = NULL;
 
-	if (slab->used == 0) {
-		if (slab->prev == NULL)
-			cache->partial = slab->next;
+	if (cache->size < SLAB_INDIRECT_CUTOFF) {
+		slab = GET_SLAB(addr);
+		free_list = (void **)((uintptr_t)addr + cache->size);
+
+#if SLAB_DEBUG != 0
+		__assert(*free_list == NULL);
+#endif
+	} else {
+		int idx = get_index(addr);
+		slab_indirect_t *prev = NULL;
+		slab_indirect_t *indirect = cache->indirect_table[idx];
+
+		while (indirect->object != addr) {
+			prev = indirect;
+			indirect = indirect->next;
+		}
+
+		if (prev)
+			prev->next = indirect->next;
 		else
-			slab->prev->next = slab->next;
+			cache->indirect_table[idx] = indirect->next;
 
-		if (slab->next)
-			slab->next->prev = slab->prev;
+		slab = indirect->slab;
+		free_list = (void **)indirect;
 
-		slab->next = cache->empty;
-		slab->prev = NULL;
-		if (slab->next)
-			slab->next->prev = slab;
-		cache->empty = slab;
-	} else if (slab->used == cache->slabobjcount - 1) {
-		if (slab->prev == NULL)
-			cache->full = slab->next;
-		else
-			slab->prev->next = slab->next;
-
-		if (slab->next)
-			slab->next->prev = slab->prev;
-
-		slab->next = cache->partial;
-		slab->prev = NULL;
-		if (slab->next)
-			slab->next->prev = slab;
-		cache->partial = slab;
+#if SLAB_DEBUG != 0
+		__assert(slab);
+		__assert(obj >= slab->base && addr < top);
+#endif
 	}
+
+	if (cache->dtor)
+		cache->dtor(cache, addr);
+
+	*free_list = slab->free;
+	slab->free = free_list;
+	--slab->reference_count;
+
+	// reorder lists if nescessary
+	if (slab->reference_count == cache->slab_object_count - 1) {
+		// if it became a partial slab, reorder the list
+		slab->prev->next = slab->next;
+		slab->next->prev = slab->prev;
+
+		slab->next = cache->list;
+		slab->prev = cache->list->prev;
+		slab->prev->next = slab;
+		slab->next->prev = slab;
+
+		cache->list = slab;
+	} else if (slab->reference_count == 0) {
+		// if it became a full slab, reorder the list
+		slab_t *search = cache->list;
+		do {
+			search = search->next;
+		} while (search != cache->list && search->reference_count != cache->size && (search->reference_count != 0 || search == slab));
+
+		// if the list points to this slab and the next one is a partial, set the list to it
+		// otherwise, the next is full (refcount 0) or empty (refcount == slab_object_count), which means this slab will be still be the next one to
+		// be allocated out of
+		if (cache->list == slab && slab->next->reference_count != 0 && slab->next->reference_count != cache->slab_object_count)
+		       cache->list = slab->next;
+
+		slab->prev->next = slab->next;
+		slab->next->prev = slab->prev;
+
+		slab->next = search;
+		slab->prev = search->prev;
+		search->prev = slab;
+		slab->prev->next = slab;
+	}
+
 	MUTEX_RELEASE(&cache->mutex);
 }
+static inline void acquire_depot_mutex(scache_t *cache) {
+	// keep track of contention for dynamic resizing of magazines in this cache
+	if (MUTEX_TRY(&cache->depot_mutex)) {
+		if (__atomic_fetch_sub(&cache->contention_count, 1, __ATOMIC_SEQ_CST) == 0)
+			cache->contention_count = 0;
 
-scache_t *slab_newcache(size_t size, size_t alignment, void (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *)) {
+	} else {
+		__atomic_add_fetch(&cache->contention_count, 1, __ATOMIC_SEQ_CST);
+		MUTEX_ACQUIRE(&cache->depot_mutex);
+	}
+}
+
+void *slab_allocate(scache_t *cache) {
+	cache_per_cpu_t *cpu_cache = &cache->per_cpu[current_cpu_internal_id()];
+
+	MUTEX_ACQUIRE(&cpu_cache->mutex);
+	// are there any rounds in the loaded magazine?
+	if (cpu_cache->loaded_magazine && cpu_cache->loaded_magazine->round_count) {
+		void *ret = cpu_cache->loaded_magazine->rounds[--cpu_cache->loaded_magazine->round_count];
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return ret;
+	}
+
+	// are there any rounds in the previous magazine?
+	if (cpu_cache->previous_magazine && cpu_cache->previous_magazine->round_count) {
+		// swap loaded and previous
+		magazine_t *swap = cpu_cache->loaded_magazine;
+		cpu_cache->loaded_magazine = cpu_cache->previous_magazine;
+		cpu_cache->previous_magazine = swap;
+
+		void *ret = cpu_cache->loaded_magazine->rounds[--cpu_cache->loaded_magazine->round_count];
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return ret;
+	}
+
+	// check the depot for full magazines
+	acquire_depot_mutex(cache);
+
+	if (cache->full_depot) {
+		// put the previous magazine into the depot
+		if (cpu_cache->previous_magazine) {
+			cpu_cache->previous_magazine->next = cache->empty_depot;
+			cache->empty_depot = cpu_cache->previous_magazine;
+		}
+
+		// load a new full magazine
+		cpu_cache->previous_magazine = cpu_cache->loaded_magazine;
+		cpu_cache->loaded_magazine = cache->full_depot;
+		cache->full_depot = cache->full_depot->next;
+
+		void *ret = cpu_cache->loaded_magazine->rounds[--cpu_cache->loaded_magazine->round_count];
+		MUTEX_RELEASE(&cache->depot_mutex);
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return ret;
+	}
+
+	MUTEX_RELEASE(&cache->depot_mutex);
+
+	// fall back into allocating from the slab directly
+	MUTEX_RELEASE(&cpu_cache->mutex);
+	return allocate_from_slab(cache);
+}
+
+static magazine_t *allocate_magazine_for_cache(scache_t *cache) {
+	return slab_allocate(magazine_cache);
+}
+
+void slab_free(scache_t *cache, void *addr) {
+	cache_per_cpu_t *cpu_cache = &cache->per_cpu[current_cpu_internal_id()];
+
+	MUTEX_ACQUIRE(&cpu_cache->mutex);
+	// can insert into the loaded magazine?
+	if (cpu_cache->loaded_magazine && cpu_cache->loaded_magazine->round_count < cache->magazine_size) {
+		cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr;
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return;
+	}
+
+	// can insert into previous magazine?
+	// the check for 0 is valid as the previous magazines will either be full or empty
+	if (cpu_cache->previous_magazine && cpu_cache->previous_magazine->round_count == 0) {
+		// swap loaded and previous
+		magazine_t *swap = cpu_cache->loaded_magazine;
+		cpu_cache->loaded_magazine = cpu_cache->previous_magazine;
+		cpu_cache->previous_magazine = swap;
+
+		cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr;
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return;
+	}
+
+	// can get an empty magazine from the depot?
+	acquire_depot_mutex(cache);
+	if (cache->empty_depot) {
+		// put the previous magazine into the depot
+		if (cpu_cache->previous_magazine) {
+			cpu_cache->previous_magazine->next = cache->full_depot;
+			cache->full_depot = cpu_cache->previous_magazine;
+		}
+
+		// load a new empty magazine
+		cpu_cache->previous_magazine = cpu_cache->loaded_magazine;
+		cpu_cache->loaded_magazine = cache->empty_depot;
+		cache->empty_depot = cache->empty_depot->next;
+
+		cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr;
+		MUTEX_RELEASE(&cache->depot_mutex);
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return;
+	}
+
+	// allocate an empty magazine
+	magazine_t *new_magazine = allocate_magazine_for_cache(cache);
+	if (new_magazine) {
+		// put previous into the depot
+		if (cpu_cache->previous_magazine) {
+			cpu_cache->previous_magazine->next = cache->full_depot;
+			cache->full_depot = cpu_cache->previous_magazine;
+		}
+
+		// load the new magazine
+		cpu_cache->previous_magazine = cpu_cache->loaded_magazine;
+		cpu_cache->loaded_magazine = new_magazine;
+
+		cpu_cache->loaded_magazine->rounds[cpu_cache->loaded_magazine->round_count++] = addr;
+		MUTEX_RELEASE(&cache->depot_mutex);
+		MUTEX_RELEASE(&cpu_cache->mutex);
+		return;
+	}
+	MUTEX_RELEASE(&cache->depot_mutex);
+
+	// fall back into freeing to the slab directly
+	MUTEX_RELEASE(&cpu_cache->mutex);
+	return free_to_slab(cache, addr);
+}
+
+#define MAGAZINE_SIZE 8
+
+static bool slab_initialize(scache_t *cache, size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *), slab_indirect_t **indirect_table) {
 	if (alignment == 0)
 		alignment = 8;
 
-	if (selfcacheinit == false) {
-		selfcacheinit = true;
-		MUTEX_INIT(&selfcache.mutex);
+	if (size >= SLAB_INDIRECT_CUTOFF && indirect_table == NULL) {
+		cache->indirect_table = alloc(sizeof(slab_indirect_t *) * 32);
+		if (cache->indirect_table == NULL)
+			return false;
+	} else if (size >= SLAB_INDIRECT_CUTOFF && indirect_table) {
+		cache->indirect_table = indirect_table;
 	}
 
-	scache_t *cache = slab_allocate(&selfcache);
-	if (cache == NULL)
-		return NULL;
-
 	cache->size = size;
 	cache->alignment = alignment;
 	size_t freeptrsize = size < SLAB_INDIRECT_CUTOFF ? sizeof(void **) : 0;
-	cache->truesize = ROUND_UP(size + freeptrsize, alignment);
+	cache->true_size = ROUND_UP(size + freeptrsize, alignment);
 	cache->ctor = ctor;
 	cache->dtor = dtor;
-	cache->slabobjcount = size < SLAB_INDIRECT_CUTOFF ? SLAB_DATA_SIZE / cache->truesize : SLAB_INDIRECT_COUNT;
-	cache->full = NULL;
-	cache->empty = NULL;
-	cache->partial = NULL;
-	MUTEX_INIT(&cache->mutex);
+	cache->slab_object_count = size < SLAB_INDIRECT_CUTOFF ? SLAB_DATA_SIZE / cache->true_size : SLAB_INDIRECT_COUNT;
+	cache->list = NULL;
+	cache->magazine_size = MAGAZINE_SIZE;
+	cache->magazine_minimum_size = MAGAZINE_SIZE;
+	cache->magazine_maximum_size = MAGAZINE_SIZE;
+	cache->contention_count = 0;
+	cache->empty_depot = NULL;
+	cache->full_depot = NULL;
 
-	printf("slab: new cache: size %lu align %lu truesize %lu objcount %lu\n", cache->size, cache->alignment, cache->truesize, cache->slabobjcount);
+	MUTEX_INIT(&cache->mutex);
+	MUTEX_INIT(&cache->depot_mutex);
+
+	for (int i = 0; i < arch_smp_get_cpu_count(); ++i) {
+		cache_per_cpu_t *cpu_cache = &cache->per_cpu[i];
+		cpu_cache->loaded_magazine = NULL;
+		cpu_cache->previous_magazine = NULL;
+		MUTEX_INIT(&cpu_cache->mutex);
+	}
+
+	return true;
+}
+
+scache_t *slab_newcache(size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *)) {
+	scache_t *cache = slab_allocate(self_cache);
+	if (cache == NULL)
+		return NULL;
+
+	if (!slab_initialize(cache, size, alignment, ctor, dtor, NULL)) {
+		slab_free(self_cache, cache);
+		return NULL;
+	}
+
+	printf("slab: new cache: size %lu align %lu true_size %lu objcount %lu\n", cache->size, cache->alignment, cache->true_size, cache->slab_object_count);
 
 	return cache;
 }
 
-static size_t purge(scache_t *cache, size_t maxcount){
-	slab_t *slab = cache->empty;
-	for (size_t done = 0; done < maxcount; ++done) {
-		if (slab == NULL)
-			return done;
 
-		slab_t *next = slab->next;
-		if (next)
-			next->prev = NULL;
+static bool magazine_ctor(scache_t *, void *obj) {
+	magazine_t *mag = obj;
+	mag->next = NULL;
+	mag->round_count = 0;
 
-		if (cache->size >= SLAB_INDIRECT_CUTOFF)
-			vmm_unmap(slab->base, cache->slabobjcount * cache->truesize, 0);
+#if SLAB_DEBUG != 0
+	memset(mag->rounds, 0xfe, MAGAZINE_SIZE * sizeof(void *));
+#endif
 
-		pmm_release(FROM_HHDM((void *)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)));
-
-		slab = cache->empty;
-		cache->empty = next;
-	}
-
-	return maxcount;
+	return true;
 }
 
-void slab_freecache(scache_t *cache) {
-	MUTEX_ACQUIRE(&cache->mutex);
-	__assert(cache->partial == NULL);
-	__assert(cache->full == NULL);
-
-	purge(cache, (size_t)-1);
-
-	slab_free(&selfcache, cache);
+static scache_t *create_new_from_vmm(size_t size, size_t alignment, bool (*ctor)(scache_t *, void *), void (*dtor)(scache_t *, void *), slab_indirect_t **table) {
+	size_t cache_size = sizeof(scache_t) + sizeof(cache_per_cpu_t) * arch_smp_get_cpu_count();
+	scache_t *cache = vmm_map(NULL, cache_size, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
+	__assert(cache);
+	__assert(slab_initialize(cache, size, alignment, ctor, dtor, table));
+	return cache;
 }
+
+// TODO dynamically change the size of magazines
+void slab_init(void) {
+	size_t cache_size = sizeof(scache_t) + sizeof(cache_per_cpu_t) * arch_smp_get_cpu_count();
+	// the cache of caches needs for the slab and indirect caches to be up
+	slab_cache = create_new_from_vmm(sizeof(slab_t), 0, NULL, NULL, self_cache_indirect_table);
+	indirect_cache = create_new_from_vmm(sizeof(slab_indirect_t), 0, NULL, NULL, NULL);
+	self_cache = create_new_from_vmm(cache_size, 0, NULL, NULL, NULL);
+
+	magazine_cache = slab_newcache(sizeof(magazine_t) + sizeof(void *) * MAGAZINE_SIZE, 0, magazine_ctor, NULL);
+}
+
+INIT_ROUTINE_DEFINE(slab, INIT_ROUTINE_FLAGS_NONE, slab_init, vmm);

kernel-src/mm/vmm.c

@@ -821,17 +821,18 @@ void vmm_unmap(void *addr, size_t size, int flags) {
 
 static scache_t *ctxcache;
 
-static void ctxctor(scache_t *cache, void *obj) {
+static bool ctxctor(scache_t *cache, void *obj) {
 	vmmcontext_t *ctx = obj;
 	ctx->space.start = USERSPACE_START;
 	ctx->space.end = USERSPACE_END;
 	MUTEX_INIT(&ctx->space.lock);
 	ctx->space.ranges = NULL;
+	return true;
 }
 
 vmmcontext_t *vmm_newcontext() {
 	if (ctxcache == NULL) {
-		ctxcache = slab_newcache(sizeof(vmmcontext_t), 0, ctxctor, ctxctor);
+		ctxcache = slab_newcache(sizeof(vmmcontext_t), 0, ctxctor, NULL);
 		__assert(ctxcache);
 	}
 
@@ -854,6 +855,8 @@ void vmm_destroycontext(vmmcontext_t *context) {
 	vmm_unmap(context->space.start, context->space.end - context->space.start, 0);
 	vmm_switchcontext(oldctx);
 	arch_mmu_destroytable(context->pagetable);
+
+	context->space.ranges = NULL;
 	slab_free(ctxcache, context);
 }
 

kernel-src/sys/syscalls/futex.c

@@ -6,7 +6,6 @@
 #include <kernel/poll.h>
 #include <kernel/alloc.h>
 
-static scache_t *futexcache;
 static mutex_t futexmutex;
 static hashtable_t hashtable;
 static bool hashtableinit;
@@ -36,29 +35,13 @@ static void removefutex(void *physical) {
 	hashtable_remove(&hashtable, &physical, sizeof(physical));
 }
 
-static void ctor(scache_t *cache, void *obj) {
-	futex_t *futex = obj;
-	POLL_INITHEADER(&futex->pollheader);
-	futex->waiting = 0;
-	futex->waking = 0;
-}
-
 syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value, timespec_t *tm) {
 	syscallret_t ret = {
 		.ret = -1
 	};
 
-	if (futexcache == NULL) {
-		futexcache = slab_newcache(sizeof(thread_t), 0, ctor, ctor);
-
-		if (futexcache == NULL) {
-			ret.errno = ENOMEM;
-			return ret;
-		}
-	}
-
-	if (hashtableinit == false) {
-		if (hashtable_init(&hashtable, 256)) {
+	if (unlikely(hashtableinit == false)) {
+		if (unlikely(hashtable_init(&hashtable, 256))) {
 			ret.errno = ENOMEM;
 			return ret;
 		}
@@ -70,7 +53,7 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value
 	timespec_t timespec = tm ? *tm : (timespec_t){0};
 	polldesc_t desc = {0};
 	ret.errno = poll_initdesc(&desc, 1);
-	if (ret.errno)
+	if (unlikely(ret.errno))
 		return ret;
 
 	MUTEX_ACQUIRE(&futexmutex);
@@ -78,7 +61,7 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value
 	bool doleave = true;
 	uint32_t word;
 	ret.errno = usercopy_fromuseratomic32(futexp, &word);
-	if (ret.errno)
+	if (unlikely(ret.errno))
 		goto cleanup;
 
 	uint32_t *physical = vmm_getphysical(futexp, false);