misc allocator optimisations

kernel-src/mm/alloc.c

@@ -18,24 +18,18 @@
 static size_t allocsizes[CACHE_COUNT] = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536};
 static scache_t *caches[CACHE_COUNT + 5];
 
-static void initarea(scache_t *cache, void *obj) {
+static void initarea(scache_t *cache, void *obj, size_t size) {
 	size_t *ptr = obj;
 	*ptr = CAPACITY_SIZE(cache);
+	*(ptr + 1) = size;
 	memset(ptr + 2, 0, *ptr);
 	#if USE_POISON == 1
 	*((size_t *)((uintptr_t)obj + cache->size - sizeof(size_t))) = POISON_VALUE;
 	#endif
 }
 
-static void dtor(scache_t *cache, void *obj) {
-#if USE_POISON == 1
-	__assert(*(size_t *)((uintptr_t)obj + cache->size - sizeof(size_t)) == POISON_VALUE);
-#endif
-	initarea(cache, obj);
-}
-
 static scache_t *getcachefromsize(size_t size) {
-	if (size <= 1)
+	if (unlikely(size <= 1))
 		return caches[0];
 
 	size_t i = 64 - __builtin_clzll(size - 1);
@@ -45,21 +39,18 @@ static scache_t *getcachefromsize(size_t size) {
 void *alloc(size_t size) {
 	scache_t *cache = getcachefromsize(size);
 	size_t *ret = slab_allocate(cache);
-	if (ret == NULL)
+	if (unlikely(ret == NULL))
 		return NULL;
+
+	initarea(cache, ret, size);
 	#if USE_POISON == 1
 		__assert(*ret == CAPACITY_SIZE(cache));
 	#endif
-	*(ret + 1) = size;
 	return ret + 2;
 }
 
 void free(void *ptr) {
-	size_t *start = ptr;
-	start -= 2;
-	size_t size = *start;
-	scache_t *cache = getcachefromsize(size);
-	slab_free(cache, start);
+	slab_free(getcachefromsize(*((size_t *)ptr - 2)), (size_t *)ptr - 2);
 }
 
 void *realloc(void *ptr, size_t size) {
@@ -87,7 +78,7 @@ void *realloc(void *ptr, size_t size) {
 	size_t *new = slab_allocate(newcache);
 	if (new == NULL)
 		return NULL;
-	*(new + 1) = size;
+	initarea(newcache, new, size);
 	memcpy((new + 2), ptr, currentsize);
 	slab_free(oldcache, start);
 	return new + 2;
@@ -95,8 +86,11 @@ void *realloc(void *ptr, size_t size) {
 
 void alloc_init() {
 	for (int i = 0; i < CACHE_COUNT; ++i) {
-		caches[i + 5] = slab_newcache(allocsizes[i] + sizeof(size_t) * 2 + sizeof(size_t) * USE_POISON, 0, initarea, dtor);
+		caches[i + 5] = slab_newcache(allocsizes[i] + sizeof(size_t) * 2 + sizeof(size_t) * USE_POISON, 0, NULL, NULL);
 		__assert(caches[i + 5]);
+		void *p = slab_allocate(caches[i + 5]);
+		__assert(p);
+		slab_free(caches[i + 5], p);
 	}
 
 	for (int i = 0; i < 5; ++i)

kernel-src/mm/slab.c

@@ -1,5 +1,6 @@
 #include <kernel/slab.h>
 #include <kernel/vmm.h>
+#include <kernel/pmm.h>
 #include <logging.h>
 #include <util.h>
 
@@ -53,20 +54,21 @@ static void initindirect(scache_t *cache, slab_t *slab, void **base, void *objba
 }
 
 static bool growcache(scache_t *cache) {
-	void *_slab = vmm_map(NULL, PAGE_SIZE, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
+	void *_slab = pmm_allocpage(PMM_SECTION_DEFAULT);
 	if (_slab == NULL)
 		return false;
-	slab_t *slab = GET_SLAB(_slab);
+
+	slab_t *slab = MAKE_HHDM(GET_SLAB(_slab));
 
 	if (cache->size < SLAB_INDIRECT_CUTOFF) {
-		initdirect(cache, slab, _slab);
+		initdirect(cache, slab, MAKE_HHDM(_slab));
 	} else {
 		void *base = vmm_map(NULL, cache->slabobjcount * cache->truesize, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
 		if (base == NULL) {
-			vmm_unmap(_slab, PAGE_SIZE, 0);
+			pmm_release(_slab);
 			return false;
 		}
-		initindirect(cache, slab, _slab, base);
+		initindirect(cache, slab, MAKE_HHDM(_slab), base);
 	}
 
 
@@ -144,22 +146,20 @@ static slab_t *returnobject(scache_t *cache, void *obj) {
 
 void *slab_allocate(scache_t *cache) {
 	MUTEX_ACQUIRE(&cache->mutex);
-	slab_t *slab = NULL;
-	if (cache->partial != NULL)
-		slab = cache->partial;
-	else if (cache->empty != NULL)
+	slab_t *slab = cache->partial;
+	if (slab == NULL)
 		slab = cache->empty;
 
-	void *ret = NULL;
-
 	if (slab == NULL) {
-		if (growcache(cache))
+		if (growcache(cache)) {
 			slab = cache->empty;
-		else
-			goto cleanup;
+		} else {
+			MUTEX_RELEASE(&cache->mutex);
+			return NULL;
+		}
 	}
 
-	ret = takeobject(cache, slab);
+	void *ret = takeobject(cache, slab);
 
 	if (slab == cache->empty) {
 		cache->empty = slab->next;
@@ -186,7 +186,6 @@ void *slab_allocate(scache_t *cache) {
 		cache->full = slab;
 	}
 
-	cleanup:
 	MUTEX_RELEASE(&cache->mutex);
 	return ret;
 }
@@ -211,9 +210,7 @@ void slab_free(scache_t *cache, void *addr) {
 		if (slab->next)
 			slab->next->prev = slab;
 		cache->empty = slab;
-	}
-	
-	if (slab->used == cache->slabobjcount - 1) {
+	} else if (slab->used == cache->slabobjcount - 1) {
 		if (slab->prev == NULL)
 			cache->full = slab->next;
 		else
@@ -274,7 +271,7 @@ static size_t purge(scache_t *cache, size_t maxcount){
 		if (cache->size >= SLAB_INDIRECT_CUTOFF)
 			vmm_unmap(slab->base, cache->slabobjcount * cache->truesize, 0);
 
-		vmm_unmap(slab, PAGE_SIZE, 0);
+		pmm_release(FROM_HHDM((void *)ROUND_DOWN((uintptr_t)slab, PAGE_SIZE)));
 
 		slab = cache->empty;
 		cache->empty = next;