Merge pull request #48 from wnsntgs/Kernel/Lib/ListOptimizations

Optimize kernel lists a bit

kernel-src/fs/abc.c

@@ -39,7 +39,7 @@ static list_node_t *try_stealing_from_dirty(abc_t *abc, uint64_t block) {
 	if ((__atomic_load_n(&blk->flags, __ATOMIC_RELAXED) & ABC_BLOCK_FLAGS_DIRTY) == 0)
 		return NULL;
 
-	list_remove(&abc->dirty_list, &blk->dirty_list_node);
+	list_remove(&blk->dirty_list_node);
 	__atomic_exchange_n(&blk->flags, ABC_BLOCK_FLAGS_BUSY, __ATOMIC_ACQUIRE);
 	return &blk->dirty_list_node;
 }
@@ -50,7 +50,7 @@ void abc_sync(abc_t *abc) {
 	EVENT_ATTACH(&listener, &abc->dirty_list_empty_event);
 
 	pushlock_acquire_exclusive(&abc->dirty_list_lock);
-	if (abc->dirty_list.head == NULL && !abc->syncing) {
+	if (list_is_empty(&abc->dirty_list) && !abc->syncing) {
 		pushlock_release_exclusive(&abc->dirty_list_lock);
 		EVENT_DETACHALL(&listener);
 		return;
@@ -77,7 +77,7 @@ static void writer_thread(void) {
 	interrupt_loweripl(ipl);
 
 	for (;;) {
-		list_t internal_list;
+		list_node_t internal_list;
 		list_init(&internal_list);
 
 		eventlistener_t listener;
@@ -133,7 +133,7 @@ static void writer_thread(void) {
 		pushlock_release_shared(&abc->lock);
 		pushlock_release_exclusive(&abc->dirty_list_lock);
 
-		abc_block_t *front_block = (abc_block_t *)internal_list.head;
+		abc_block_t *front_block = (abc_block_t *)list_get_first(&internal_list);
 		size_t blocks_per_page = PAGE_SIZE / abc->block_size;
 		size_t iovec_count = (block_count + front_block->block % blocks_per_page + blocks_per_page - 1) / blocks_per_page;
 		iovec_t iovec[iovec_count];
@@ -164,7 +164,7 @@ static void writer_thread(void) {
 			__assert(written == block_count * abc->block_size);
 		}
 
-		list_for_each_safe(&internal_list, list_node) {
+		list_drain_front (&internal_list, list_node) {
 			abc_block_t *blk = (abc_block_t *)list_node;
 			if (__atomic_and_fetch(&blk->flags, ~ABC_BLOCK_FLAGS_BUSY, __ATOMIC_ACQUIRE) & ABC_BLOCK_FLAGS_DIRTY) {
 				pushlock_acquire_exclusive(&abc->dirty_list_lock);

kernel-src/fs/vfs.c

@@ -50,7 +50,7 @@ typedef struct {
 	void *ctx;
 } root_event_waiter_t;
 
-static list_t root_event_list;
+static list_node_t root_event_list;
 static bool root_mounted;
 static mutex_t root_event_mutex;
 
@@ -59,7 +59,7 @@ void vfs_root_event_signal(void) {
 	root_mounted = true;
 	MUTEX_RELEASE(&root_event_mutex);
 
-	list_for_each_safe (&root_event_list, node) {
+	list_drain_front (&root_event_list, node) {
 		root_event_waiter_t *rew = (root_event_waiter_t *)node;
 		rew->fn(rew->ctx);
 		free(rew);

kernel-src/include/kernel/abc.h

@@ -25,7 +25,7 @@ typedef struct {
 	size_t block_size;
 
 	pushlock_t dirty_list_lock;
-	list_t dirty_list;
+	list_node_t dirty_list;
 	bool syncing;
 	eventheader_t dirty_list_sync_event;
 	eventheader_t dirty_list_empty_event;

kernel-src/include/kernel/dpc.h

@@ -3,13 +3,13 @@
 
 #include <arch/context.h>
 #include <stdbool.h>
+#include <list.h>
 
 typedef void * dpcarg_t;
 typedef void (*dpcfn_t)(context_t *, dpcarg_t);
 
 typedef struct _dpc_t {
-	struct _dpc_t *next;
+	list_node_t list_node;
-	struct _dpc_t *prev;
 	bool enqueued;
 	dpcfn_t fn;
 	dpcarg_t arg;

kernel-src/include/kernel/input.h

@@ -264,7 +264,7 @@ typedef struct {
 
 struct input_device {
 	spinlock_t lock;
-	list_t listeners;
+	list_node_t listeners;
 
 	int clock_id;
 

kernel-src/include/kernel/scheduler.h

@@ -20,15 +20,12 @@
 #define SCHED_SLEEP_TIME_LIMIT_CPU_US 1000
 
 typedef struct {
-	 struct {
+	list_node_t queues[SCHED_RUN_QUEUE_SIZE];
-		 thread_t *ins;
-		 thread_t *run;
-	 } queues[SCHED_RUN_QUEUE_SIZE];
 	bitmap_t thread_bitmap;
 } sched_run_queue_t;
 
 typedef struct {
-	thread_t *queues[SCHED_RUN_QUEUE_SIZE];
+	list_node_t queues[SCHED_RUN_QUEUE_SIZE];
 	int ins;
 	int run;
 	size_t thread_count;

kernel-src/include/kernel/thread.h

@@ -7,6 +7,7 @@
 #include <kernel/signal.h>
 #include <kernel/event.h>
 #include <kernel/kcov.h>
+#include <list.h>
 
 #define THREAD_FLAGS_QUEUED 1
 #define THREAD_FLAGS_RUNNING 2
@@ -23,8 +24,7 @@ struct proc_t;
 
 typedef struct thread_t {
 	void *kernelstacktop;
-	struct thread_t *next;
+	list_node_t queue_node;
-	struct thread_t *prev;
 	struct thread_t *sleepnext;
 	struct thread_t *sleepprev;
 	struct thread_t *procnext;

kernel-src/include/kernel/work.h

@@ -11,9 +11,9 @@ typedef void (*work_callback_t)(void *, size_t);
 typedef struct {
 	const char *name;
 	spinlock_t lock;
-	list_t work_list;
+	list_node_t work_list;
-	list_t active_list;
+	list_node_t active_list;
-	list_t waiter_list;
+	list_node_t waiter_list;
 	semaphore_t pending_work;
 	thread_t **threads;
 	size_t thread_count;

kernel-src/include/kernel/xhci.h

@@ -377,7 +377,7 @@ typedef struct xhci_ctrl {
 	usb_ctrl_t ctrl;
 	pcienum_t *pci_enum;
 
-	list_t hubs;
+	list_node_t hubs;
 
 	uint32_t port_count;
 	uint32_t slot_count;
@@ -394,7 +394,7 @@ typedef struct xhci_ctrl {
 	spinlock_t event_lock;
 	mutex_t callback_mutex;
 	bitmap_t port_change_bitmap;
-	list_t completion_list;
+	list_node_t completion_list;
 	ringbuffer_t callback_ring;
 	dpc_t dpc;
 

kernel-src/include/list.h

@@ -1,6 +1,7 @@
 #ifndef _LIST_H
 #define _LIST_H
 
+#include <stdbool.h>
 #include <stddef.h>
 
 typedef struct _list_node {
@@ -8,77 +9,75 @@ typedef struct _list_node {
 	struct _list_node *next;
 } list_node_t;
 
-typedef struct {
+#define LIST_INIT_VALUE(NAME) ((list_node_t) {.prev = &(NAME), .next = &(NAME)})
-	list_node_t *head;
+#define LIST_DEFINE(NAME) list_node_t NAME = LIST_INIT_VALUE(NAME)
-	list_node_t *tail;
-} list_t;
 
-#define LIST_INIT_VALUE ((list_t) {.head = NULL, .tail = NULL})
+static inline void list_init(list_node_t *list) {
-
+	list->prev = list;
-static inline void list_init(list_t *list) {
+	list->next = list;
-	list->head = NULL;
-	list->tail = NULL;
 }
 
-static inline void list_push_front(list_t *list, list_node_t *node) {
+static inline bool list_is_empty(const list_node_t *list) {
-	node->prev = NULL;
+	return list->next == list;
-	node->next = list->head;
-	if (list->head != NULL) {
-		list->head->prev = node;
-	} else {
-		list->tail = node;
-	}
-	list->head = node;
 }
 
-static inline void list_push_back(list_t *list, list_node_t *node) {
+static inline list_node_t *list_get_first(const list_node_t *list) {
-	node->next = NULL;
+	return list_is_empty(list) ? NULL : list->next;
-	node->prev = list->tail;
-	if (list->tail != NULL) {
-		list->tail->next = node;
-	} else {
-		list->head = node;
-	}
-	list->tail = node;
 }
 
-static inline void list_remove(list_t *list, list_node_t *node) {
+static inline list_node_t *list_get_last(const list_node_t *list) {
-	if (node->prev != NULL) {
+	return list_is_empty(list) ? NULL : list->prev;
+}
+
+static inline void list_push_front(list_node_t *list, list_node_t *node) {
+	node->prev = list;
+	node->next = list->next;
+	list->next->prev = node;
+	list->next = node;
+}
+
+static inline void list_push_back(list_node_t *list, list_node_t *node) {
+	node->next = list;
+	node->prev = list->prev;
+	list->prev->next = node;
+	list->prev = node;
+}
+
+static inline void list_remove(list_node_t *node) {
 	node->prev->next = node->next;
-	} else {
-		list->head = node->next;
-	}
-	if (node->next != NULL) {
 	node->next->prev = node->prev;
-	} else {
-		list->tail = node->prev;
-	}
 	node->prev = NULL;
 	node->next = NULL;
 }
 
-static inline list_node_t *list_pop_front(list_t *list) {
+static inline list_node_t *list_pop_front(list_node_t *list) {
-	if (list->head == NULL) {
+	list_node_t *node = list->next;
+	if (node == list)
 		return NULL;
-	}
+	list_remove(node);
-	list_node_t *node = list->head;
-	list_remove(list, node);
 	return node;
 }
 
-static inline list_node_t *list_pop_back(list_t *list) {
+static inline list_node_t *list_pop_back(list_node_t *list) {
-	if (list->tail == NULL) {
+	list_node_t *node = list->prev;
+	if (node == list)
 		return NULL;
-	}
+	list_remove(node);
-	list_node_t *node = list->tail;
-	list_remove(list, node);
 	return node;
 }
 
 #define list_for_each(LIST, ITER) \
-	for (list_node_t *ITER = (LIST)->head; ITER != NULL; ITER = ITER->next)
+	for (list_node_t *ITER = (LIST)->next; ITER != (LIST); ITER = ITER->next)
 
-#define list_for_each_safe(LIST, ITER) \
+#define list_for_each_safe(LIST, ITER, NEXT) \
-	for (list_node_t *ITER = (LIST)->head, *_next = (ITER ? ITER->next : NULL); ITER != NULL; ITER = _next, _next = (ITER ? ITER->next : NULL))
+	for (list_node_t *ITER = (LIST)->next, *NEXT = ITER->next; \
+		 ITER != (LIST); \
+		 ITER = NEXT, NEXT = ITER->next)
+
+#define list_drain_front(LIST, ITER) \
+	for (list_node_t *ITER = list_pop_front(LIST); ITER != NULL; ITER = list_pop_front(LIST))
+
+#define list_drain_back(LIST, ITER) \
+	for (list_node_t *ITER = list_pop_back(LIST); ITER != NULL; ITER = list_pop_back(LIST))
 
 #endif

kernel-src/include/x86-64/arch/cpu.h

@@ -46,7 +46,7 @@ typedef struct cpu_t {
 	timer_t *timer;
 
 	isr_t *dpcisr;
-	dpc_t *dpcqueue;
+	list_node_t dpcqueue;
 
 	thread_t *idlethread;
 	timerentry_t schedtimerentry;

kernel-src/io/acpi/glue.c

@@ -365,7 +365,7 @@ struct acpi_work {
 struct acpi_workctx {
 	work_queue_t *queue;
 	spinlock_t pool_lock;
-	list_t free_list;
+	list_node_t free_list;
 	struct acpi_work pool[ACPI_WORK_POOL_SIZE];
 	bool target_bsp;
 };

kernel-src/io/audio/hda.c

@@ -197,7 +197,7 @@ typedef struct hda hda_t;
 typedef struct hda_output_path hda_output_path_t;
 
 typedef struct {
-	list_t waiter_list;
+	list_node_t waiter_list;
 	spinlock_t waiter_list_lock;
 	size_t function_group_count;
 	size_t starting_node;
@@ -630,7 +630,7 @@ static hda_path_t hda_find_path(hda_fg_t *fg, hda_widget_t *pin, int goal_type)
 		hda_path_t path;
 	} path_entry_t;
 
-	list_t queue;
+	list_node_t queue;
 	list_init(&queue);
 
 	bool *visited = alloc(sizeof(bool) * fg->widget_count);

kernel-src/io/input.c

@@ -142,7 +142,7 @@ static int input_listener_close(int minor, int flags) {
 	MUTEX_RELEASE(&input_dev_table_lock);
 
 	long ipl = spinlock_acquire_raise_ipl(&lis->device->lock, IPL_INPUT);
-	list_remove(&lis->device->listeners, &lis->node);
+	list_remove(&lis->node);
 	spinlock_release_lower_ipl(&lis->device->lock, ipl);
 
 	ringbuffer_destroy(&lis->buffer);

kernel-src/io/net/raw.c

@@ -11,7 +11,7 @@
 #define SOCKET_BUFFER (16 * 1024)
 
 SPINLOCK_DEFINE(list_lock);
-static list_t socket_list;
+static LIST_DEFINE(socket_list);
 
 typedef struct {
 	uint8_t sender[6];
@@ -260,7 +260,7 @@ static int raw_getpeername(socket_t *socket, sockaddr_t *addr) {
 static void raw_destroy(socket_t *handle) {
 	raw_socket_t *socket = (raw_socket_t *)handle;
 	long ipl = spinlock_acquire_raise_ipl(&list_lock, IPL_DPC);
-	list_remove(&socket_list, &socket->list_node);
+	list_remove(&socket->list_node);
 	spinlock_release_lower_ipl(&list_lock, ipl);
 	free(socket);
 }

kernel-src/sys/dpc.c

@@ -4,30 +4,9 @@
 #include <logging.h>
 
 
-static void remove(dpc_t *dpc) {
-	if (dpc->prev)
-		dpc->prev->next = dpc->next;
-	else 
-		current_cpu()->dpcqueue = dpc->next;
-
-	if (dpc->next)
-		dpc->next->prev = dpc->prev;
-}
-
-static void insert(dpc_t *dpc) {
-	dpc->prev = NULL;
-	dpc->next = current_cpu()->dpcqueue;
-
-	if (dpc->next)
-		dpc->next->prev = dpc;
-
-	current_cpu()->dpcqueue = dpc;
-}
-
 static void isrfn(isr_t *self, context_t *context) {
-	while (current_cpu()->dpcqueue) {
+	list_drain_front (&current_cpu()->dpcqueue, node) {
-		dpc_t *dpc = current_cpu()->dpcqueue;
+		dpc_t *dpc = (dpc_t *)node;
-		remove(dpc);
 		dpcarg_t arg = dpc->arg;
 
 		__assert(dpc->enqueued);
@@ -42,8 +21,8 @@ static void isrfn(isr_t *self, context_t *context) {
 void dpc_prepare(dpc_t *dpc, dpcfn_t fn) {
 	dpc->fn = fn;
 	dpc->enqueued = false;
-	dpc->next = NULL;
+	dpc->list_node.next = NULL;
-	dpc->prev = NULL;
+	dpc->list_node.prev = NULL;
 }
 
 void dpc_enqueue(dpc_t *dpc, dpcarg_t arg) {
@@ -54,7 +33,7 @@ void dpc_enqueue(dpc_t *dpc, dpcarg_t arg) {
 
 	dpc->arg = arg;
 	dpc->enqueued = true;
-	insert(dpc);
+	list_push_back(&current_cpu()->dpcqueue, &dpc->list_node);
 	interrupt_raise(current_cpu()->dpcisr);
 
 	cleanup:
@@ -68,13 +47,14 @@ void dpc_dequeue(dpc_t *dpc) {
 		goto cleanup;
 
 	dpc->enqueued = false;
-	remove(dpc);
+	list_remove(&dpc->list_node);
 
 	cleanup:
 	interrupt_set(entrystate);
 }
 
 void dpc_init() {
+	list_init(&current_cpu()->dpcqueue);
 	current_cpu()->dpcisr = interrupt_allocate(isrfn, NULL, IPL_DPC);
 	__assert(current_cpu()->dpcisr);
 }

kernel-src/sys/scheduler/queue.c

@@ -54,16 +54,7 @@ static cpu_t *pick_cpu(thread_t *thread) {
 static void insert_in_run_queue(sched_run_queue_t *run_queue, thread_t *thread) {
 	int idx = sched_thread_run_queue_index(thread->metrics.interactivity_score);
 
-	thread->prev = run_queue->queues[idx].ins;
+	list_push_back(&run_queue->queues[idx], &thread->queue_node);
-
-	if (thread->prev)
-		thread->prev->next = thread;
-	else
-		run_queue->queues[idx].run = thread;
-
-	thread->next = NULL;
-	run_queue->queues[idx].ins = thread;
-
 	bitmap_set(&run_queue->thread_bitmap, idx, 1);
 }
 
@@ -72,14 +63,12 @@ static thread_t *pop_from_run_queue(sched_run_queue_t *run_queue) {
 	if (idx == -1)
 		return NULL;
 
-	thread_t *thread = run_queue->queues[idx].run;
+	list_node_t *node = run_queue->queues[idx].next;
+	thread_t *thread = container_of(node, thread_t, queue_node);
 
-	run_queue->queues[idx].run = thread->next;
+	list_remove(node);
 
-	if (thread->next) {
+	if (list_is_empty(&run_queue->queues[idx])) {
-		thread->next->prev = NULL;
-	} else {
-		run_queue->queues[idx].ins = NULL;
 		bitmap_set(&run_queue->thread_bitmap, idx, 0);
 	}
 
@@ -92,22 +81,14 @@ static thread_t *steal_from_calendar_queue(sched_calendar_queue_t *calendar_queu
 
 	int run = calendar_queue->run;
 
-	thread_t *thread = NULL;
 	do {
-		thread = calendar_queue->queues[run];
+		list_for_each (&calendar_queue->queues[run], node) {
-		thread_t *prev = NULL;
+			thread_t *thread = container_of(node, thread_t, queue_node);
-		while (thread) {
 			if (thread->cputarget) {
-				prev = thread;
-				thread = thread->next;
 				continue;
 			}
 
-			if (prev)
+			list_remove(node);
-				prev->next = thread->next;
-			else
-				calendar_queue->queues[run] = thread->next;
-
 			--calendar_queue->thread_count;
 			return thread;
 		}
@@ -125,26 +106,18 @@ static thread_t *steal_from_run_queue(sched_run_queue_t *run_queue) {
 		return NULL;
 
 	for (; idx < SCHED_RUN_QUEUE_SIZE; ++idx) {
-		for (thread_t *search = run_queue->queues[idx].run; search; search = search->next) {
+		list_for_each (&run_queue->queues[idx], node) {
-			if (search->cputarget)
+			thread_t *thread = container_of(node, thread_t, queue_node);
+			if (thread->cputarget) {
 				continue;
-
-			if (search->next) {
-				search->next->prev = search->prev;
-			} else {
-				run_queue->queues[idx].ins = search->prev;
 			}
 
-			if (search->prev) {
+			list_remove(node);
-				search->prev->next = search->next;
+			if (list_is_empty(&run_queue->queues[idx])) {
-			} else {
-				run_queue->queues[idx].run = search->next;
-			}
-
-			if (run_queue->queues[idx].run == NULL)
 				bitmap_set(&run_queue->thread_bitmap, idx, 0);
+			}
 
-			return search;
+			return thread;
 		}
 	}
 
@@ -155,8 +128,7 @@ static thread_t *steal_from_run_queue(sched_run_queue_t *run_queue) {
 static void insert_in_calendar_queue(sched_calendar_queue_t *calendar_queue, thread_t *thread) {
 	int idx = (calendar_queue->ins + thread->metrics.interactivity_score - SCHED_INTERACTIVITY_LIMIT) % SCHED_RUN_QUEUE_SIZE;
 
-	thread->next = calendar_queue->queues[idx];
+	list_push_front(&calendar_queue->queues[idx], &thread->queue_node);
-	calendar_queue->queues[idx] = thread;
 	++calendar_queue->thread_count;
 }
 
@@ -164,22 +136,21 @@ static thread_t *pop_from_calendar_queue(sched_calendar_queue_t *calendar_queue)
 	if (calendar_queue->thread_count == 0)
 		return NULL;
 
-	thread_t *thread;
+	list_node_t *node;
 
 	do {
-		thread = calendar_queue->queues[calendar_queue->run];
+		node = list_pop_front(&calendar_queue->queues[calendar_queue->run]);
-		if (thread)
+		if (node == NULL) {
-			calendar_queue->queues[calendar_queue->run] = thread->next;
-		else
 			calendar_queue->run = (calendar_queue->run + 1) % SCHED_RUN_QUEUE_SIZE;
+		}
 
 		if (calendar_queue->run == calendar_queue->ins)
 			calendar_queue->ins = (calendar_queue->ins + 1) % SCHED_RUN_QUEUE_SIZE;
-	} while (thread == NULL);
+	} while (node == NULL);
 
 	--calendar_queue->thread_count;
 
-	return thread;
+	return container_of(node, thread_t, queue_node);
 }
 
 // requires cpu queue to be locked

kernel-src/sys/scheduler/scheduler.c

@@ -346,9 +346,17 @@ void sched_sleep_us(size_t us) {
 	sched_yield();
 }
 
-static void set_up_bitmaps(void) {
+static void set_up_queues(void) {
-	__assert(bitmap_init(&current_cpu()->rt_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0);
+	cpu_t *cpu = current_cpu();
-	__assert(bitmap_init(&current_cpu()->idle_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0);
+
+	for (size_t i = 0; i < SCHED_RUN_QUEUE_SIZE; ++i) {
+		list_init(&cpu->rt_queue.queues[i]);
+		list_init(&cpu->idle_queue.queues[i]);
+		list_init(&cpu->ts_queue.queues[i]);
+	}
+
+	__assert(bitmap_init(&cpu->rt_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0);
+	__assert(bitmap_init(&cpu->idle_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0);
 }
 
 void sched_calendar_tick(context_t *, dpcarg_t);
@@ -358,7 +366,7 @@ void sched_ap_entry() {
 	SPINLOCK_INIT(current_cpu()->sched_lock);
 
 	dpc_prepare(&current_cpu()->reschedule_dpc, sched_reschedule_dpc);
-	set_up_bitmaps();
+	set_up_queues();
 
 	current_cpu()->schedulerstack = mm_map(NULL, SCHEDULER_STACK_SIZE, MM_RANGE_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
 	__assert(current_cpu()->schedulerstack);
@@ -386,7 +394,7 @@ void sched_init() {
 	SPINLOCK_INIT(sched_idle_cpu_bitmap_lock);
 
 	dpc_prepare(&current_cpu()->reschedule_dpc, sched_reschedule_dpc);
-	set_up_bitmaps();
+	set_up_queues();
 
 	__assert(bitmap_init(&sched_idle_cpu_bitmap, arch_smp_get_cpu_count()) == 0);
 

kernel-src/sys/syscalls/futex.c

@@ -1,4 +1,5 @@
 #include <kernel/syscalls.h>
+#include <kernel/init.h>
 #include <kernel/mm.h>
 #include <list.h>
 #include <semaphore.h>
@@ -19,7 +20,7 @@ typedef struct {
 
 typedef struct {
 	mutex_t mutex;
-	list_t list;
+	list_node_t list;
 } futex_bucket_t;
 
 #define BUCKET_COUNT 256
@@ -94,7 +95,7 @@ static int wait_on_bucket(futex_bucket_t *bucket, uintptr_t key, size_t timeoutu
 	else if (state == ENTRY_STATE_TIMEOUT)
 		error = ETIMEDOUT;
 
-	list_remove(&bucket->list, &entry.list_node);
+	list_remove(&entry.list_node);
 
 	MUTEX_RELEASE(&bucket->mutex);
 
@@ -180,3 +181,10 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value
 	mm_unlock_and_release_page(physical);
 	return ret;
 }
+
+static void futex_init(void) {
+	for (size_t i = 0; i < BUCKET_COUNT; ++i)
+		list_init(&buckets[i].list);
+}
+
+INIT_ROUTINE_DEFINE(futex, INIT_ROUTINE_FLAGS_NONE, futex_init, bsp_early);

kernel-src/sys/work.c

@@ -32,11 +32,11 @@ static bool work_active(work_queue_t *wq, work_t *work) {
 }
 
 static void wake_waiters(work_queue_t *wq, work_t *work) {
-	list_for_each_safe (&wq->waiter_list, node) {
+	list_for_each_safe (&wq->waiter_list, node, next) {
 		work_waiter_t *ww = (work_waiter_t *)node;
 
 		if (ww->work == work) {
-			list_remove(&wq->waiter_list, &ww->list_node);
+			list_remove(&ww->list_node);
 			sched_wakeup(ww->thread, SCHED_WAKEUP_REASON_NORMAL);
 		}
 	}
@@ -83,7 +83,7 @@ static void worker() {
 
 		ipl = spinlock_acquire_raise_ipl(&wq->lock, wq->lock_ipl);
 		wake_waiters(wq, work);
-		list_remove(&wq->active_list, &aw.list_node);
+		list_remove(&aw.list_node);
 		spinlock_release_lower_ipl(&wq->lock, ipl);
 	}
 }
@@ -163,7 +163,7 @@ int work_dequeue(work_queue_t *wq, work_t *work) {
 		goto leave;
 
 	wake_waiters(wq, work);
-	list_remove(&wq->work_list, &work->list_node);
+	list_remove(&work->list_node);
 	work->pending = 0;
 
 leave: