scheduler: continue rework

kernel-src/arch/x86-64/apic.c

@@ -168,13 +168,13 @@ void arch_apic_initap() {
 	interrupt_register(0xff, spurious, NULL, IPL_IGNORE);
 	writelapic(APIC_REG_SPURIOUS, 0x1FF);
 
-	current_cpu()->id = readlapic(APIC_REG_ID) >> 24;
+	current_cpu()->hardware_id = readlapic(APIC_REG_ID) >> 24;
 
 	// get the acpi id for the local nmi sources
 
 	for (size_t i = 0; i < lapic_count; ++i) {
 		struct acpi_madt_lapic *current = getentry(ACPI_MADT_ENTRY_TYPE_LAPIC, i);
-		if (current_cpu()->id == current->id) {
+		if (current_cpu()->hardware_id == current->id) {
 			current_cpu()->acpiid = current->uid;
 			break;
 		}
@@ -233,7 +233,7 @@ void arch_apic_timerinit() {
 
 	writelapic(APIC_TIMER_INITIAL_COUNT, 0);
 
-	printf("cpu%lu: local apic timer calibrated at %lu ticks per us. ISR vector %lu\n", current_cpu()->id, ticksperus, vec);
+	printf("cpu%lu: local apic timer calibrated at %lu ticks per us. ISR vector %lu\n", current_cpu()->internal_id, ticksperus, vec);
 
 	writelapic(APIC_LVT_TIMER, vec);
 

kernel-src/arch/x86-64/cpu.c

@@ -211,7 +211,7 @@ void arch_cpu_init() {
 	char str_buf[128];
 	size_t done = 0;
 
-	done += snprintf(str_buf, 128, "cpu%d: %s@", current_cpu()->id, current_cpu()->vendor);
+	done += snprintf(str_buf, 128, "cpu%d: %s@", current_cpu()->internal_id, current_cpu()->vendor);
 	for (int i = 0; i < topology_depth; ++i)
 		done += snprintf(str_buf + done, 128 - done, "%d%c", topology_ids[i], i == topology_depth - 1 ? 0 : ':');
 	str_buf[done] = '\n';

kernel-src/arch/x86-64/smp.c

@@ -22,9 +22,13 @@ static void cpuwakeuphalt(struct limine_smp_info *info) {
 	for (;;) CPU_HALT();
 }
 
+static long next_id = 1;
+
 static void cpuwakeup(struct limine_smp_info *info) {
 	cpu_set((cpu_t *)info->extra_argument);
 
+	current_cpu()->internal_id = __atomic_fetch_add(&next_id, 1, __ATOMIC_SEQ_CST);
+
 	arch_gdt_reload();
 	arch_idt_reload();
 	interrupt_register(0xfd, (void *)cpuwakeuphalt, NULL, IPL_IGNORE);
@@ -46,16 +50,32 @@ static void cpuwakeup(struct limine_smp_info *info) {
 }
 
 void arch_smp_sendipi(cpu_t *targcpu, isr_t *isr, int target, bool nmi) {
-	arch_apic_sendipi(targcpu ? targcpu->id : 0, INTERRUPT_IDTOVECTOR(isr->id), target, nmi ? APIC_MODE_NMI : 0, 0);
+	arch_apic_sendipi(targcpu ? targcpu->hardware_id : 0, INTERRUPT_IDTOVECTOR(isr->id), target, nmi ? APIC_MODE_NMI : 0, 0);
 }
 
-void arch_smp_haltallothers() {
+void arch_smp_haltallothers(void) {
 	arch_smp_sendipi(NULL, &current_cpu()->isr[0xfd], ARCH_SMP_IPI_OTHERCPUS, true);
 }
 
+size_t arch_smp_get_cpu_count(void) {
+	if (cmdline_get("nosmp"))
+		return 1;
+
+	struct limine_smp_response *response = smprequest.response;
+	if (response == NULL)
+		return 1;
+
+	return response->cpu_count;
+}
+
 cpu_t **smp_cpus;
 
-void arch_smp_wakeup() {
+cpu_t *arch_smp_get_cpu_by_internal_id(long id) {
+	__assert(smp_cpus && id < arch_smp_get_cpu_count());
+	return smp_cpus[id];
+}
+
+void arch_smp_wakeup(void) {
 	struct limine_smp_response *response = smprequest.response;
 	if (response == NULL) {
 		// TODO try to detect the other processors manually

kernel-src/include/bitmap.h

@@ -0,0 +1,16 @@
+#ifndef _BITMAP_H
+#define _BITMAP_H
+
+#include <stddef.h>
+
+typedef struct {
+	long *data;
+	size_t size;
+} bitmap_t;
+
+int bitmap_init(bitmap_t *bitmap, size_t size);
+void bitmap_set(bitmap_t *bitmap, long idx, int v);
+long bitmap_find_first_set(bitmap_t *bitmap, long start_offset);
+
+
+#endif

kernel-src/include/kernel/proc.h

@@ -79,7 +79,6 @@ typedef struct proc_t {
 	} timer;
 } proc_t;
 
-#include <arch/cpu.h>
 
 #define UMASK(mode) ((mode) & ~current_thread()->proc->umask)
 #define PROC_HOLD(v) __atomic_add_fetch(&(v)->refcount, 1, __ATOMIC_SEQ_CST)

kernel-src/include/kernel/scheduler.h

@@ -3,6 +3,7 @@
 
 #include <kernel/proc.h>
 #include <kernel/thread.h>
+#include <bitmap.h>
 
 #define SCHED_WAKEUP_REASON_NORMAL 0
 #define SCHED_WAKEUP_REASON_INTERRUPTED -1
@@ -10,6 +11,27 @@
 #define STACK_TOP (void *)0x0000800000000000
 #define INTERP_BASE (void *)0x00000beef0000000
 
+#define SCHED_RUN_QUEUE_SIZE 32
+#define SCHED_MAX_INTERACTIVITY 100
+
+typedef struct {
+	 struct {
+		 thread_t *ins;
+		 thread_t *run;
+	 } queues[SCHED_RUN_QUEUE_SIZE];
+	bitmap_t thread_bitmap;
+} sched_run_queue_t;
+
+typedef struct {
+	thread_t *queues[SCHED_RUN_QUEUE_SIZE];
+	int ins;
+	int run;
+	size_t thread_count;
+} sched_calendar_queue_t;
+
+extern bitmap_t sched_idle_cpu_bitmap;
+extern spinlock_t sched_idle_cpu_bitmap_lock;
+
 void sched_init();
 void sched_ap_entry();
 
@@ -27,4 +49,7 @@ void sched_thread_running_callback(thread_t *thread);
 void sched_thread_stopping_callback(thread_t *thread, bool sleeping);
 void sched_thread_wakeup_callback(thread_t *thread);
 
+thread_t *sched_select_next_thread(void);
+void sched_insert_in_cpu_queue(struct cpu_t *cpu, thread_t *thread);
+
 #endif

kernel-src/include/kernel/thread.h

@@ -16,6 +16,10 @@
 
 struct proc_t;
 
+#define THREAD_CLASS_TIMESHARE 0
+#define THREAD_CLASS_REAL_TIME 1
+#define THREAD_CLASS_IDLE 2
+
 typedef struct thread_t {
 	void *kernelstacktop;
 	struct thread_t *next;
@@ -40,6 +44,7 @@ typedef struct thread_t {
 	bool shouldexit;
 	void *kernelarg;
 	context_t *usercopyctx;
+	int class;
 	struct {
 		spinlock_t lock;
 		eventheader_t waitpendingevent;

kernel-src/include/x86-64/arch/cpu.h

@@ -24,7 +24,8 @@ typedef struct cpu_t {
 
 	vmmcontext_t *vmmctx; // expected to be here by other code
 
-	long id; // expected to be here by other code
+	long hardware_id; // expected to be here by other code
+	long internal_id;
 
 	timekeeper_source_t *timekeeper_source;
 	timekeeper_source_info_t *timekeeper_source_info;
@@ -49,6 +50,14 @@ typedef struct cpu_t {
 	dpc_t  reschedule_dpc;
 	isr_t *reschedule_isr;
 
+	sched_run_queue_t rt_queue;
+	sched_calendar_queue_t ts_queue;
+	sched_run_queue_t idle_queue;
+	spinlock_t sched_lock;
+
+	long calendar_ticks;
+	timerentry_t calendar_tick_timer_entry;
+
 	// architecture specific, does not need to be exposed
 
 	uint64_t gdt[7];

kernel-src/include/x86-64/arch/smp.h

@@ -11,8 +11,9 @@
 extern size_t arch_smp_cpusawake;
 extern cpu_t **smp_cpus;
 
-void arch_smp_wakeup();
+void arch_smp_wakeup(void);
 void arch_smp_sendipi(cpu_t *targcpu, isr_t *isr, int target, bool nmi);
-void arch_smp_haltallothers();
+void arch_smp_haltallothers(void);
+size_t arch_smp_get_cpu_count(void);
 
 #endif

kernel-src/lib/bitmap.c

@@ -0,0 +1,40 @@
+#include <bitmap.h>
+#include <errno.h>
+#include <kernel/alloc.h>
+#include <util.h>
+#include <logging.h>
+
+int bitmap_init(bitmap_t *bitmap, size_t size) {
+	bitmap->data = alloc(size);
+	bitmap->size = size;
+
+	return bitmap->data ? 0 : ENOMEM;
+}
+
+void bitmap_set(bitmap_t *bitmap, long idx, int v) {
+	__assert(bitmap->size > idx);
+
+	long bit = 1llu << (idx % sizeof(*bitmap->data));
+
+	if (v)
+		bitmap->data[idx / sizeof(*bitmap->data)] |= bit;
+	else
+		bitmap->data[idx / sizeof(*bitmap->data)] &= ~bit;
+}
+
+// TODO fix start offset search
+long bitmap_find_first_set(bitmap_t *bitmap, long start_offset) {
+	size_t loop_size = ROUND_UP(bitmap->size, sizeof(*bitmap->data)) / sizeof(*bitmap->data);
+
+	long offset = -1;
+
+	for (size_t i = start_offset; i < loop_size; ++i) {
+		if (bitmap->data[i] == 0)
+			continue;
+
+		offset = i * sizeof(*bitmap->data) + __builtin_ctz(bitmap->data[i]);
+		break;
+	}
+
+	return offset;
+}

kernel-src/sys/scheduler/metrics.c

@@ -1,8 +1,8 @@
 #include <kernel/scheduler.h>
+#include <kernel/timekeeper.h>
 #include <util.h>
 
-#define MAX_INTERACTIVITY 100
-#define SCALING_FACTOR (MAX_INTERACTIVITY / 2)
+#define SCALING_FACTOR (SCHED_MAX_INTERACTIVITY / 2)
 
 #define HISTORY_LIMIT_USEC 5000000
 
@@ -11,9 +11,9 @@ static void update_interactivity(thread_t *thread) {
 	time_t run_time = thread->metrics.run_time_avg_us;
 
 	if (sleep_time > run_time)
-		thread->metrics.interactivity_score = min((SCALING_FACTOR * run_time) / sleep_time, MAX_INTERACTIVITY);
+		thread->metrics.interactivity_score = min((SCALING_FACTOR * run_time) / sleep_time, SCHED_MAX_INTERACTIVITY);
 	else
-		thread->metrics.interactivity_score = min((SCALING_FACTOR * sleep_time) / run_time + SCALING_FACTOR, MAX_INTERACTIVITY);
+		thread->metrics.interactivity_score = min((SCALING_FACTOR * sleep_time) / run_time + SCALING_FACTOR, SCHED_MAX_INTERACTIVITY);
 
 	// TODO nice value impacting this 
 }

kernel-src/sys/scheduler/queue.c

@@ -0,0 +1,148 @@
+#include <kernel/scheduler.h>
+#include <arch/cpu.h>
+#include <logging.h>
+
+#define INTERACTIVITY_LIMIT 30
+
+// set = idle
+bitmap_t sched_idle_cpu_bitmap;
+spinlock_t sched_idle_cpu_bitmap_lock;
+
+static cpu_t *pick_cpu(thread_t *thread) {
+	if (thread->cputarget)
+		return thread->cputarget;
+
+	return current_cpu();
+}
+
+static void insert_in_run_queue(sched_run_queue_t *run_queue, thread_t *thread) {
+	int idx = min(thread->metrics.interactivity_score * SCHED_RUN_QUEUE_SIZE / SCHED_MAX_INTERACTIVITY, SCHED_RUN_QUEUE_SIZE - 1);
+
+	thread->prev = run_queue->queues[idx].ins;
+
+	if (thread->prev)
+		thread->prev->next = thread;
+	else
+		run_queue->queues[idx].run = thread;
+
+	thread->next = NULL;
+	run_queue->queues[idx].ins = thread;
+
+	bitmap_set(&run_queue->thread_bitmap, idx, 1);
+}
+
+static thread_t *pop_from_run_queue(sched_run_queue_t *run_queue) {
+	long idx = bitmap_find_first_set(&run_queue->thread_bitmap, 0);
+	if (idx == -1)
+		return NULL;
+
+
+	thread_t *thread = run_queue->queues[idx].run;
+	
+	run_queue->queues[idx].run = thread->next;
+
+	if (thread->next) {
+		thread->next->prev = NULL;
+	} else {
+		run_queue->queues[idx].ins = NULL;
+		bitmap_set(&run_queue->thread_bitmap, idx, 0);
+	}
+
+	return thread;
+}
+
+static void insert_in_calendar_queue(sched_calendar_queue_t *calendar_queue, thread_t *thread) {
+	int idx = (calendar_queue->ins + thread->metrics.interactivity_score - INTERACTIVITY_LIMIT) % SCHED_RUN_QUEUE_SIZE;
+
+	thread->next = calendar_queue->queues[idx];
+	calendar_queue->queues[idx] = thread;
+	++calendar_queue->thread_count;
+}
+
+static thread_t *pop_from_calendar_queue(sched_calendar_queue_t *calendar_queue) {
+	if (calendar_queue->thread_count == 0)
+		return NULL;
+
+	thread_t *thread;
+
+	do {
+		thread = calendar_queue->queues[calendar_queue->run];
+		if (thread)
+			calendar_queue->queues[calendar_queue->run] = thread->next;
+
+		calendar_queue->run = (calendar_queue->run + 1) % SCHED_RUN_QUEUE_SIZE;
+		if (calendar_queue->run == calendar_queue->ins)
+			calendar_queue->ins = (calendar_queue->ins + 1) % SCHED_RUN_QUEUE_SIZE;
+	} while (thread == NULL);
+
+	--calendar_queue->thread_count;
+
+	return thread;
+}
+
+// requires cpu queue to be locked
+thread_t *sched_select_next_thread(void) {
+	thread_t *thread = pop_from_run_queue(&current_cpu()->rt_queue);
+	if (thread)
+		goto found;
+
+	thread = pop_from_calendar_queue(&current_cpu()->ts_queue);
+	if (thread)
+		goto found;
+
+	thread = pop_from_run_queue(&current_cpu()->idle_queue);
+	if (thread == NULL)
+		return NULL;
+found:
+	thread->flags &= ~THREAD_FLAGS_QUEUED;
+	return thread;
+}
+
+// requires cpu queue to be locked
+void sched_insert_in_cpu_queue(cpu_t *cpu, thread_t *thread) {
+	__assert((thread->flags & THREAD_FLAGS_RUNNING) == 0);
+	thread->flags |= THREAD_FLAGS_QUEUED;
+
+	switch (thread->class) {
+		case THREAD_CLASS_TIMESHARE:
+			if (thread->metrics.interactivity_score > INTERACTIVITY_LIMIT) {
+				// thread is not interactive
+				insert_in_calendar_queue(&cpu->ts_queue, thread);
+				break;
+			}
+		case THREAD_CLASS_REAL_TIME:
+			insert_in_run_queue(&cpu->rt_queue, thread);
+			break;
+		case THREAD_CLASS_IDLE:
+			insert_in_run_queue(&cpu->idle_queue, thread);
+			
+			break;
+	}
+
+	spinlock_acquire(&sched_idle_cpu_bitmap_lock);
+
+	bitmap_set(&sched_idle_cpu_bitmap, cpu->internal_id, 0);
+
+	spinlock_release(&sched_idle_cpu_bitmap_lock);
+}
+
+void sched_queue(thread_t *thread) {
+	bool status = interrupt_set(false);
+	cpu_t *cpu = pick_cpu(thread);
+
+	spinlock_acquire(&cpu->sched_lock);
+
+	__assert((thread->flags & THREAD_FLAGS_QUEUED) == 0 && (thread->flags & THREAD_FLAGS_RUNNING) == 0);
+	sched_insert_in_cpu_queue(cpu, thread);
+	
+	spinlock_release_irq_restore(&cpu->sched_lock, status);
+}
+
+// ran every 10 ms per core
+void sched_calendar_tick(context_t *, dpcarg_t) {
+	spinlock_acquire(&current_cpu()->sched_lock);
+
+	current_cpu()->ts_queue.ins = (current_cpu()->ts_queue.ins + 1) % SCHED_RUN_QUEUE_SIZE;
+
+	spinlock_release(&current_cpu()->sched_lock);
+}

kernel-src/sys/scheduler/scheduler.c

@@ -18,65 +18,6 @@
 #define QUANTUM_US 100000
 #define SCHEDULER_STACK_SIZE PAGE_SIZE * 16
 
-#define RUNQUEUE_COUNT 64
-
-typedef struct {
-	thread_t *list;
-	thread_t *last;
-} rqueue_t;
-
-static rqueue_t runqueue[RUNQUEUE_COUNT];
-static uint64_t runqueuebitmap;
-static spinlock_t runqueuelock;
-
-static thread_t *getinrunqueue(rqueue_t *rq) {
-	thread_t *thread = rq->list;
-
-	while (thread) {
-		if (thread->cputarget == NULL || thread->cputarget == current_cpu())
-			break;
-
-		thread = thread->next;
-	}
-
-	if (thread) {
-		if (thread->prev)
-			thread->prev->next = thread->next;
-		else
-			rq->list = thread->next;
-
-		if (thread->next)
-			thread->next->prev = thread->prev;
-		else
-			rq->last = thread->prev;
-	}
-
-	return thread;
-}
-
-static thread_t *runqueuenext(int minprio) {
-	bool intstate = interrupt_set(false);
-
-	thread_t *thread = NULL;
-
-	if (runqueuebitmap == 0)
-		goto leave;
-
-	// TODO use the bitmap for this
-	for (int i = 0; i < RUNQUEUE_COUNT && i <= minprio && thread == NULL; ++i) {
-		thread = getinrunqueue(&runqueue[i]);
-		if (thread && runqueue[i].list == NULL)
-			runqueuebitmap &= ~((uint64_t)1 << i);
-	}
-
-	if (thread)
-		thread->flags &= ~THREAD_FLAGS_QUEUED;
-
-	leave:
-	interrupt_set(intstate);
-	return thread;
-}
-
 static __attribute__((noreturn)) void switch_thread(thread_t *thread) {
 	interrupt_set(false);
 	thread_t* current = current_thread();
@@ -90,8 +31,6 @@ static __attribute__((noreturn)) void switch_thread(thread_t *thread) {
 
 	current_cpu()->intstatus = ARCH_CONTEXT_INTSTATUS(&thread->context);
 	thread->cpu = current_cpu();
-	// XXX make the locking of the flags field better, candidate for a r/w lock
-	spinlock_acquire(&runqueuelock);
 	if (current)
 		current->flags &= ~THREAD_FLAGS_RUNNING;
 
@@ -102,7 +41,6 @@ static __attribute__((noreturn)) void switch_thread(thread_t *thread) {
 
 	thread->flags |= THREAD_FLAGS_RUNNING;
 	__assert((thread->flags & THREAD_FLAGS_QUEUED) == 0);
-	spinlock_release(&runqueuelock);
 
 	void *schedulerstack = current_cpu()->schedulerstack;
 	__assert(!((void *)thread->context.rsp < schedulerstack && (void *)thread->context.rsp >= (schedulerstack - SCHEDULER_STACK_SIZE)));
@@ -111,49 +49,18 @@ static __attribute__((noreturn)) void switch_thread(thread_t *thread) {
 	__builtin_unreachable();
 }
 
-static void runqueueinsert(thread_t *thread) {
-	__assert((thread->flags & THREAD_FLAGS_RUNNING) == 0);
-	thread->flags |= THREAD_FLAGS_QUEUED;
-
-	runqueuebitmap |= ((uint64_t)1 << thread->priority);
-
-	thread->prev = runqueue[thread->priority].last;
-	if (thread->prev)
-		thread->prev->next = thread;
-	else
-		runqueue[thread->priority].list = thread;
-
-	thread->next = NULL;
-	runqueue[thread->priority].last = thread;
-}
-
-void sched_queue(thread_t *thread) {
-	bool intstate = interrupt_set(false);
-	spinlock_acquire(&runqueuelock);
-
-	// maybe instead of an assert, a simple return would suffice as the thread would already be queued anyways
-	__assert((thread->flags & THREAD_FLAGS_QUEUED) == 0 && (thread->flags & THREAD_FLAGS_RUNNING) == 0);
-
-	runqueueinsert(thread);
-
-	spinlock_release(&runqueuelock);
-	interrupt_set(intstate);
-	// TODO yield if higher priority than current thread (or send another CPU an IPI)
-}
-
 // no metrics callback here as this will be called from things like threads stopping
 __attribute__((noreturn)) void sched_stop_current_thread() {
 	interrupt_set(false);
 
-	spinlock_acquire(&runqueuelock);
 	if (current_thread())
 		current_thread()->flags &= ~THREAD_FLAGS_RUNNING;
 
-	thread_t *next = runqueuenext(0x0fffffff);
-	if (next == NULL)
-		next = current_cpu()->idlethread;
+	spinlock_acquire(&current_cpu()->sched_lock);
+	thread_t *next = sched_select_next_thread();
+	spinlock_release(&current_cpu()->sched_lock);
 
-	spinlock_release(&runqueuelock);
+	__assert(next);
 
 	switch_thread(next);
 }
@@ -222,11 +129,8 @@ __attribute__((no_caller_saved_registers)) void sched_userspacecheck(context_t *
 static void yield(context_t *context, void *) {
 	thread_t *thread = current_thread();
 
-	spinlock_acquire(&runqueuelock);
-
 	bool sleeping = thread->flags & THREAD_FLAGS_SLEEP;
 
-	thread_t *next = runqueuenext(sleeping ? 0x0fffffff : thread->priority);
 	bool gotsignal = false;
 	for (int i = 1; i < NSIG && thread->proc; ++i) {
 		void *action = thread->proc->signals.actions[i].address;
@@ -245,32 +149,31 @@ static void yield(context_t *context, void *) {
 
 	if (sleeping && (thread->shouldexit || gotsignal) && (thread->flags & THREAD_FLAGS_INTERRUPTIBLE)) {
 		sleeping = false;
-
-		if (next)
-			runqueueinsert(next);
-		next = NULL;
-
 		thread->flags &= ~(THREAD_FLAGS_SLEEP | THREAD_FLAGS_INTERRUPTIBLE);
 		thread->wakeupreason = SCHED_WAKEUP_REASON_INTERRUPTED;
 		spinlock_release(&thread->sleeplock);
 	}
 
-	if (next || sleeping) {
+	spinlock_acquire(&current_cpu()->sched_lock);
+
+	thread->flags &= ~THREAD_FLAGS_RUNNING;
+
+	if (sleeping == false) {
+		sched_insert_in_cpu_queue(current_cpu(), thread);
+	}
+
+	thread_t *next = sched_select_next_thread();
+
+	thread->flags |= THREAD_FLAGS_RUNNING;
+
+	spinlock_release(&current_cpu()->sched_lock);
+
+	if (next != thread || sleeping) {
 		ARCH_CONTEXT_THREADSAVE(thread, context);
 
-		thread->flags &= ~THREAD_FLAGS_RUNNING;
-
-		if (sleeping == false)
-			runqueueinsert(thread);
-
-		if (next == NULL)
-			next = current_cpu()->idlethread;
-
-		spinlock_release(&runqueuelock);
 		switch_thread(next);
 	}
 
-	spinlock_release(&runqueuelock);
 	sched_thread_running_callback(current_thread());
 }
 
@@ -319,20 +222,16 @@ bool sched_wakeup(thread_t *thread, int reason) {
 // once a scheduler dpc gets run, the return context is set to this function using the scheduler stack
 static void dopreempt() {
 	// interrupts are disabled, the thread context is already saved
-	spinlock_acquire(&runqueuelock);
-
 	thread_t *current = current_thread();
-	thread_t *next = runqueuenext(current->priority);
-
 	current->flags &= ~THREAD_FLAGS_PREEMPTED;
-	if (next) {
-		current->flags &= ~THREAD_FLAGS_RUNNING;
-		runqueueinsert(current);
-	} else {
-		next = current;
-	}
 
-	spinlock_release(&runqueuelock);
+	spinlock_acquire(&current_cpu()->sched_lock);
+
+	current->flags &= ~THREAD_FLAGS_RUNNING;
+	sched_insert_in_cpu_queue(current_cpu(), current);
+	thread_t *next = sched_select_next_thread();
+
+	spinlock_release(&current_cpu()->sched_lock);
 
 	switch_thread(next);
 }
@@ -385,21 +284,18 @@ static void reschedule_yield(context_t *context, void *_cpu) {
 	thread_t *thread = current_thread();
 	cpu_t *cpu = _cpu;
 
-	spinlock_acquire(&runqueuelock);
-
-	thread_t *next = runqueuenext(0x0fffffff);
-
 	ARCH_CONTEXT_THREADSAVE(thread, context);
 
+	spinlock_acquire(&cpu->sched_lock);
 	thread->flags &= ~THREAD_FLAGS_RUNNING;
-	runqueueinsert(thread);
+	sched_insert_in_cpu_queue(cpu, thread);
+	spinlock_release(&cpu->sched_lock);
 
-	if (next == NULL)
-		next = current_cpu()->idlethread;
+	spinlock_acquire(&current_cpu()->sched_lock);
+	thread_t *next = sched_select_next_thread();
+	spinlock_release(&current_cpu()->sched_lock);
 
 	arch_smp_sendipi(cpu, cpu->reschedule_isr, ARCH_SMP_IPI_TARGET, false);
-
-	spinlock_release(&runqueuelock);
 	switch_thread(next);
 }
 
@@ -442,18 +338,32 @@ void sched_sleep_us(size_t us) {
 	sched_yield();
 }
 
+static void set_up_bitmaps(void) {
+	__assert(bitmap_init(&current_cpu()->rt_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0);
+	__assert(bitmap_init(&current_cpu()->idle_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0);
+}
+
+void sched_calendar_tick(context_t *, dpcarg_t);
+
 void sched_ap_entry() {
+	SPINLOCK_INIT(current_cpu()->sched_lock);
+
+	set_up_bitmaps();
+
 	current_cpu()->schedulerstack = vmm_map(NULL, SCHEDULER_STACK_SIZE, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
 	__assert(current_cpu()->schedulerstack);
 	current_cpu()->schedulerstack = (void *)((uintptr_t)current_cpu()->schedulerstack + SCHEDULER_STACK_SIZE);
 
 	current_cpu()->idlethread = sched_newthread(cpuidlethread, PAGE_SIZE * 4, 3, NULL, NULL);
 	__assert(current_cpu()->idlethread);
+	current_cpu()->idlethread->class = THREAD_CLASS_IDLE;
+	sched_queue(current_cpu()->idlethread);
 
 	current_cpu()->reschedule_isr = interrupt_allocate(reschedule_ipi, ARCH_EOI, IPL_MAX);
 	__assert(current_cpu()->reschedule_isr);
 
 	timer_insert(current_cpu()->timer, &current_cpu()->schedtimerentry, reschedule_timer_dpc, NULL, QUANTUM_US, true);
+	timer_insert(current_cpu()->timer, &current_cpu()->calendar_tick_timer_entry, sched_calendar_tick, NULL, 1000, true);
 	timer_resume(current_cpu()->timer);
 	sched_stop_current_thread();
 }
@@ -461,17 +371,25 @@ void sched_ap_entry() {
 void sched_init() {
 	proc_init();
 
+	SPINLOCK_INIT(current_cpu()->sched_lock);
+	SPINLOCK_INIT(sched_idle_cpu_bitmap_lock);
+	
+	set_up_bitmaps();
+
+	__assert(bitmap_init(&sched_idle_cpu_bitmap, arch_smp_get_cpu_count()) == 0);
+
 	current_cpu()->schedulerstack = vmm_map(NULL, SCHEDULER_STACK_SIZE, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
 	__assert(current_cpu()->schedulerstack);
 	current_cpu()->schedulerstack = (void *)((uintptr_t)current_cpu()->schedulerstack + SCHEDULER_STACK_SIZE);
 
-	SPINLOCK_INIT(runqueuelock);
-
 	current_cpu()->idlethread = sched_newthread(cpuidlethread, PAGE_SIZE * 4, 3, NULL, NULL);
 	__assert(current_cpu()->idlethread);
+	current_cpu()->idlethread->class = THREAD_CLASS_IDLE;
 	current_cpu()->thread = sched_newthread(NULL, PAGE_SIZE * 32, 0, NULL, NULL);
 	__assert(current_thread());
 
+	sched_queue(current_cpu()->idlethread);
+
 	current_cpu()->reschedule_isr = interrupt_allocate(reschedule_ipi, ARCH_EOI, IPL_MAX);
 	__assert(current_cpu()->reschedule_isr);
 
@@ -480,6 +398,7 @@ void sched_init() {
 	sched_thread_running_callback(current_thread());
 
 	timer_insert(current_cpu()->timer, &current_cpu()->schedtimerentry, reschedule_timer_dpc, NULL, QUANTUM_US, true);
+	timer_insert(current_cpu()->timer, &current_cpu()->calendar_tick_timer_entry, sched_calendar_tick, NULL, 10000, true);
 	// XXX move this resume to a more appropriate place
 	timer_resume(current_cpu()->timer);
 }

kernel-src/sys/syscalls/fork.c

@@ -5,6 +5,7 @@
 #include <kernel/interrupt.h>
 #include <kernel/jobctl.h>
 #include <kernel/interrupt.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_fork(context_t *ctx) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/getresgid.c

@@ -1,6 +1,7 @@
 #include <kernel/abi.h>
 #include <kernel/syscalls.h>
 #include <kernel/cred.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_getresgid(context_t *, gid_t *gidp, gid_t *egidp, gid_t *sgidp) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/getresuid.c

@@ -1,6 +1,7 @@
 #include <kernel/abi.h>
 #include <kernel/syscalls.h>
 #include <kernel/cred.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_getresuid(context_t *, uid_t *uidp, uid_t *euidp, uid_t *suidp) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/gettid.c

@@ -1,5 +1,6 @@
 #include <kernel/syscalls.h>
 #include <kernel/scheduler.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_gettid(context_t *) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/ioctl.c

@@ -1,6 +1,7 @@
 #include <kernel/syscalls.h>
 #include <kernel/file.h>
 #include <kernel/vfs.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_ioctl(context_t *, int fd, unsigned long request, void *arg) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/linkat.c

@@ -1,6 +1,6 @@
 #include <kernel/syscalls.h>
 #include <kernel/alloc.h>
-
+#include <arch/cpu.h>
 
 syscallret_t syscall_linkat(context_t *, int olddirfd, char *uoldpath, int newdirfd, char *unewpath, int flags, int type) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/newthread.c

@@ -2,6 +2,7 @@
 #include <kernel/scheduler.h>
 #include <kernel/cmdline.h>
 #include <kernel/alloc.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_newthread(context_t *, void *entry, void *stack) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/seek.c

@@ -1,6 +1,7 @@
 #include <kernel/syscalls.h>
 #include <kernel/file.h>
 #include <kernel/vfs.h>
+#include <arch/cpu.h>
 #include <errno.h>
 
 #define SEEK_SET 0

kernel-src/sys/syscalls/setresgid.c

@@ -1,6 +1,7 @@
 #include <kernel/abi.h>
 #include <kernel/syscalls.h>
 #include <kernel/cred.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_setresgid(context_t *, gid_t gid, gid_t egid, gid_t sgid) {
 	syscallret_t ret;

kernel-src/sys/syscalls/setresuid.c

@@ -1,6 +1,7 @@
 #include <kernel/abi.h>
 #include <kernel/syscalls.h>
 #include <kernel/cred.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_setresuid(context_t *, uid_t uid, uid_t euid, uid_t suid) {
 	syscallret_t ret;

kernel-src/sys/syscalls/truncate.c

@@ -1,5 +1,6 @@
 #include <kernel/syscalls.h>
 #include <mutex.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_ftruncate(context_t *, int fd, size_t size) {
 	syscallret_t ret = {

kernel-src/sys/syscalls/write.c

@@ -2,6 +2,7 @@
 #include <kernel/vfs.h>
 #include <kernel/file.h>
 #include <errno.h>
+#include <arch/cpu.h>
 
 syscallret_t syscall_write(context_t *context, int fd, void *buffer, size_t size) {
 	syscallret_t ret = {

kernel-src/sys/thread.c

@@ -45,6 +45,7 @@ thread_t *sched_newthread(void *ip, size_t kstacksize, int priority, proc_t *pro
 	EVENT_INITHEADER(&thread->signals.waitpendingevent);
 	memset(&thread->metrics, 0, sizeof(thread->metrics));
 	thread->metrics.sleep_time_avg_us = 1; // to prevent a division by 0 when the thread first gets scheduled
+	thread->class = THREAD_CLASS_TIMESHARE;
 
 	return thread;
 }