From 136c86354c9b55a34d4d1b906fd4fde52fca0d8b Mon Sep 17 00:00:00 2001 From: Matheus Date: Wed, 20 Aug 2025 22:00:25 -0300 Subject: [PATCH] scheduler: continue rework --- kernel-src/arch/x86-64/apic.c | 6 +- kernel-src/arch/x86-64/cpu.c | 2 +- kernel-src/arch/x86-64/smp.c | 26 +++- kernel-src/include/bitmap.h | 16 +++ kernel-src/include/kernel/proc.h | 1 - kernel-src/include/kernel/scheduler.h | 25 ++++ kernel-src/include/kernel/thread.h | 5 + kernel-src/include/x86-64/arch/cpu.h | 11 +- kernel-src/include/x86-64/arch/smp.h | 5 +- kernel-src/lib/bitmap.c | 40 ++++++ kernel-src/sys/scheduler/metrics.c | 8 +- kernel-src/sys/scheduler/queue.c | 148 +++++++++++++++++++ kernel-src/sys/scheduler/scheduler.c | 195 ++++++++------------------ kernel-src/sys/syscalls/fork.c | 1 + kernel-src/sys/syscalls/getresgid.c | 1 + kernel-src/sys/syscalls/getresuid.c | 1 + kernel-src/sys/syscalls/gettid.c | 1 + kernel-src/sys/syscalls/ioctl.c | 1 + kernel-src/sys/syscalls/linkat.c | 2 +- kernel-src/sys/syscalls/newthread.c | 1 + kernel-src/sys/syscalls/seek.c | 1 + kernel-src/sys/syscalls/setresgid.c | 1 + kernel-src/sys/syscalls/setresuid.c | 1 + kernel-src/sys/syscalls/truncate.c | 1 + kernel-src/sys/syscalls/write.c | 1 + kernel-src/sys/thread.c | 1 + 26 files changed, 348 insertions(+), 154 deletions(-) create mode 100644 kernel-src/include/bitmap.h create mode 100644 kernel-src/lib/bitmap.c create mode 100644 kernel-src/sys/scheduler/queue.c diff --git a/kernel-src/arch/x86-64/apic.c b/kernel-src/arch/x86-64/apic.c index 1e941e97..cae47ffa 100644 --- a/kernel-src/arch/x86-64/apic.c +++ b/kernel-src/arch/x86-64/apic.c @@ -168,13 +168,13 @@ void arch_apic_initap() { interrupt_register(0xff, spurious, NULL, IPL_IGNORE); writelapic(APIC_REG_SPURIOUS, 0x1FF); - current_cpu()->id = readlapic(APIC_REG_ID) >> 24; + current_cpu()->hardware_id = readlapic(APIC_REG_ID) >> 24; // get the acpi id for the local nmi sources for (size_t i = 0; i < lapic_count; ++i) { struct acpi_madt_lapic *current = getentry(ACPI_MADT_ENTRY_TYPE_LAPIC, i); - if (current_cpu()->id == current->id) { + if (current_cpu()->hardware_id == current->id) { current_cpu()->acpiid = current->uid; break; } @@ -233,7 +233,7 @@ void arch_apic_timerinit() { writelapic(APIC_TIMER_INITIAL_COUNT, 0); - printf("cpu%lu: local apic timer calibrated at %lu ticks per us. ISR vector %lu\n", current_cpu()->id, ticksperus, vec); + printf("cpu%lu: local apic timer calibrated at %lu ticks per us. ISR vector %lu\n", current_cpu()->internal_id, ticksperus, vec); writelapic(APIC_LVT_TIMER, vec); diff --git a/kernel-src/arch/x86-64/cpu.c b/kernel-src/arch/x86-64/cpu.c index 687b5d5a..8688d71e 100644 --- a/kernel-src/arch/x86-64/cpu.c +++ b/kernel-src/arch/x86-64/cpu.c @@ -211,7 +211,7 @@ void arch_cpu_init() { char str_buf[128]; size_t done = 0; - done += snprintf(str_buf, 128, "cpu%d: %s@", current_cpu()->id, current_cpu()->vendor); + done += snprintf(str_buf, 128, "cpu%d: %s@", current_cpu()->internal_id, current_cpu()->vendor); for (int i = 0; i < topology_depth; ++i) done += snprintf(str_buf + done, 128 - done, "%d%c", topology_ids[i], i == topology_depth - 1 ? 0 : ':'); str_buf[done] = '\n'; diff --git a/kernel-src/arch/x86-64/smp.c b/kernel-src/arch/x86-64/smp.c index abe840b9..bfcde305 100644 --- a/kernel-src/arch/x86-64/smp.c +++ b/kernel-src/arch/x86-64/smp.c @@ -22,9 +22,13 @@ static void cpuwakeuphalt(struct limine_smp_info *info) { for (;;) CPU_HALT(); } +static long next_id = 1; + static void cpuwakeup(struct limine_smp_info *info) { cpu_set((cpu_t *)info->extra_argument); + current_cpu()->internal_id = __atomic_fetch_add(&next_id, 1, __ATOMIC_SEQ_CST); + arch_gdt_reload(); arch_idt_reload(); interrupt_register(0xfd, (void *)cpuwakeuphalt, NULL, IPL_IGNORE); @@ -46,16 +50,32 @@ static void cpuwakeup(struct limine_smp_info *info) { } void arch_smp_sendipi(cpu_t *targcpu, isr_t *isr, int target, bool nmi) { - arch_apic_sendipi(targcpu ? targcpu->id : 0, INTERRUPT_IDTOVECTOR(isr->id), target, nmi ? APIC_MODE_NMI : 0, 0); + arch_apic_sendipi(targcpu ? targcpu->hardware_id : 0, INTERRUPT_IDTOVECTOR(isr->id), target, nmi ? APIC_MODE_NMI : 0, 0); } -void arch_smp_haltallothers() { +void arch_smp_haltallothers(void) { arch_smp_sendipi(NULL, ¤t_cpu()->isr[0xfd], ARCH_SMP_IPI_OTHERCPUS, true); } +size_t arch_smp_get_cpu_count(void) { + if (cmdline_get("nosmp")) + return 1; + + struct limine_smp_response *response = smprequest.response; + if (response == NULL) + return 1; + + return response->cpu_count; +} + cpu_t **smp_cpus; -void arch_smp_wakeup() { +cpu_t *arch_smp_get_cpu_by_internal_id(long id) { + __assert(smp_cpus && id < arch_smp_get_cpu_count()); + return smp_cpus[id]; +} + +void arch_smp_wakeup(void) { struct limine_smp_response *response = smprequest.response; if (response == NULL) { // TODO try to detect the other processors manually diff --git a/kernel-src/include/bitmap.h b/kernel-src/include/bitmap.h new file mode 100644 index 00000000..9af8d78e --- /dev/null +++ b/kernel-src/include/bitmap.h @@ -0,0 +1,16 @@ +#ifndef _BITMAP_H +#define _BITMAP_H + +#include + +typedef struct { + long *data; + size_t size; +} bitmap_t; + +int bitmap_init(bitmap_t *bitmap, size_t size); +void bitmap_set(bitmap_t *bitmap, long idx, int v); +long bitmap_find_first_set(bitmap_t *bitmap, long start_offset); + + +#endif diff --git a/kernel-src/include/kernel/proc.h b/kernel-src/include/kernel/proc.h index a31a6f0a..b9df6768 100644 --- a/kernel-src/include/kernel/proc.h +++ b/kernel-src/include/kernel/proc.h @@ -79,7 +79,6 @@ typedef struct proc_t { } timer; } proc_t; -#include #define UMASK(mode) ((mode) & ~current_thread()->proc->umask) #define PROC_HOLD(v) __atomic_add_fetch(&(v)->refcount, 1, __ATOMIC_SEQ_CST) diff --git a/kernel-src/include/kernel/scheduler.h b/kernel-src/include/kernel/scheduler.h index 821e9707..20167500 100644 --- a/kernel-src/include/kernel/scheduler.h +++ b/kernel-src/include/kernel/scheduler.h @@ -3,6 +3,7 @@ #include #include +#include #define SCHED_WAKEUP_REASON_NORMAL 0 #define SCHED_WAKEUP_REASON_INTERRUPTED -1 @@ -10,6 +11,27 @@ #define STACK_TOP (void *)0x0000800000000000 #define INTERP_BASE (void *)0x00000beef0000000 +#define SCHED_RUN_QUEUE_SIZE 32 +#define SCHED_MAX_INTERACTIVITY 100 + +typedef struct { + struct { + thread_t *ins; + thread_t *run; + } queues[SCHED_RUN_QUEUE_SIZE]; + bitmap_t thread_bitmap; +} sched_run_queue_t; + +typedef struct { + thread_t *queues[SCHED_RUN_QUEUE_SIZE]; + int ins; + int run; + size_t thread_count; +} sched_calendar_queue_t; + +extern bitmap_t sched_idle_cpu_bitmap; +extern spinlock_t sched_idle_cpu_bitmap_lock; + void sched_init(); void sched_ap_entry(); @@ -27,4 +49,7 @@ void sched_thread_running_callback(thread_t *thread); void sched_thread_stopping_callback(thread_t *thread, bool sleeping); void sched_thread_wakeup_callback(thread_t *thread); +thread_t *sched_select_next_thread(void); +void sched_insert_in_cpu_queue(struct cpu_t *cpu, thread_t *thread); + #endif diff --git a/kernel-src/include/kernel/thread.h b/kernel-src/include/kernel/thread.h index 06cae8e1..131596a4 100644 --- a/kernel-src/include/kernel/thread.h +++ b/kernel-src/include/kernel/thread.h @@ -16,6 +16,10 @@ struct proc_t; +#define THREAD_CLASS_TIMESHARE 0 +#define THREAD_CLASS_REAL_TIME 1 +#define THREAD_CLASS_IDLE 2 + typedef struct thread_t { void *kernelstacktop; struct thread_t *next; @@ -40,6 +44,7 @@ typedef struct thread_t { bool shouldexit; void *kernelarg; context_t *usercopyctx; + int class; struct { spinlock_t lock; eventheader_t waitpendingevent; diff --git a/kernel-src/include/x86-64/arch/cpu.h b/kernel-src/include/x86-64/arch/cpu.h index 50c576a8..57adf5e8 100644 --- a/kernel-src/include/x86-64/arch/cpu.h +++ b/kernel-src/include/x86-64/arch/cpu.h @@ -24,7 +24,8 @@ typedef struct cpu_t { vmmcontext_t *vmmctx; // expected to be here by other code - long id; // expected to be here by other code + long hardware_id; // expected to be here by other code + long internal_id; timekeeper_source_t *timekeeper_source; timekeeper_source_info_t *timekeeper_source_info; @@ -49,6 +50,14 @@ typedef struct cpu_t { dpc_t reschedule_dpc; isr_t *reschedule_isr; + sched_run_queue_t rt_queue; + sched_calendar_queue_t ts_queue; + sched_run_queue_t idle_queue; + spinlock_t sched_lock; + + long calendar_ticks; + timerentry_t calendar_tick_timer_entry; + // architecture specific, does not need to be exposed uint64_t gdt[7]; diff --git a/kernel-src/include/x86-64/arch/smp.h b/kernel-src/include/x86-64/arch/smp.h index 9b149e75..357a7665 100644 --- a/kernel-src/include/x86-64/arch/smp.h +++ b/kernel-src/include/x86-64/arch/smp.h @@ -11,8 +11,9 @@ extern size_t arch_smp_cpusawake; extern cpu_t **smp_cpus; -void arch_smp_wakeup(); +void arch_smp_wakeup(void); void arch_smp_sendipi(cpu_t *targcpu, isr_t *isr, int target, bool nmi); -void arch_smp_haltallothers(); +void arch_smp_haltallothers(void); +size_t arch_smp_get_cpu_count(void); #endif diff --git a/kernel-src/lib/bitmap.c b/kernel-src/lib/bitmap.c new file mode 100644 index 00000000..58455f61 --- /dev/null +++ b/kernel-src/lib/bitmap.c @@ -0,0 +1,40 @@ +#include +#include +#include +#include +#include + +int bitmap_init(bitmap_t *bitmap, size_t size) { + bitmap->data = alloc(size); + bitmap->size = size; + + return bitmap->data ? 0 : ENOMEM; +} + +void bitmap_set(bitmap_t *bitmap, long idx, int v) { + __assert(bitmap->size > idx); + + long bit = 1llu << (idx % sizeof(*bitmap->data)); + + if (v) + bitmap->data[idx / sizeof(*bitmap->data)] |= bit; + else + bitmap->data[idx / sizeof(*bitmap->data)] &= ~bit; +} + +// TODO fix start offset search +long bitmap_find_first_set(bitmap_t *bitmap, long start_offset) { + size_t loop_size = ROUND_UP(bitmap->size, sizeof(*bitmap->data)) / sizeof(*bitmap->data); + + long offset = -1; + + for (size_t i = start_offset; i < loop_size; ++i) { + if (bitmap->data[i] == 0) + continue; + + offset = i * sizeof(*bitmap->data) + __builtin_ctz(bitmap->data[i]); + break; + } + + return offset; +} diff --git a/kernel-src/sys/scheduler/metrics.c b/kernel-src/sys/scheduler/metrics.c index 93efaa82..11434b92 100644 --- a/kernel-src/sys/scheduler/metrics.c +++ b/kernel-src/sys/scheduler/metrics.c @@ -1,8 +1,8 @@ #include +#include #include -#define MAX_INTERACTIVITY 100 -#define SCALING_FACTOR (MAX_INTERACTIVITY / 2) +#define SCALING_FACTOR (SCHED_MAX_INTERACTIVITY / 2) #define HISTORY_LIMIT_USEC 5000000 @@ -11,9 +11,9 @@ static void update_interactivity(thread_t *thread) { time_t run_time = thread->metrics.run_time_avg_us; if (sleep_time > run_time) - thread->metrics.interactivity_score = min((SCALING_FACTOR * run_time) / sleep_time, MAX_INTERACTIVITY); + thread->metrics.interactivity_score = min((SCALING_FACTOR * run_time) / sleep_time, SCHED_MAX_INTERACTIVITY); else - thread->metrics.interactivity_score = min((SCALING_FACTOR * sleep_time) / run_time + SCALING_FACTOR, MAX_INTERACTIVITY); + thread->metrics.interactivity_score = min((SCALING_FACTOR * sleep_time) / run_time + SCALING_FACTOR, SCHED_MAX_INTERACTIVITY); // TODO nice value impacting this } diff --git a/kernel-src/sys/scheduler/queue.c b/kernel-src/sys/scheduler/queue.c new file mode 100644 index 00000000..ee53dd8a --- /dev/null +++ b/kernel-src/sys/scheduler/queue.c @@ -0,0 +1,148 @@ +#include +#include +#include + +#define INTERACTIVITY_LIMIT 30 + +// set = idle +bitmap_t sched_idle_cpu_bitmap; +spinlock_t sched_idle_cpu_bitmap_lock; + +static cpu_t *pick_cpu(thread_t *thread) { + if (thread->cputarget) + return thread->cputarget; + + return current_cpu(); +} + +static void insert_in_run_queue(sched_run_queue_t *run_queue, thread_t *thread) { + int idx = min(thread->metrics.interactivity_score * SCHED_RUN_QUEUE_SIZE / SCHED_MAX_INTERACTIVITY, SCHED_RUN_QUEUE_SIZE - 1); + + thread->prev = run_queue->queues[idx].ins; + + if (thread->prev) + thread->prev->next = thread; + else + run_queue->queues[idx].run = thread; + + thread->next = NULL; + run_queue->queues[idx].ins = thread; + + bitmap_set(&run_queue->thread_bitmap, idx, 1); +} + +static thread_t *pop_from_run_queue(sched_run_queue_t *run_queue) { + long idx = bitmap_find_first_set(&run_queue->thread_bitmap, 0); + if (idx == -1) + return NULL; + + + thread_t *thread = run_queue->queues[idx].run; + + run_queue->queues[idx].run = thread->next; + + if (thread->next) { + thread->next->prev = NULL; + } else { + run_queue->queues[idx].ins = NULL; + bitmap_set(&run_queue->thread_bitmap, idx, 0); + } + + return thread; +} + +static void insert_in_calendar_queue(sched_calendar_queue_t *calendar_queue, thread_t *thread) { + int idx = (calendar_queue->ins + thread->metrics.interactivity_score - INTERACTIVITY_LIMIT) % SCHED_RUN_QUEUE_SIZE; + + thread->next = calendar_queue->queues[idx]; + calendar_queue->queues[idx] = thread; + ++calendar_queue->thread_count; +} + +static thread_t *pop_from_calendar_queue(sched_calendar_queue_t *calendar_queue) { + if (calendar_queue->thread_count == 0) + return NULL; + + thread_t *thread; + + do { + thread = calendar_queue->queues[calendar_queue->run]; + if (thread) + calendar_queue->queues[calendar_queue->run] = thread->next; + + calendar_queue->run = (calendar_queue->run + 1) % SCHED_RUN_QUEUE_SIZE; + if (calendar_queue->run == calendar_queue->ins) + calendar_queue->ins = (calendar_queue->ins + 1) % SCHED_RUN_QUEUE_SIZE; + } while (thread == NULL); + + --calendar_queue->thread_count; + + return thread; +} + +// requires cpu queue to be locked +thread_t *sched_select_next_thread(void) { + thread_t *thread = pop_from_run_queue(¤t_cpu()->rt_queue); + if (thread) + goto found; + + thread = pop_from_calendar_queue(¤t_cpu()->ts_queue); + if (thread) + goto found; + + thread = pop_from_run_queue(¤t_cpu()->idle_queue); + if (thread == NULL) + return NULL; +found: + thread->flags &= ~THREAD_FLAGS_QUEUED; + return thread; +} + +// requires cpu queue to be locked +void sched_insert_in_cpu_queue(cpu_t *cpu, thread_t *thread) { + __assert((thread->flags & THREAD_FLAGS_RUNNING) == 0); + thread->flags |= THREAD_FLAGS_QUEUED; + + switch (thread->class) { + case THREAD_CLASS_TIMESHARE: + if (thread->metrics.interactivity_score > INTERACTIVITY_LIMIT) { + // thread is not interactive + insert_in_calendar_queue(&cpu->ts_queue, thread); + break; + } + case THREAD_CLASS_REAL_TIME: + insert_in_run_queue(&cpu->rt_queue, thread); + break; + case THREAD_CLASS_IDLE: + insert_in_run_queue(&cpu->idle_queue, thread); + + break; + } + + spinlock_acquire(&sched_idle_cpu_bitmap_lock); + + bitmap_set(&sched_idle_cpu_bitmap, cpu->internal_id, 0); + + spinlock_release(&sched_idle_cpu_bitmap_lock); +} + +void sched_queue(thread_t *thread) { + bool status = interrupt_set(false); + cpu_t *cpu = pick_cpu(thread); + + spinlock_acquire(&cpu->sched_lock); + + __assert((thread->flags & THREAD_FLAGS_QUEUED) == 0 && (thread->flags & THREAD_FLAGS_RUNNING) == 0); + sched_insert_in_cpu_queue(cpu, thread); + + spinlock_release_irq_restore(&cpu->sched_lock, status); +} + +// ran every 10 ms per core +void sched_calendar_tick(context_t *, dpcarg_t) { + spinlock_acquire(¤t_cpu()->sched_lock); + + current_cpu()->ts_queue.ins = (current_cpu()->ts_queue.ins + 1) % SCHED_RUN_QUEUE_SIZE; + + spinlock_release(¤t_cpu()->sched_lock); +} diff --git a/kernel-src/sys/scheduler/scheduler.c b/kernel-src/sys/scheduler/scheduler.c index f75b3548..b7da9772 100644 --- a/kernel-src/sys/scheduler/scheduler.c +++ b/kernel-src/sys/scheduler/scheduler.c @@ -18,65 +18,6 @@ #define QUANTUM_US 100000 #define SCHEDULER_STACK_SIZE PAGE_SIZE * 16 -#define RUNQUEUE_COUNT 64 - -typedef struct { - thread_t *list; - thread_t *last; -} rqueue_t; - -static rqueue_t runqueue[RUNQUEUE_COUNT]; -static uint64_t runqueuebitmap; -static spinlock_t runqueuelock; - -static thread_t *getinrunqueue(rqueue_t *rq) { - thread_t *thread = rq->list; - - while (thread) { - if (thread->cputarget == NULL || thread->cputarget == current_cpu()) - break; - - thread = thread->next; - } - - if (thread) { - if (thread->prev) - thread->prev->next = thread->next; - else - rq->list = thread->next; - - if (thread->next) - thread->next->prev = thread->prev; - else - rq->last = thread->prev; - } - - return thread; -} - -static thread_t *runqueuenext(int minprio) { - bool intstate = interrupt_set(false); - - thread_t *thread = NULL; - - if (runqueuebitmap == 0) - goto leave; - - // TODO use the bitmap for this - for (int i = 0; i < RUNQUEUE_COUNT && i <= minprio && thread == NULL; ++i) { - thread = getinrunqueue(&runqueue[i]); - if (thread && runqueue[i].list == NULL) - runqueuebitmap &= ~((uint64_t)1 << i); - } - - if (thread) - thread->flags &= ~THREAD_FLAGS_QUEUED; - - leave: - interrupt_set(intstate); - return thread; -} - static __attribute__((noreturn)) void switch_thread(thread_t *thread) { interrupt_set(false); thread_t* current = current_thread(); @@ -90,8 +31,6 @@ static __attribute__((noreturn)) void switch_thread(thread_t *thread) { current_cpu()->intstatus = ARCH_CONTEXT_INTSTATUS(&thread->context); thread->cpu = current_cpu(); - // XXX make the locking of the flags field better, candidate for a r/w lock - spinlock_acquire(&runqueuelock); if (current) current->flags &= ~THREAD_FLAGS_RUNNING; @@ -102,7 +41,6 @@ static __attribute__((noreturn)) void switch_thread(thread_t *thread) { thread->flags |= THREAD_FLAGS_RUNNING; __assert((thread->flags & THREAD_FLAGS_QUEUED) == 0); - spinlock_release(&runqueuelock); void *schedulerstack = current_cpu()->schedulerstack; __assert(!((void *)thread->context.rsp < schedulerstack && (void *)thread->context.rsp >= (schedulerstack - SCHEDULER_STACK_SIZE))); @@ -111,49 +49,18 @@ static __attribute__((noreturn)) void switch_thread(thread_t *thread) { __builtin_unreachable(); } -static void runqueueinsert(thread_t *thread) { - __assert((thread->flags & THREAD_FLAGS_RUNNING) == 0); - thread->flags |= THREAD_FLAGS_QUEUED; - - runqueuebitmap |= ((uint64_t)1 << thread->priority); - - thread->prev = runqueue[thread->priority].last; - if (thread->prev) - thread->prev->next = thread; - else - runqueue[thread->priority].list = thread; - - thread->next = NULL; - runqueue[thread->priority].last = thread; -} - -void sched_queue(thread_t *thread) { - bool intstate = interrupt_set(false); - spinlock_acquire(&runqueuelock); - - // maybe instead of an assert, a simple return would suffice as the thread would already be queued anyways - __assert((thread->flags & THREAD_FLAGS_QUEUED) == 0 && (thread->flags & THREAD_FLAGS_RUNNING) == 0); - - runqueueinsert(thread); - - spinlock_release(&runqueuelock); - interrupt_set(intstate); - // TODO yield if higher priority than current thread (or send another CPU an IPI) -} - // no metrics callback here as this will be called from things like threads stopping __attribute__((noreturn)) void sched_stop_current_thread() { interrupt_set(false); - spinlock_acquire(&runqueuelock); if (current_thread()) current_thread()->flags &= ~THREAD_FLAGS_RUNNING; - thread_t *next = runqueuenext(0x0fffffff); - if (next == NULL) - next = current_cpu()->idlethread; + spinlock_acquire(¤t_cpu()->sched_lock); + thread_t *next = sched_select_next_thread(); + spinlock_release(¤t_cpu()->sched_lock); - spinlock_release(&runqueuelock); + __assert(next); switch_thread(next); } @@ -222,11 +129,8 @@ __attribute__((no_caller_saved_registers)) void sched_userspacecheck(context_t * static void yield(context_t *context, void *) { thread_t *thread = current_thread(); - spinlock_acquire(&runqueuelock); - bool sleeping = thread->flags & THREAD_FLAGS_SLEEP; - thread_t *next = runqueuenext(sleeping ? 0x0fffffff : thread->priority); bool gotsignal = false; for (int i = 1; i < NSIG && thread->proc; ++i) { void *action = thread->proc->signals.actions[i].address; @@ -245,32 +149,31 @@ static void yield(context_t *context, void *) { if (sleeping && (thread->shouldexit || gotsignal) && (thread->flags & THREAD_FLAGS_INTERRUPTIBLE)) { sleeping = false; - - if (next) - runqueueinsert(next); - next = NULL; - thread->flags &= ~(THREAD_FLAGS_SLEEP | THREAD_FLAGS_INTERRUPTIBLE); thread->wakeupreason = SCHED_WAKEUP_REASON_INTERRUPTED; spinlock_release(&thread->sleeplock); } - if (next || sleeping) { + spinlock_acquire(¤t_cpu()->sched_lock); + + thread->flags &= ~THREAD_FLAGS_RUNNING; + + if (sleeping == false) { + sched_insert_in_cpu_queue(current_cpu(), thread); + } + + thread_t *next = sched_select_next_thread(); + + thread->flags |= THREAD_FLAGS_RUNNING; + + spinlock_release(¤t_cpu()->sched_lock); + + if (next != thread || sleeping) { ARCH_CONTEXT_THREADSAVE(thread, context); - thread->flags &= ~THREAD_FLAGS_RUNNING; - - if (sleeping == false) - runqueueinsert(thread); - - if (next == NULL) - next = current_cpu()->idlethread; - - spinlock_release(&runqueuelock); switch_thread(next); } - spinlock_release(&runqueuelock); sched_thread_running_callback(current_thread()); } @@ -319,20 +222,16 @@ bool sched_wakeup(thread_t *thread, int reason) { // once a scheduler dpc gets run, the return context is set to this function using the scheduler stack static void dopreempt() { // interrupts are disabled, the thread context is already saved - spinlock_acquire(&runqueuelock); - thread_t *current = current_thread(); - thread_t *next = runqueuenext(current->priority); - current->flags &= ~THREAD_FLAGS_PREEMPTED; - if (next) { - current->flags &= ~THREAD_FLAGS_RUNNING; - runqueueinsert(current); - } else { - next = current; - } - spinlock_release(&runqueuelock); + spinlock_acquire(¤t_cpu()->sched_lock); + + current->flags &= ~THREAD_FLAGS_RUNNING; + sched_insert_in_cpu_queue(current_cpu(), current); + thread_t *next = sched_select_next_thread(); + + spinlock_release(¤t_cpu()->sched_lock); switch_thread(next); } @@ -385,21 +284,18 @@ static void reschedule_yield(context_t *context, void *_cpu) { thread_t *thread = current_thread(); cpu_t *cpu = _cpu; - spinlock_acquire(&runqueuelock); - - thread_t *next = runqueuenext(0x0fffffff); - ARCH_CONTEXT_THREADSAVE(thread, context); + spinlock_acquire(&cpu->sched_lock); thread->flags &= ~THREAD_FLAGS_RUNNING; - runqueueinsert(thread); + sched_insert_in_cpu_queue(cpu, thread); + spinlock_release(&cpu->sched_lock); - if (next == NULL) - next = current_cpu()->idlethread; + spinlock_acquire(¤t_cpu()->sched_lock); + thread_t *next = sched_select_next_thread(); + spinlock_release(¤t_cpu()->sched_lock); arch_smp_sendipi(cpu, cpu->reschedule_isr, ARCH_SMP_IPI_TARGET, false); - - spinlock_release(&runqueuelock); switch_thread(next); } @@ -442,18 +338,32 @@ void sched_sleep_us(size_t us) { sched_yield(); } +static void set_up_bitmaps(void) { + __assert(bitmap_init(¤t_cpu()->rt_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0); + __assert(bitmap_init(¤t_cpu()->idle_queue.thread_bitmap, SCHED_RUN_QUEUE_SIZE) == 0); +} + +void sched_calendar_tick(context_t *, dpcarg_t); + void sched_ap_entry() { + SPINLOCK_INIT(current_cpu()->sched_lock); + + set_up_bitmaps(); + current_cpu()->schedulerstack = vmm_map(NULL, SCHEDULER_STACK_SIZE, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); __assert(current_cpu()->schedulerstack); current_cpu()->schedulerstack = (void *)((uintptr_t)current_cpu()->schedulerstack + SCHEDULER_STACK_SIZE); current_cpu()->idlethread = sched_newthread(cpuidlethread, PAGE_SIZE * 4, 3, NULL, NULL); __assert(current_cpu()->idlethread); + current_cpu()->idlethread->class = THREAD_CLASS_IDLE; + sched_queue(current_cpu()->idlethread); current_cpu()->reschedule_isr = interrupt_allocate(reschedule_ipi, ARCH_EOI, IPL_MAX); __assert(current_cpu()->reschedule_isr); timer_insert(current_cpu()->timer, ¤t_cpu()->schedtimerentry, reschedule_timer_dpc, NULL, QUANTUM_US, true); + timer_insert(current_cpu()->timer, ¤t_cpu()->calendar_tick_timer_entry, sched_calendar_tick, NULL, 1000, true); timer_resume(current_cpu()->timer); sched_stop_current_thread(); } @@ -461,17 +371,25 @@ void sched_ap_entry() { void sched_init() { proc_init(); + SPINLOCK_INIT(current_cpu()->sched_lock); + SPINLOCK_INIT(sched_idle_cpu_bitmap_lock); + + set_up_bitmaps(); + + __assert(bitmap_init(&sched_idle_cpu_bitmap, arch_smp_get_cpu_count()) == 0); + current_cpu()->schedulerstack = vmm_map(NULL, SCHEDULER_STACK_SIZE, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL); __assert(current_cpu()->schedulerstack); current_cpu()->schedulerstack = (void *)((uintptr_t)current_cpu()->schedulerstack + SCHEDULER_STACK_SIZE); - SPINLOCK_INIT(runqueuelock); - current_cpu()->idlethread = sched_newthread(cpuidlethread, PAGE_SIZE * 4, 3, NULL, NULL); __assert(current_cpu()->idlethread); + current_cpu()->idlethread->class = THREAD_CLASS_IDLE; current_cpu()->thread = sched_newthread(NULL, PAGE_SIZE * 32, 0, NULL, NULL); __assert(current_thread()); + sched_queue(current_cpu()->idlethread); + current_cpu()->reschedule_isr = interrupt_allocate(reschedule_ipi, ARCH_EOI, IPL_MAX); __assert(current_cpu()->reschedule_isr); @@ -480,6 +398,7 @@ void sched_init() { sched_thread_running_callback(current_thread()); timer_insert(current_cpu()->timer, ¤t_cpu()->schedtimerentry, reschedule_timer_dpc, NULL, QUANTUM_US, true); + timer_insert(current_cpu()->timer, ¤t_cpu()->calendar_tick_timer_entry, sched_calendar_tick, NULL, 10000, true); // XXX move this resume to a more appropriate place timer_resume(current_cpu()->timer); } diff --git a/kernel-src/sys/syscalls/fork.c b/kernel-src/sys/syscalls/fork.c index 5d03e72a..00eb8fda 100644 --- a/kernel-src/sys/syscalls/fork.c +++ b/kernel-src/sys/syscalls/fork.c @@ -5,6 +5,7 @@ #include #include #include +#include syscallret_t syscall_fork(context_t *ctx) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/getresgid.c b/kernel-src/sys/syscalls/getresgid.c index 576c80c6..285b26a7 100644 --- a/kernel-src/sys/syscalls/getresgid.c +++ b/kernel-src/sys/syscalls/getresgid.c @@ -1,6 +1,7 @@ #include #include #include +#include syscallret_t syscall_getresgid(context_t *, gid_t *gidp, gid_t *egidp, gid_t *sgidp) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/getresuid.c b/kernel-src/sys/syscalls/getresuid.c index f35f7e12..7474b368 100644 --- a/kernel-src/sys/syscalls/getresuid.c +++ b/kernel-src/sys/syscalls/getresuid.c @@ -1,6 +1,7 @@ #include #include #include +#include syscallret_t syscall_getresuid(context_t *, uid_t *uidp, uid_t *euidp, uid_t *suidp) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/gettid.c b/kernel-src/sys/syscalls/gettid.c index 88e525aa..6e284c5d 100644 --- a/kernel-src/sys/syscalls/gettid.c +++ b/kernel-src/sys/syscalls/gettid.c @@ -1,5 +1,6 @@ #include #include +#include syscallret_t syscall_gettid(context_t *) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/ioctl.c b/kernel-src/sys/syscalls/ioctl.c index 614207f3..b7b91f0c 100644 --- a/kernel-src/sys/syscalls/ioctl.c +++ b/kernel-src/sys/syscalls/ioctl.c @@ -1,6 +1,7 @@ #include #include #include +#include syscallret_t syscall_ioctl(context_t *, int fd, unsigned long request, void *arg) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/linkat.c b/kernel-src/sys/syscalls/linkat.c index a860bf29..f5d61422 100644 --- a/kernel-src/sys/syscalls/linkat.c +++ b/kernel-src/sys/syscalls/linkat.c @@ -1,6 +1,6 @@ #include #include - +#include syscallret_t syscall_linkat(context_t *, int olddirfd, char *uoldpath, int newdirfd, char *unewpath, int flags, int type) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/newthread.c b/kernel-src/sys/syscalls/newthread.c index fade3e92..a7f9b305 100644 --- a/kernel-src/sys/syscalls/newthread.c +++ b/kernel-src/sys/syscalls/newthread.c @@ -2,6 +2,7 @@ #include #include #include +#include syscallret_t syscall_newthread(context_t *, void *entry, void *stack) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/seek.c b/kernel-src/sys/syscalls/seek.c index dbf7c034..2322e9a8 100644 --- a/kernel-src/sys/syscalls/seek.c +++ b/kernel-src/sys/syscalls/seek.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #define SEEK_SET 0 diff --git a/kernel-src/sys/syscalls/setresgid.c b/kernel-src/sys/syscalls/setresgid.c index fe8ffe51..c3248a58 100644 --- a/kernel-src/sys/syscalls/setresgid.c +++ b/kernel-src/sys/syscalls/setresgid.c @@ -1,6 +1,7 @@ #include #include #include +#include syscallret_t syscall_setresgid(context_t *, gid_t gid, gid_t egid, gid_t sgid) { syscallret_t ret; diff --git a/kernel-src/sys/syscalls/setresuid.c b/kernel-src/sys/syscalls/setresuid.c index 865cf8a5..0c9f165f 100644 --- a/kernel-src/sys/syscalls/setresuid.c +++ b/kernel-src/sys/syscalls/setresuid.c @@ -1,6 +1,7 @@ #include #include #include +#include syscallret_t syscall_setresuid(context_t *, uid_t uid, uid_t euid, uid_t suid) { syscallret_t ret; diff --git a/kernel-src/sys/syscalls/truncate.c b/kernel-src/sys/syscalls/truncate.c index 911e897e..f342b12e 100644 --- a/kernel-src/sys/syscalls/truncate.c +++ b/kernel-src/sys/syscalls/truncate.c @@ -1,5 +1,6 @@ #include #include +#include syscallret_t syscall_ftruncate(context_t *, int fd, size_t size) { syscallret_t ret = { diff --git a/kernel-src/sys/syscalls/write.c b/kernel-src/sys/syscalls/write.c index bc5704dd..1defaee0 100644 --- a/kernel-src/sys/syscalls/write.c +++ b/kernel-src/sys/syscalls/write.c @@ -2,6 +2,7 @@ #include #include #include +#include syscallret_t syscall_write(context_t *context, int fd, void *buffer, size_t size) { syscallret_t ret = { diff --git a/kernel-src/sys/thread.c b/kernel-src/sys/thread.c index 0f5711c5..b091529c 100644 --- a/kernel-src/sys/thread.c +++ b/kernel-src/sys/thread.c @@ -45,6 +45,7 @@ thread_t *sched_newthread(void *ip, size_t kstacksize, int priority, proc_t *pro EVENT_INITHEADER(&thread->signals.waitpendingevent); memset(&thread->metrics, 0, sizeof(thread->metrics)); thread->metrics.sleep_time_avg_us = 1; // to prevent a division by 0 when the thread first gets scheduled + thread->class = THREAD_CLASS_TIMESHARE; return thread; }