turnstile fixes

2026-10-22 01:34:45 +02:00
parent dbb543de66
commit b83f1fe6d9
10 changed files with 419 additions and 82 deletions
@@ -9,7 +9,7 @@ choice SCHED_ALG
 config SCHED_DUMB
  bool "Dumb Scheduler"
  help
-    Very basic scheduler, without any priorities.
+    Very basic scheduler.

 endchoice

@@ -28,9 +28,36 @@ void sched_to_user_context(uintptr_t ctx) {
    arch_sched_jump_to_context((struct arch_context*)ctx);
 }

+static struct spinlock task_cache_lock;
+static tailq_head(struct task) task_cache = {nullptr, &task_cache.tqh_first};
+
+static struct task* task_alloc() {
+    spin_lock(&task_cache_lock);
+    struct task* task = tailq_first(&task_cache);
+    if (task)
+        tailq_remove(&task_cache, task, next);
+    spin_unlock(&task_cache_lock);
+
+    if (task == nullptr)
+        return mem_alloc(sizeof(struct task), 0);
+
+    size_t keep_start = offsetof(struct task, on_cpu);
+    size_t keep_end = keep_start + sizeof(task->on_cpu);
+    memset(task, 0, keep_start);
+    memset((char*)task + keep_end, 0, sizeof(struct task) - keep_end);
+    return task;
+}
+
+static void task_release(struct task* task) {
+    spin_lock(&task_cache_lock);
+    tailq_insert_tail(&task_cache, task, next);
+    spin_unlock(&task_cache_lock);
+}
+
 static void task_init_common(struct task* task, const char* name, struct vma* space) {
    task->space = space;
    spin_init(&task->lock);
+    atomic_store(&task->lockp, &task->lock, ATOMIC_RELAXED);
    strncpy(task->name, name, sizeof(task->name));
    task->base_prio = SCHED_PRIO_DEFAULT;
    task->time_slice = SCHED_TIME_SLICE_NS;
@@ -41,13 +68,13 @@ static void task_init_common(struct task* task, const char* name, struct vma* sp
 }

 errno_t task_create_current(const char* name, struct vma* space, struct task** out) {
-    struct task* task = mem_alloc(sizeof(struct task), 0);
+    struct task* task = task_alloc();
    if (task == nullptr)
        return ENOMEM;

    task->turnstile = turnstile_alloc();
    if (task->turnstile == nullptr) {
-        mem_free(task);
+        task_release(task);
        return ENOMEM;
    }

@@ -62,7 +89,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
    if (!out)
        return EINVAL;

-    struct task* new_task = mem_alloc(sizeof(struct task), 0);
+    struct task* new_task = task_alloc();
    if (new_task == nullptr)
        return ENOMEM;

@@ -70,7 +97,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
    new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE);
    new_task->turnstile = turnstile_alloc();
    if (new_task->turnstile == nullptr) {
-        mem_free(new_task);
+        task_release(new_task);
        return ENOMEM;
    }

@@ -78,7 +105,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
    errno_t s = stack_init(&new_task->kernel_stack, KERNEL_STACK_SIZE);
    if (s != 0) {
        turnstile_free(new_task->turnstile);
-        mem_free(new_task);
+        task_release(new_task);
        return s;
    }
    new_task->user_stack = 0;
@@ -88,7 +115,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
    if (s != 0) {
        stack_destroy(&new_task->kernel_stack);
        turnstile_free(new_task->turnstile);
-        mem_free(new_task);
+        task_release(new_task);
        return s;
    }

@@ -102,5 +129,19 @@ void task_destroy(struct task* task) {
    turnstile_free(task->turnstile);
    arch_task_fini(&task->context);
    stack_destroy(&task->kernel_stack);
-    mem_free(task);
+    task_release(task);
+}
+
+void task_lock(struct task* task) {
+    while (true) {
+        struct spinlock* lock = atomic_load(&task->lockp, ATOMIC_ACQUIRE);
+        spin_lock(lock);
+        if (atomic_load(&task->lockp, ATOMIC_RELAXED) == lock)
+            return;
+        spin_unlock(lock);
+    }
+}
+
+void task_unlock(struct task* task) {
+    spin_unlock(atomic_load(&task->lockp, ATOMIC_RELAXED));
 }
@@ -112,6 +112,11 @@ static uint8_t effective_prio(struct task* task) {
    return MAX(prio, task->lent_prio);
 }

+static void refresh_prio(struct task* task) {
+    if (task->blocked_ts == nullptr)
+        atomic_store(&task->priority, effective_prio(task), ATOMIC_RELAXED);
+}
+
 static size_t quantum(struct task* task) {
    return SCHED_TIME_SLICE_NS * (task->penalty + 1);
 }
@@ -120,7 +125,7 @@ static void feedback_reset(struct scheduler* sched, struct task* task) {
    task->penalty = 0;
    task->time_slice = quantum(task);
    task->feedback_epoch = sched->boost_epoch;
-    atomic_store(&task->priority, effective_prio(task), ATOMIC_RELAXED);
+    refresh_prio(task);
 }

 static void boost(struct scheduler* sched) {
@@ -153,7 +158,7 @@ static void account(struct scheduler* sched, struct task* task, uint64_t now) {
        task->penalty++;
    task->time_slice = quantum(task);
    task->feedback_epoch = sched->boost_epoch;
-    atomic_store(&task->priority, effective_prio(task), ATOMIC_RELAXED);
+    refresh_prio(task);
    sched->need_resched = true;
 }

@@ -164,6 +169,7 @@ static void enqueue(struct scheduler* sched, struct task* task) {
    task->last_rq = sched;
    if (task->feedback_epoch < sched->boost_epoch)
        feedback_reset(sched, task);
+    refresh_prio(task);
    rq_insert(sched, task);
    bool preempt = atomic_load(&task->priority, ATOMIC_RELAXED) > atomic_load(&sched->current_priority, ATOMIC_RELAXED);
    spin_unlock(&sched->lock);
@@ -273,9 +279,9 @@ void sched_block(struct scheduler* sched) {
    irq_lock();
    struct task* current = sched->current;

-    spin_lock(&current->lock);
+    task_lock(current);
    if (atomic_exchange(&current->wake_pending, false, ATOMIC_ACQ_REL)) {
-        spin_unlock(&current->lock);
+        task_unlock(current);
        irq_unlock();
        return;
    }
@@ -286,7 +292,7 @@ void sched_block(struct scheduler* sched) {
    spin_unlock(&sched->lock);

    current->state = TASK_STATE_BLOCKED;
-    spin_unlock(&current->lock);
+    task_unlock(current);

    do_reschedule(sched, now);
 }
@@ -315,9 +321,9 @@ void sched_wake_locked(struct task* task) {
 }

 void sched_wake(struct task* task) {
-    spin_lock(&task->lock);
+    task_lock(task);
    sched_wake_locked(task);
-    spin_unlock(&task->lock);
+    task_unlock(task);
 }

 bool sched_update_priority_locked(struct task* task) {
@@ -357,13 +363,12 @@ bool sched_update_priority_locked(struct task* task) {

 void sched_set_priority(struct task* task, uint8_t priority) {
    assert(priority <= SCHED_PRIO_MAX, "Priority %u is out of range", priority);
-    spin_lock(&task->lock);
+    task_lock(task);
    task->base_prio = priority;
-    bool changed = sched_update_priority_locked(task);
-    spin_unlock(&task->lock);
-
-    if (changed)
+    if (sched_update_priority_locked(task))
        turnstile_reprioritize(task);
+    else
+        task_unlock(task);
 }

 void task_entry(task_fn_t entry, uintptr_t arg0, struct task* prev) {