turnstile fixes

2026-10-22 01:34:45 +02:00
parent dbb543de66
commit b83f1fe6d9
10 changed files with 419 additions and 82 deletions
@@ -3,10 +3,12 @@
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/atomic.h>
+#include <zinnia/irq.h>
 #include <zinnia/percpu.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/turnstile.h>
+#include <zinnia/utils.h>
 #include <arch/cpu.h>
 #include <sched/internal.h>

@@ -20,6 +22,7 @@ struct turnstile_chain {
 };

 static struct turnstile_chain chains[TURNSTILE_CHAINS];
+static struct spinlock loser_lock;

 // Gets the chain from a lock object address.
 static struct turnstile_chain* chain_for(const void* obj) {
@@ -115,23 +118,26 @@ static bool recompute_locked(struct task* task) {
 }

 // Makes the waiters of `ts` lend their priority to `owner`, and recomputes the old owner's priority.
-static void set_owner(struct turnstile* ts, struct task* owner) {
+static void set_owner(struct turnstile_chain* chain, struct turnstile* ts, struct task* owner) {
    struct task* old = ts->owner;
    if (old == owner)
        return;

-    if (old) {
-        spin_lock(&old->lock);
+    if (old && ts->lending) {
+        assert(old == percpu_get()->sched.current, "Turnstile %p is owned by another task", ts);
+        task_lock(old);
        tailq_remove(&old->pi_lenders, ts, lender_link);
        recompute_locked(old);
-        spin_unlock(&old->lock);
+        task_unlock(old);
    }

    ts->owner = owner;
+    ts->lending = false;
    if (owner) {
-        spin_lock(&owner->lock);
+        assert(atomic_load(&owner->lockp, ATOMIC_RELAXED) == &chain->lock, "New owner %p isn't blocked", owner);
        tailq_insert_tail(&owner->pi_lenders, ts, lender_link);
-        spin_unlock(&owner->lock);
+        ts->lending = true;
+        recompute_locked(owner);
    }
 }

@@ -140,54 +146,94 @@ static void requeue(struct turnstile* ts, struct task* task) {
    waiters_insert(ts, task);
 }

-// Recomputes the priority of the owner of `ts` along the owner chain.
-static void propagate(struct turnstile_chain* chain, struct turnstile* ts) {
-    bool retried = false;
+static struct spinlock* lock_owner(struct task* owner, struct spinlock* root, struct spinlock* held, bool* lost) {
+    while (true) {
+        struct spinlock* lock = atomic_load(&owner->lockp, ATOMIC_ACQUIRE);
+        if (lock == root || lock == held)
+            return nullptr;
+
+        if (lock == &owner->lock || lock > MAX(root, held)) {
+            spin_lock(lock);
+        } else if (!spin_trylock(lock)) {
+            *lost = true;
+            return lock;
+        }
+
+        if (atomic_load(&owner->lockp, ATOMIC_RELAXED) == lock)
+            return lock;
+        spin_unlock(lock);
+    }
+}
+
+// Recomputes the priority of the owners along the blocking chain of `blocker`.
+static void propagate(struct turnstile_chain* root, struct task* blocker) {
+    struct turnstile_chain* chain = root;
+    struct turnstile* ts = blocker->blocked_ts;
+    size_t depth = 0;
+    size_t forced = 0;
+    bool loser = false;
    while (ts->owner) {
        struct task* owner = ts->owner;
-        spin_lock(&owner->lock);
-        bool changed = recompute_locked(owner);
-        const void* next_obj = atomic_load(&owner->blocked_obj, ATOMIC_RELAXED);
-        spin_unlock(&owner->lock);
-        if ((!changed && !retried) || next_obj == nullptr)
-            break;
-
-        struct turnstile_chain* next_chain = chain_for(next_obj);
-        if (next_chain > chain) {
-            spin_lock(&next_chain->lock);
-        } else if (next_chain < chain && !spin_trylock(&next_chain->lock)) {
-            const void* obj = ts->obj;
-            turnstile_unlock(chain);
-            chain = turnstile_lock(obj);
-            ts = chain_lookup(chain, obj);
-            if (ts == nullptr)
+        assert(owner != blocker, "Deadlock: cycle in the blocking chain of task %p", blocker);
+        bool lost = false;
+        struct spinlock* locked = lock_owner(owner, &root->lock, &chain->lock, &lost);
+        if (lost) {
+            if (chain != root)
+                turnstile_unlock(chain);
+            turnstile_unlock(root);
+            spin_lock(locked);
+            spin_unlock(locked);
+            if (!loser) {
+                spin_lock(&loser_lock);
+                loser = true;
+            }
+            spin_lock(&root->lock);
+            chain = root;
+            forced = MAX(forced, depth + 1);
+            depth = 0;
+            if (atomic_load(&blocker->lockp, ATOMIC_RELAXED) != &root->lock)
                break;
-            retried = true;
+            ts = blocker->blocked_ts;
            continue;
        }

-        struct turnstile* next = chain_lookup(next_chain, next_obj);
-        bool blocked = next && atomic_load(&owner->blocked_ts, ATOMIC_RELAXED) == next;
-        if (next_chain != chain)
+        if (!ts->lending) {
+            tailq_insert_tail(&owner->pi_lenders, ts, lender_link);
+            ts->lending = true;
+        }
+        bool changed = recompute_locked(owner);
+        struct turnstile* next = owner->blocked_ts;
+        if ((!changed && depth >= forced) || next == nullptr) {
+            if (locked)
+                spin_unlock(locked);
+            break;
+        }
+
+        struct turnstile_chain* next_chain = chain_for(next->obj);
+        assert(atomic_load(&owner->lockp, ATOMIC_RELAXED) == &next_chain->lock, "Blocked task %p isn't locked", owner);
+        if (chain != root && chain != next_chain)
            turnstile_unlock(chain);
        chain = next_chain;
-        if (!blocked)
-            break;

        requeue(next, owner);
-        if (!update_top(next))
+        if (!update_top(next) && depth + 1 >= forced)
            break;

        ts = next;
-        retried = false;
+        depth++;
    }
-    turnstile_unlock(chain);
+    if (chain != root)
+        turnstile_unlock(chain);
+    turnstile_unlock(root);
+    if (loser)
+        spin_unlock(&loser_lock);
 }

 void turnstile_block(struct turnstile_chain* chain, const void* obj, struct task* owner, enum turnstile_queue queue) {
    struct task* self = percpu_get()->sched.current;
    assert(self->turnstile != nullptr, "Task is already blocked on a turnstile");
    assert(owner != self, "Task tried to block on a lock it owns");
+    irq_lock();

    // Lend our turnstile to the lock if it doesn't have one yet, otherwise donate it.
    struct turnstile* ts = chain_lookup(chain, obj);
@@ -195,6 +241,7 @@ void turnstile_block(struct turnstile_chain* chain, const void* obj, struct task
        ts = self->turnstile;
        ts->obj = obj;
        ts->owner = nullptr;
+        ts->lending = false;
        ts->spare = nullptr;
        ts->chain_next = chain->head;
        chain->head = ts;
@@ -204,24 +251,28 @@ void turnstile_block(struct turnstile_chain* chain, const void* obj, struct task
    }
    self->turnstile = nullptr;

+    if (owner && ts->owner == nullptr)
+        ts->owner = owner;
+    assert(owner == nullptr || ts->owner == owner, "Turnstile %p has a stale owner", ts);
+
    spin_lock(&self->lock);
    atomic_store(&self->ts_woken, false, ATOMIC_RELAXED);
-    atomic_store(&self->blocked_ts, ts, ATOMIC_RELAXED);
-    atomic_store(&self->blocked_obj, obj, ATOMIC_RELAXED);
+    self->blocked_ts = ts;
    self->ts_queue = queue;
    waiters_insert(ts, self);
    update_top(ts);
+    atomic_store(&self->lockp, &chain->lock, ATOMIC_RELEASE);
    spin_unlock(&self->lock);

-    set_owner(ts, owner);
-    propagate(chain, ts);
+    propagate(chain, self);

    // Check if we were actually woken.
    while (!atomic_load(&self->ts_woken, ATOMIC_ACQUIRE))
        sched_block(&percpu_get()->sched);

-    spin_lock(&self->lock);
-    spin_unlock(&self->lock);
+    task_lock(self);
+    task_unlock(self);
+    irq_unlock();
 }

 struct task* turnstile_first(
@@ -237,16 +288,15 @@ struct task* turnstile_first(
 }

 void turnstile_wake(struct turnstile_chain* chain, const void* obj, struct task* task, struct task* new_owner) {
-    struct turnstile* ts = atomic_load(&task->blocked_ts, ATOMIC_RELAXED);
+    struct turnstile* ts = task->blocked_ts;
    assert(ts && ts->obj == obj, "Task %p isn't blocked on %p", task, obj);
+    assert(atomic_load(&task->lockp, ATOMIC_RELAXED) == &chain->lock, "Task %p isn't locked by %p", task, chain);

    tailq_remove(&ts->waiters[task->ts_queue], task, ts_link);
-    atomic_store(&task->blocked_ts, nullptr, ATOMIC_RELAXED);
-    atomic_store(&task->blocked_obj, nullptr, ATOMIC_RELAXED);

    if (!has_waiters(ts)) {
        // The last waiter takes the lock's turnstile with it.
-        set_owner(ts, nullptr);
+        set_owner(chain, ts, nullptr);
        chain_remove(chain, ts);
        ts->obj = nullptr;
        task->turnstile = ts;
@@ -254,36 +304,35 @@ void turnstile_wake(struct turnstile_chain* chain, const void* obj, struct task*
        task->turnstile = ts->spare;
        ts->spare = ts->spare->spare;
        update_top(ts);
-        set_owner(ts, new_owner);
-        if (new_owner) {
-            spin_lock(&new_owner->lock);
-            recompute_locked(new_owner);
-            spin_unlock(&new_owner->lock);
-        }
+        set_owner(chain, ts, new_owner);
    }
+    task->blocked_ts = nullptr;

    spin_lock(&task->lock);
+    atomic_store(&task->lockp, &task->lock, ATOMIC_RELEASE);
    atomic_store(&task->ts_woken, true, ATOMIC_RELEASE);
    sched_wake_locked(task);
    spin_unlock(&task->lock);
 }

-void turnstile_reprioritize(struct task* task) {
-    const void* obj = atomic_load(&task->blocked_obj, ATOMIC_RELAXED);
-    if (obj == nullptr)
-        return;
-
-    struct turnstile_chain* chain = turnstile_lock(obj);
+void turnstile_waive(struct turnstile_chain* chain, const void* obj) {
    struct turnstile* ts = chain_lookup(chain, obj);
-    if (ts == nullptr || atomic_load(&task->blocked_ts, ATOMIC_RELAXED) != ts) {
-        turnstile_unlock(chain);
+    if (ts)
+        set_owner(chain, ts, nullptr);
+}
+
+void turnstile_reprioritize(struct task* task) {
+    struct turnstile* ts = task->blocked_ts;
+    if (ts == nullptr) {
+        task_unlock(task);
        return;
    }

+    struct turnstile_chain* chain = chain_for(ts->obj);
    requeue(ts, task);
    if (!update_top(ts)) {
        turnstile_unlock(chain);
        return;
    }
-    propagate(chain, ts);
+    propagate(chain, task);
 }