ke/turnstile: add support for multiple owners

src/kern/mutex.zig

@@ -59,7 +59,7 @@ pub const Mutex = struct {
             }
 
             // Block until woken.
-            ki.turnstile.block(ts, m, owner.?, .Exclusive);
+            ki.turnstile.block(ts, m, .{ .single = owner.? }, .Exclusive);
 
             // Re-try acquisition after wakeup.
             if (m.owner.cmpxchgStrong(null, curtd, .acquire, .monotonic) == null) {
@@ -87,6 +87,7 @@ pub const Mutex = struct {
         // has been shown to be better because this avoids lock convoys, see this mysterious 70s paper:
         // https://dl.acm.org/doi/pdf/10.1145/850657.850659
         ki.turnstile.wakeup(ts.?, .Exclusive, ts.?.waiters, null);
+        ki.turnstile.exit(m);
 
         ke.ipl.lower(ipl);
     }

src/kern/turnstile.zig

@@ -12,6 +12,43 @@ pub const Waiter = struct {
     thread: *ke.Thread,
 };
 
+/// One donation edge into a thread's `turnstiles_owned` list.
+pub const Boost = struct {
+    /// Linkage into the boosted thread's `turnstiles_owned` list.
+    link: rtl.List.Entry,
+    /// Priority currently donated, null means this edge is not boosting anyone.
+    donated: ?u8,
+};
+
+/// A thread a turnstile donates to, and the donation made on its behalf.
+pub const Owner = struct {
+    link: rtl.List.Entry,
+    thread: *ke.Thread,
+    boost: Boost,
+};
+
+pub const Ownership = union(enum) {
+    /// Single owner.
+    single: *ke.Thread,
+    /// Multiple owners.
+    shared: *rtl.List,
+};
+
+const OwnerSet = union(enum) {
+    none,
+    single: struct {
+        thread: *ke.Thread,
+        boost: Boost,
+    },
+    shared: *rtl.List,
+};
+
+/// One thread a turnstile donates to.
+const Target = struct {
+    thread: *ke.Thread,
+    boost: *Boost,
+};
+
 pub const Turnstile = struct {
     /// Linkage for hash chain.
     link: rtl.List.Entry,
@@ -23,13 +60,19 @@ pub const Turnstile = struct {
     waiters: usize,
     /// Object threads are waiting on.
     obj: *anyopaque,
-    /// Thread currently holding the object, and the target of any donation.
-    owner: ?*ke.Thread,
-    /// Linkage into `owner`'s `turnstiles_owned` list.
-    boost_link: rtl.List.Entry,
-    /// Priority currently donated to `owner`, null means this turnstile is not
-    /// boosting anyone.
-    donated: ?u8,
+    /// Threads this turnstile donates to.
+    owners: OwnerSet,
+
+    fn detach(ts: *Turnstile) void {
+        ts.owners = .none;
+    }
+
+    pub fn reset(ts: *Turnstile) void {
+        ts.next_free = null;
+        ts.queues[0].init();
+        ts.queues[1].init();
+        ts.detach();
+    }
 };
 
 const Chain = struct {
@@ -41,9 +84,7 @@ const Chain = struct {
 
 /// Which queue to block or wakeup threads on.
 pub const Queue = enum(u1) {
-    /// Only one waiter will be woken up at once.
     Exclusive = 0,
-    /// All waiters will be woken up.
     Shared = 1,
 };
 
@@ -53,6 +94,10 @@ fn hash_obj(obj: *const anyopaque) usize {
     return (@intFromPtr(obj) >> 6) & hash_mask;
 }
 
+fn chain_for(obj: *const anyopaque) *Chain {
+    return &chains[hash_obj(obj)];
+}
+
 /// Highest effective priority among all waiters on `ts`, or 0 if none.
 /// The caller must hold `ts`'s chain lock so the waiter set is stable.
 fn highest_waiter(ts: *Turnstile) ?u8 {
@@ -69,14 +114,56 @@ fn highest_waiter(ts: *Turnstile) ?u8 {
     return best;
 }
 
+/// Walks the threads a turnstile donates to, this is to abstract over the
+/// different kinds of ownership.
+const Targets = struct {
+    single: ?Target,
+    list: ?rtl.List.Iterator,
+
+    fn next(self: *Targets) ?Target {
+        if (self.single) |t| {
+            self.single = null;
+            return t;
+        }
+
+        if (self.list) |*it| {
+            if (!it.next()) return null;
+            const o: *Owner = @fieldParentPtr("link", it.get());
+            it.advance();
+            return .{ .thread = o.thread, .boost = &o.boost };
+        }
+
+        return null;
+    }
+};
+
+fn targets(ts: *Turnstile) Targets {
+    return switch (ts.owners) {
+        .none => .{ .single = null, .list = null },
+        .single => .{ .single = sole_owner(ts), .list = null },
+        .shared => |list| .{ .single = null, .list = list.iterator() },
+    };
+}
+
+/// The turnstile's owner when there is exactly one to follow, else null.
+fn sole_owner(ts: *Turnstile) ?Target {
+    return switch (ts.owners) {
+        .single => .{
+            .thread = ts.owners.single.thread,
+            .boost = &ts.owners.single.boost,
+        },
+        else => null,
+    };
+}
+
 /// Compute the inherited priority of `td` from the turnstiles it
 /// owns. `td.lock` must be held.
 fn recompute_inherited(td: *ke.Thread) void {
     var pri: u8 = 0;
     var it = td.turnstiles_owned.iterator();
     while (it.next()) : (it.advance()) {
-        const ts: *Turnstile = @fieldParentPtr("boost_link", it.get());
-        pri = @max(pri, ts.donated orelse 0);
+        const boost: *Boost = @fieldParentPtr("link", it.get());
+        pri = @max(pri, boost.donated orelse 0);
     }
     td.inherited_prio = pri;
 }
@@ -86,51 +173,74 @@ fn update_prio(td: *ke.Thread) void {
     if (pri != td.priority) ki.sched.update_priority_locked(td, pri);
 }
 
-/// Donate priority `pri` to `to` through turnstile `ts`. The caller holds
-/// `to.lock` and `ts`'s chain lock. Returns whether `to`'s effective priority
-/// actually rose (false means the chain stops here).
-fn donate_to(ts: *Turnstile, to: *ke.Thread, pri: u8) bool {
-    std.debug.assert(ts.owner == to);
-
+/// Donate priority `pri` to `to` along the edge `boost`. The caller holds
+/// `to.lock` and the owning turnstile's chain lock. Returns whether `to`'s
+/// priority actually rose (false means the chain stops here).
+fn donate_to(boost: *Boost, to: *ke.Thread, pri: u8) bool {
     // A lend only ever raises.
-    if (ts.donated != null and pri <= ts.donated.?) return false;
+    if (boost.donated != null and pri <= boost.donated.?) return false;
 
     const before = to.priority;
 
-    if (ts.donated == null) to.turnstiles_owned.insert_head(&ts.boost_link);
-    ts.donated = pri;
+    if (boost.donated == null) to.turnstiles_owned.insert_head(&boost.link);
+    boost.donated = pri;
     if (pri > to.inherited_prio) to.inherited_prio = pri;
 
     update_prio(to);
     return to.priority > before;
 }
 
-/// Detach `ts`'s donation from whatever thread it is currently boosting and
-/// recompute that thread's priority (which may lower it). Caller holds `ts`'s
-/// chain lock.
-fn revoke(ts: *Turnstile) void {
-    // Not boosting anyone.
-    if (ts.donated == null) return;
+/// Undo the donation currently boosting `to`. The caller
+/// holds the owning turnstile's chain lock.
+fn undonate(boost: *Boost, to: *ke.Thread) void {
+    if (boost.donated == null) return;
 
-    const to = ts.owner.?;
     to.lock.acquire_no_ipl();
-    ts.boost_link.remove();
-    ts.donated = null;
+    boost.link.remove();
+    boost.donated = null;
     // Recompute the priority as the floor may drop now.
     recompute_inherited(to);
     update_prio(to);
     to.lock.release_no_ipl();
 }
 
-/// (Re)establish `ts`'s donation to `to` from its current set of waiters.
-/// Caller holds `ts`'s chain lock.
-fn reboost(ts: *Turnstile, to: *ke.Thread) void {
-    const pri = highest_waiter(ts);
-    if (pri == null) return;
+/// Detach `ts`'s donations from the threads they boost and recompute their
+/// priorities. Caller holds `ts`'s chain lock.
+fn revoke(ts: *Turnstile) void {
+    var it = targets(ts);
+    while (it.next()) |t| undonate(t.boost, t.thread);
+}
 
-    to.lock.acquire_no_ipl();
-    _ = donate_to(ts, to, pri.?);
-    to.lock.release_no_ipl();
+/// (Re)establish `ts`'s donations from its current set of waiters.
+/// Caller holds `ts`'s chain lock.
+fn reboost(ts: *Turnstile) void {
+    const pri = highest_waiter(ts) orelse return;
+    var it = targets(ts);
+    while (it.next()) |t| {
+        t.thread.lock.acquire_no_ipl();
+        _ = donate_to(t.boost, t.thread, pri);
+        t.thread.lock.release_no_ipl();
+    }
+}
+
+fn attach(ts: *Turnstile, own: Ownership) void {
+    ts.owners = switch (own) {
+        .single => |td| .{ .single = .{
+            .thread = td,
+            .boost = .{ .link = undefined, .donated = null },
+        } },
+        .shared => |list| .{ .shared = list },
+    };
+}
+
+fn attached(ts: *Turnstile, own: Ownership) bool {
+    return switch (own) {
+        .single => |td| if (sole_owner(ts)) |t| t.thread == td else false,
+        .shared => |list| switch (ts.owners) {
+            .shared => |cur| cur == list,
+            else => false,
+        },
+    };
 }
 
 /// Lend `curtd`'s effective priority down the blocking chain, boosting each
@@ -151,7 +261,7 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void {
         // We hold `thread.lock` (curtd's on the first
         // pass, the previous owner's after a hop) and `root`, `held` is null.
         const obj = thread.waiting_on orelse break;
-        const bucket = &chains[hash_obj(obj)];
+        const bucket = chain_for(obj);
 
         if (bucket != root) {
             if (!bucket.lock.try_acquire_no_ipl()) {
@@ -175,11 +285,16 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void {
         std.debug.assert(thread.waiting_on == obj);
 
         const ts = thread.turnstile;
-        const owner = ts.owner orelse break;
+
+        if (sole_owner(ts)) |o| {
+            // For single-owner locks, do multi-hop priority inheritance.
+            // Multi-hop means we handle the case where a boosted thread itself
+            // is waiting on another resource.
+            const owner = o.thread;
             if (owner == curtd) @panic("turnstile: cycle in blocking chain");
 
             owner.lock.acquire_no_ipl();
-        if (!donate_to(ts, owner, donate)) {
+            if (!donate_to(o.boost, owner, donate)) {
                 owner.lock.release_no_ipl();
                 break;
             }
@@ -193,6 +308,19 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void {
             if (held) |h| h.lock.release_no_ipl();
             held = null;
             thread = owner;
+        } else {
+            // Only do single-hop priority boosting for multiple owners.
+            // This is fine as we only use this for SMR read sections,
+            // and they are forbidden to explicitly block.
+            var it = targets(ts);
+            while (it.next()) |t| {
+                if (t.thread == curtd) @panic("turnstile: cycle in blocking chain");
+                t.thread.lock.acquire_no_ipl();
+                _ = donate_to(t.boost, t.thread, donate);
+                t.thread.lock.release_no_ipl();
+            }
+            break;
+        }
     }
 
     // Leave the caller holding exactly curtd.lock and root.
@@ -210,11 +338,27 @@ pub fn init_turnstiles() void {
     }
 }
 
+/// Add an owner to a turnstile whose ownership is shared. The caller inserts
+/// `o` into the shared list and holds the chain lock.
+pub fn owner_enter(ts: *Turnstile, o: *Owner) void {
+    const pri = highest_waiter(ts) orelse return;
+
+    o.thread.lock.acquire_no_ipl();
+    _ = donate_to(&o.boost, o.thread, pri);
+    o.thread.lock.release_no_ipl();
+}
+
+/// Revoke one owner's donation. The caller removes `o` from the shared list
+/// and holds the chain lock.
+pub fn owner_leave(o: *Owner) void {
+    undonate(&o.boost, o.thread);
+}
+
 /// Look up the turnstile for the specified object.
 /// This acquires the turnstile chain lock and must be called at IPL dispatch.
 /// Returns null if no turnstile is found.
 pub fn lookup(obj: *const anyopaque) ?*Turnstile {
-    const chain = &chains[hash_obj(obj)];
+    const chain = chain_for(obj);
 
     chain.lock.acquire_no_ipl();
 
@@ -231,14 +375,17 @@ pub fn lookup(obj: *const anyopaque) ?*Turnstile {
 
 /// Drop the lock protecting the chain for obj.
 pub fn exit(obj: *const anyopaque) void {
-    chains[hash_obj(obj)].lock.release_no_ipl();
+    chain_for(obj).lock.release_no_ipl();
 }
 
-/// Block the current thread on a synchronization object backed by a turnstile.
-/// This must be called with the appropriate turnstile chain lock held,
-/// and returns with the chain lock released. IPL is kept as before on return.
-pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, owner: *ke.Thread, queue: Queue) void {
-    const chain = &chains[hash_obj(obj)];
+/// Block the current thread on a synchronization object backed by a turnstile,
+/// donating to the owner set described by `own`.
+///
+/// This must be called with the appropriate turnstile chain lock held, and
+/// returns with the chain lock released.
+/// IPL is kept as before on return.
+pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Queue) void {
+    const chain = chain_for(obj);
     const curtd = ki.sched.percpu.local().current_thread.?;
     const ipl = ke.ipl.current();
 
@@ -254,20 +401,19 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, owner: *ke.Thread, queue:
         turn.next_free = curtd.turnstile;
         curtd.turnstile = turn;
 
-        if (turn.owner) |cur| {
-            std.debug.assert(cur == owner);
-        } else {
-            // Owner might be null because of a partial wakeup
-            // where we cleared it, restore it here.
-            turn.owner = owner;
-            reboost(turn, owner);
+        if (!attached(turn, own)) {
+            // Either a partial wakeup detached the turnstile or the object
+            // changed ownership, move the donation over.
+            revoke(turn);
+            attach(turn, own);
+            reboost(turn);
         }
     } else {
         // This is the first thread to block on this object.
         // Lend its turnstile and add it to the hash chain.
         ts = curtd.turnstile;
         ts.?.obj = obj;
-        ts.?.owner = owner;
+        attach(ts.?, own);
         chain.list.insert_head(&ts.?.link);
     }
 
@@ -295,56 +441,34 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, owner: *ke.Thread, queue:
 
     // Now block on the event.
     _ = ke.ipl.lower(.Passive);
-    _ = ke.wait.wait_one(&waiter.event.hdr, null) catch {
-        _ = ke.ipl.raise(ipl);
-
-        // If wait_one aborted (e.g. due to a timeout), remove the waiter and fix
-        // up the owner's inheritance.
-        chain.lock.acquire_no_ipl();
-
-        if (curtd.turnstile_waiter == &waiter) {
-            const active = ts.?;
-            const holder = active.owner;
-            revoke(active);
-            dequeue(active, curtd);
-            if (active.waiters > 0) {
-                if (holder) |o| reboost(active, o);
-            }
-        }
-
-        chain.lock.release_no_ipl();
-        return;
-    };
+    _ = ke.wait.wait_one(&waiter.event.hdr, null) catch unreachable;
     _ = ke.ipl.raise(ipl);
 }
 
 /// Wake whomever was waiting on the turnstile.
 /// Do hand-off to new_owner if specified.
-pub fn wakeup(turnstile: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread) void {
-    const chain = &chains[hash_obj(turnstile.obj)];
+/// The chain lock is still held on return.
+pub fn wakeup(ts: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread) void {
     const queue_idx = @intFromEnum(queue);
 
-    // The current owner is giving up the object, so it stops inheriting from
-    // this turnstile entirely.
-    revoke(turnstile);
+    // Revoke any priority we gave to the owner(s).
+    revoke(ts);
 
     if (new_owner) |no| {
         // Hand the object to a specific waiter, the rest now boost it.
-        dequeue(turnstile, no);
-        if (turnstile.waiters > 0) {
-            turnstile.owner = no;
-            reboost(turnstile, no);
+        dequeue(ts, no);
+        if (ts.waiters > 0) {
+            attach(ts, .{ .single = no });
+            reboost(ts);
         }
     } else {
         for (0..count) |_| {
-            if (turnstile.queues[queue_idx].is_empty()) break;
-            const waiter: *Waiter = @fieldParentPtr("link", turnstile.queues[queue_idx].first());
-            dequeue(turnstile, waiter.thread);
+            if (ts.queues[queue_idx].is_empty()) break;
+            const waiter: *Waiter = @fieldParentPtr("link", ts.queues[queue_idx].first());
+            dequeue(ts, waiter.thread);
         }
-        if (turnstile.waiters > 0) turnstile.owner = null;
+        if (ts.waiters > 0) ts.detach();
     }
-
-    chain.lock.release_no_ipl();
 }
 
 /// Remove a single waiter from the turnstile and wake it. Caller holds the
@@ -365,11 +489,7 @@ fn dequeue(ts: *Turnstile, td: *ke.Thread) void {
     } else {
         // Last waiter, pull the turnstile off the chain and keep it for ourselves.
         ts.link.remove();
-        ts.queues[0].init();
-        ts.queues[1].init();
-        ts.next_free = null;
-        ts.owner = null;
-        ts.donated = null;
+        ts.reset();
     }
 
     td.turnstile_waiter = null;

src/proc/private.zig

@@ -12,14 +12,11 @@ fn turnstile_ctor(ts: *ki.turnstile.Turnstile) void {
         .link = undefined,
         .next_free = null,
         .obj = undefined,
-        .owner = null,
-        .boost_link = undefined,
-        .donated = null,
         .queues = undefined,
         .waiters = 0,
+        .owners = .none,
     };
-    ts.queues[0].init();
-    ts.queues[1].init();
+    ts.reset();
 }
 
 /// Initialize the process subsystem.