diff --git a/src/kern/mutex.zig b/src/kern/mutex.zig index 8d82d99..9e44841 100644 --- a/src/kern/mutex.zig +++ b/src/kern/mutex.zig @@ -59,7 +59,7 @@ pub const Mutex = struct { } // Block until woken. - ki.turnstile.block(ts, m, owner.?, .Exclusive); + ki.turnstile.block(ts, m, .{ .single = owner.? }, .Exclusive); // Re-try acquisition after wakeup. if (m.owner.cmpxchgStrong(null, curtd, .acquire, .monotonic) == null) { @@ -87,6 +87,7 @@ pub const Mutex = struct { // has been shown to be better because this avoids lock convoys, see this mysterious 70s paper: // https://dl.acm.org/doi/pdf/10.1145/850657.850659 ki.turnstile.wakeup(ts.?, .Exclusive, ts.?.waiters, null); + ki.turnstile.exit(m); ke.ipl.lower(ipl); } diff --git a/src/kern/turnstile.zig b/src/kern/turnstile.zig index 0e988a7..c5851a9 100644 --- a/src/kern/turnstile.zig +++ b/src/kern/turnstile.zig @@ -12,6 +12,43 @@ pub const Waiter = struct { thread: *ke.Thread, }; +/// One donation edge into a thread's `turnstiles_owned` list. +pub const Boost = struct { + /// Linkage into the boosted thread's `turnstiles_owned` list. + link: rtl.List.Entry, + /// Priority currently donated, null means this edge is not boosting anyone. + donated: ?u8, +}; + +/// A thread a turnstile donates to, and the donation made on its behalf. +pub const Owner = struct { + link: rtl.List.Entry, + thread: *ke.Thread, + boost: Boost, +}; + +pub const Ownership = union(enum) { + /// Single owner. + single: *ke.Thread, + /// Multiple owners. + shared: *rtl.List, +}; + +const OwnerSet = union(enum) { + none, + single: struct { + thread: *ke.Thread, + boost: Boost, + }, + shared: *rtl.List, +}; + +/// One thread a turnstile donates to. +const Target = struct { + thread: *ke.Thread, + boost: *Boost, +}; + pub const Turnstile = struct { /// Linkage for hash chain. link: rtl.List.Entry, @@ -23,13 +60,19 @@ pub const Turnstile = struct { waiters: usize, /// Object threads are waiting on. obj: *anyopaque, - /// Thread currently holding the object, and the target of any donation. - owner: ?*ke.Thread, - /// Linkage into `owner`'s `turnstiles_owned` list. - boost_link: rtl.List.Entry, - /// Priority currently donated to `owner`, null means this turnstile is not - /// boosting anyone. - donated: ?u8, + /// Threads this turnstile donates to. + owners: OwnerSet, + + fn detach(ts: *Turnstile) void { + ts.owners = .none; + } + + pub fn reset(ts: *Turnstile) void { + ts.next_free = null; + ts.queues[0].init(); + ts.queues[1].init(); + ts.detach(); + } }; const Chain = struct { @@ -41,9 +84,7 @@ const Chain = struct { /// Which queue to block or wakeup threads on. pub const Queue = enum(u1) { - /// Only one waiter will be woken up at once. Exclusive = 0, - /// All waiters will be woken up. Shared = 1, }; @@ -53,6 +94,10 @@ fn hash_obj(obj: *const anyopaque) usize { return (@intFromPtr(obj) >> 6) & hash_mask; } +fn chain_for(obj: *const anyopaque) *Chain { + return &chains[hash_obj(obj)]; +} + /// Highest effective priority among all waiters on `ts`, or 0 if none. /// The caller must hold `ts`'s chain lock so the waiter set is stable. fn highest_waiter(ts: *Turnstile) ?u8 { @@ -69,14 +114,56 @@ fn highest_waiter(ts: *Turnstile) ?u8 { return best; } +/// Walks the threads a turnstile donates to, this is to abstract over the +/// different kinds of ownership. +const Targets = struct { + single: ?Target, + list: ?rtl.List.Iterator, + + fn next(self: *Targets) ?Target { + if (self.single) |t| { + self.single = null; + return t; + } + + if (self.list) |*it| { + if (!it.next()) return null; + const o: *Owner = @fieldParentPtr("link", it.get()); + it.advance(); + return .{ .thread = o.thread, .boost = &o.boost }; + } + + return null; + } +}; + +fn targets(ts: *Turnstile) Targets { + return switch (ts.owners) { + .none => .{ .single = null, .list = null }, + .single => .{ .single = sole_owner(ts), .list = null }, + .shared => |list| .{ .single = null, .list = list.iterator() }, + }; +} + +/// The turnstile's owner when there is exactly one to follow, else null. +fn sole_owner(ts: *Turnstile) ?Target { + return switch (ts.owners) { + .single => .{ + .thread = ts.owners.single.thread, + .boost = &ts.owners.single.boost, + }, + else => null, + }; +} + /// Compute the inherited priority of `td` from the turnstiles it /// owns. `td.lock` must be held. fn recompute_inherited(td: *ke.Thread) void { var pri: u8 = 0; var it = td.turnstiles_owned.iterator(); while (it.next()) : (it.advance()) { - const ts: *Turnstile = @fieldParentPtr("boost_link", it.get()); - pri = @max(pri, ts.donated orelse 0); + const boost: *Boost = @fieldParentPtr("link", it.get()); + pri = @max(pri, boost.donated orelse 0); } td.inherited_prio = pri; } @@ -86,51 +173,74 @@ fn update_prio(td: *ke.Thread) void { if (pri != td.priority) ki.sched.update_priority_locked(td, pri); } -/// Donate priority `pri` to `to` through turnstile `ts`. The caller holds -/// `to.lock` and `ts`'s chain lock. Returns whether `to`'s effective priority -/// actually rose (false means the chain stops here). -fn donate_to(ts: *Turnstile, to: *ke.Thread, pri: u8) bool { - std.debug.assert(ts.owner == to); - +/// Donate priority `pri` to `to` along the edge `boost`. The caller holds +/// `to.lock` and the owning turnstile's chain lock. Returns whether `to`'s +/// priority actually rose (false means the chain stops here). +fn donate_to(boost: *Boost, to: *ke.Thread, pri: u8) bool { // A lend only ever raises. - if (ts.donated != null and pri <= ts.donated.?) return false; + if (boost.donated != null and pri <= boost.donated.?) return false; const before = to.priority; - if (ts.donated == null) to.turnstiles_owned.insert_head(&ts.boost_link); - ts.donated = pri; + if (boost.donated == null) to.turnstiles_owned.insert_head(&boost.link); + boost.donated = pri; if (pri > to.inherited_prio) to.inherited_prio = pri; update_prio(to); return to.priority > before; } -/// Detach `ts`'s donation from whatever thread it is currently boosting and -/// recompute that thread's priority (which may lower it). Caller holds `ts`'s -/// chain lock. -fn revoke(ts: *Turnstile) void { - // Not boosting anyone. - if (ts.donated == null) return; +/// Undo the donation currently boosting `to`. The caller +/// holds the owning turnstile's chain lock. +fn undonate(boost: *Boost, to: *ke.Thread) void { + if (boost.donated == null) return; - const to = ts.owner.?; to.lock.acquire_no_ipl(); - ts.boost_link.remove(); - ts.donated = null; + boost.link.remove(); + boost.donated = null; // Recompute the priority as the floor may drop now. recompute_inherited(to); update_prio(to); to.lock.release_no_ipl(); } -/// (Re)establish `ts`'s donation to `to` from its current set of waiters. -/// Caller holds `ts`'s chain lock. -fn reboost(ts: *Turnstile, to: *ke.Thread) void { - const pri = highest_waiter(ts); - if (pri == null) return; +/// Detach `ts`'s donations from the threads they boost and recompute their +/// priorities. Caller holds `ts`'s chain lock. +fn revoke(ts: *Turnstile) void { + var it = targets(ts); + while (it.next()) |t| undonate(t.boost, t.thread); +} - to.lock.acquire_no_ipl(); - _ = donate_to(ts, to, pri.?); - to.lock.release_no_ipl(); +/// (Re)establish `ts`'s donations from its current set of waiters. +/// Caller holds `ts`'s chain lock. +fn reboost(ts: *Turnstile) void { + const pri = highest_waiter(ts) orelse return; + var it = targets(ts); + while (it.next()) |t| { + t.thread.lock.acquire_no_ipl(); + _ = donate_to(t.boost, t.thread, pri); + t.thread.lock.release_no_ipl(); + } +} + +fn attach(ts: *Turnstile, own: Ownership) void { + ts.owners = switch (own) { + .single => |td| .{ .single = .{ + .thread = td, + .boost = .{ .link = undefined, .donated = null }, + } }, + .shared => |list| .{ .shared = list }, + }; +} + +fn attached(ts: *Turnstile, own: Ownership) bool { + return switch (own) { + .single => |td| if (sole_owner(ts)) |t| t.thread == td else false, + .shared => |list| switch (ts.owners) { + .shared => |cur| cur == list, + else => false, + }, + }; } /// Lend `curtd`'s effective priority down the blocking chain, boosting each @@ -151,7 +261,7 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void { // We hold `thread.lock` (curtd's on the first // pass, the previous owner's after a hop) and `root`, `held` is null. const obj = thread.waiting_on orelse break; - const bucket = &chains[hash_obj(obj)]; + const bucket = chain_for(obj); if (bucket != root) { if (!bucket.lock.try_acquire_no_ipl()) { @@ -175,24 +285,42 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void { std.debug.assert(thread.waiting_on == obj); const ts = thread.turnstile; - const owner = ts.owner orelse break; - if (owner == curtd) @panic("turnstile: cycle in blocking chain"); - owner.lock.acquire_no_ipl(); - if (!donate_to(ts, owner, donate)) { - owner.lock.release_no_ipl(); + if (sole_owner(ts)) |o| { + // For single-owner locks, do multi-hop priority inheritance. + // Multi-hop means we handle the case where a boosted thread itself + // is waiting on another resource. + const owner = o.thread; + if (owner == curtd) @panic("turnstile: cycle in blocking chain"); + + owner.lock.acquire_no_ipl(); + if (!donate_to(o.boost, owner, donate)) { + owner.lock.release_no_ipl(); + break; + } + + // The next hop blocks owner, so lend the owner's priority + // down the chain. + donate = owner.priority; + + // Keep the owner locked, drop everything below it. + thread.lock.release_no_ipl(); + if (held) |h| h.lock.release_no_ipl(); + held = null; + thread = owner; + } else { + // Only do single-hop priority boosting for multiple owners. + // This is fine as we only use this for SMR read sections, + // and they are forbidden to explicitly block. + var it = targets(ts); + while (it.next()) |t| { + if (t.thread == curtd) @panic("turnstile: cycle in blocking chain"); + t.thread.lock.acquire_no_ipl(); + _ = donate_to(t.boost, t.thread, donate); + t.thread.lock.release_no_ipl(); + } break; } - - // The next hop blocks owner, so lend the owner's priority - // down the chain. - donate = owner.priority; - - // Keep the owner locked, drop everything below it. - thread.lock.release_no_ipl(); - if (held) |h| h.lock.release_no_ipl(); - held = null; - thread = owner; } // Leave the caller holding exactly curtd.lock and root. @@ -210,11 +338,27 @@ pub fn init_turnstiles() void { } } +/// Add an owner to a turnstile whose ownership is shared. The caller inserts +/// `o` into the shared list and holds the chain lock. +pub fn owner_enter(ts: *Turnstile, o: *Owner) void { + const pri = highest_waiter(ts) orelse return; + + o.thread.lock.acquire_no_ipl(); + _ = donate_to(&o.boost, o.thread, pri); + o.thread.lock.release_no_ipl(); +} + +/// Revoke one owner's donation. The caller removes `o` from the shared list +/// and holds the chain lock. +pub fn owner_leave(o: *Owner) void { + undonate(&o.boost, o.thread); +} + /// Look up the turnstile for the specified object. /// This acquires the turnstile chain lock and must be called at IPL dispatch. /// Returns null if no turnstile is found. pub fn lookup(obj: *const anyopaque) ?*Turnstile { - const chain = &chains[hash_obj(obj)]; + const chain = chain_for(obj); chain.lock.acquire_no_ipl(); @@ -231,14 +375,17 @@ pub fn lookup(obj: *const anyopaque) ?*Turnstile { /// Drop the lock protecting the chain for obj. pub fn exit(obj: *const anyopaque) void { - chains[hash_obj(obj)].lock.release_no_ipl(); + chain_for(obj).lock.release_no_ipl(); } -/// Block the current thread on a synchronization object backed by a turnstile. -/// This must be called with the appropriate turnstile chain lock held, -/// and returns with the chain lock released. IPL is kept as before on return. -pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, owner: *ke.Thread, queue: Queue) void { - const chain = &chains[hash_obj(obj)]; +/// Block the current thread on a synchronization object backed by a turnstile, +/// donating to the owner set described by `own`. +/// +/// This must be called with the appropriate turnstile chain lock held, and +/// returns with the chain lock released. +/// IPL is kept as before on return. +pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Queue) void { + const chain = chain_for(obj); const curtd = ki.sched.percpu.local().current_thread.?; const ipl = ke.ipl.current(); @@ -254,20 +401,19 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, owner: *ke.Thread, queue: turn.next_free = curtd.turnstile; curtd.turnstile = turn; - if (turn.owner) |cur| { - std.debug.assert(cur == owner); - } else { - // Owner might be null because of a partial wakeup - // where we cleared it, restore it here. - turn.owner = owner; - reboost(turn, owner); + if (!attached(turn, own)) { + // Either a partial wakeup detached the turnstile or the object + // changed ownership, move the donation over. + revoke(turn); + attach(turn, own); + reboost(turn); } } else { // This is the first thread to block on this object. // Lend its turnstile and add it to the hash chain. ts = curtd.turnstile; ts.?.obj = obj; - ts.?.owner = owner; + attach(ts.?, own); chain.list.insert_head(&ts.?.link); } @@ -295,56 +441,34 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, owner: *ke.Thread, queue: // Now block on the event. _ = ke.ipl.lower(.Passive); - _ = ke.wait.wait_one(&waiter.event.hdr, null) catch { - _ = ke.ipl.raise(ipl); - - // If wait_one aborted (e.g. due to a timeout), remove the waiter and fix - // up the owner's inheritance. - chain.lock.acquire_no_ipl(); - - if (curtd.turnstile_waiter == &waiter) { - const active = ts.?; - const holder = active.owner; - revoke(active); - dequeue(active, curtd); - if (active.waiters > 0) { - if (holder) |o| reboost(active, o); - } - } - - chain.lock.release_no_ipl(); - return; - }; + _ = ke.wait.wait_one(&waiter.event.hdr, null) catch unreachable; _ = ke.ipl.raise(ipl); } /// Wake whomever was waiting on the turnstile. /// Do hand-off to new_owner if specified. -pub fn wakeup(turnstile: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread) void { - const chain = &chains[hash_obj(turnstile.obj)]; +/// The chain lock is still held on return. +pub fn wakeup(ts: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread) void { const queue_idx = @intFromEnum(queue); - // The current owner is giving up the object, so it stops inheriting from - // this turnstile entirely. - revoke(turnstile); + // Revoke any priority we gave to the owner(s). + revoke(ts); if (new_owner) |no| { // Hand the object to a specific waiter, the rest now boost it. - dequeue(turnstile, no); - if (turnstile.waiters > 0) { - turnstile.owner = no; - reboost(turnstile, no); + dequeue(ts, no); + if (ts.waiters > 0) { + attach(ts, .{ .single = no }); + reboost(ts); } } else { for (0..count) |_| { - if (turnstile.queues[queue_idx].is_empty()) break; - const waiter: *Waiter = @fieldParentPtr("link", turnstile.queues[queue_idx].first()); - dequeue(turnstile, waiter.thread); + if (ts.queues[queue_idx].is_empty()) break; + const waiter: *Waiter = @fieldParentPtr("link", ts.queues[queue_idx].first()); + dequeue(ts, waiter.thread); } - if (turnstile.waiters > 0) turnstile.owner = null; + if (ts.waiters > 0) ts.detach(); } - - chain.lock.release_no_ipl(); } /// Remove a single waiter from the turnstile and wake it. Caller holds the @@ -365,11 +489,7 @@ fn dequeue(ts: *Turnstile, td: *ke.Thread) void { } else { // Last waiter, pull the turnstile off the chain and keep it for ourselves. ts.link.remove(); - ts.queues[0].init(); - ts.queues[1].init(); - ts.next_free = null; - ts.owner = null; - ts.donated = null; + ts.reset(); } td.turnstile_waiter = null; diff --git a/src/proc/private.zig b/src/proc/private.zig index 0c56ea0..40c42cd 100644 --- a/src/proc/private.zig +++ b/src/proc/private.zig @@ -12,14 +12,11 @@ fn turnstile_ctor(ts: *ki.turnstile.Turnstile) void { .link = undefined, .next_free = null, .obj = undefined, - .owner = null, - .boost_link = undefined, - .donated = null, .queues = undefined, .waiters = 0, + .owners = .none, }; - ts.queues[0].init(); - ts.queues[1].init(); + ts.reset(); } /// Initialize the process subsystem.