diff --git a/kernel-src/lib/pushlock.c b/kernel-src/lib/pushlock.c index c3093718..0cf2b062 100644 --- a/kernel-src/lib/pushlock.c +++ b/kernel-src/lib/pushlock.c @@ -2,284 +2,453 @@ #include #include -#define CAS_ACQUIRE(ptr, saved, new) \ - __atomic_compare_exchange_n(ptr, &saved, new, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED) +// A push lock is exactly one pointer-sized word. +// +// Without PUSHLOCK_FLAGS_CONTENDED, the high bits contain the shared owner +// count: +// +// 0 unlocked +// ACQUIRED exclusive owner +// ACQUIRED | n*SHARED_INC n shared owners +// +// With PUSHLOCK_FLAGS_CONTENDED, the high bits point at the newest stack +// allocated waiter. Waiters link toward older waiters. `newer` and `oldest` are +// populated lazily by the owner of PUSHLOCK_FLAGS_WAKING so releases don't +// repeatedly scan the whole chain. +// +// PUSHLOCK_FLAGS_WAKING allows one thread to update queue links and choose whom +// to wake. It doesn't grant lock ownership. Other threads may still add +// waiters, and an exclusive acquirer may set ACQUIRED while maintenance is in +// progress. Shared releasers may walk the chain while their displaced ownership +// keeps it alive. +// +// If contention begins while more than one shared owner exists, the exact +// displaced reader count is stored in the oldest waiter. A zero waiter count +// means there's no multi-reader count to maintain; for a shared owner in +// contended mode that means it's the sole shared owner. +// +// Every waiter, including its semaphore, is initialized before its address is +// stored in the lock word. The semaphore retains an early signal, so +// semaphore_signal() may safely finish before semaphore_wait() starts. -#define CAS_RELEASE(ptr, saved, new) \ - __atomic_compare_exchange_n(ptr, &saved, new, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED) - -#define PUSHLOCK_FLAGS_ACQUIRED 1 -#define PUSHLOCK_FLAGS_EXCLUSIVE 2 -#define PUSHLOCK_FLAGS_CONTENDED 4 +#define PUSHLOCK_FLAGS_ACQUIRED ((pushlock_t)1u << 0) +#define PUSHLOCK_FLAGS_CONTENDED ((pushlock_t)1u << 1) +#define PUSHLOCK_FLAGS_WAKING ((pushlock_t)1u << 2) +#define PUSHLOCK_SHARED_INC ((pushlock_t)1u << 3) #define PUSHLOCK_MASK_FLAGS \ - (PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE | PUSHLOCK_FLAGS_CONTENDED) + (PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_CONTENDED | PUSHLOCK_FLAGS_WAKING) #define PUSHLOCK_VALUE_ALIGNMENT (PUSHLOCK_MASK_FLAGS + 1) -#define PUSHLOCK_MASK_POINTER_SHARED_COUNT (~(pushlock_t)PUSHLOCK_MASK_FLAGS) +#define PUSHLOCK_MASK_POINTER_SHARED_COUNT (~PUSHLOCK_MASK_FLAGS) -_Static_assert( - (PUSHLOCK_VALUE_ALIGNMENT & (PUSHLOCK_VALUE_ALIGNMENT - 1)) == 0, - "push lock flags must occupy contiguous low bits"); +#define PUSHLOCK_GET_POINTER(state) \ + ((pushlock_wait_block_t *)((state) & PUSHLOCK_MASK_POINTER_SHARED_COUNT)) + +#define PUSHLOCK_GET_SHARED_COUNT(state) \ + ((size_t)((state) / PUSHLOCK_SHARED_INC)) typedef struct pushlock_wait_block_t { - struct pushlock_wait_block_t *next; + struct pushlock_wait_block_t *older; + struct pushlock_wait_block_t *newer; + struct pushlock_wait_block_t *oldest; size_t shared_count; - bool exclusive; semaphore_t semaphore; + bool exclusive; } __attribute__((aligned(PUSHLOCK_VALUE_ALIGNMENT))) pushlock_wait_block_t; -#define PUSHLOCK_GET_POINTER(x) \ - ((pushlock_wait_block_t *)((x) & PUSHLOCK_MASK_POINTER_SHARED_COUNT)) +_Static_assert(sizeof(pushlock_t) == sizeof(uintptr_t), + "push locks must fit in one uintptr_t"); +_Static_assert(PUSHLOCK_VALUE_ALIGNMENT == 8, + "push lock flags must occupy exactly the low three bits"); +_Static_assert(PUSHLOCK_SHARED_INC == PUSHLOCK_VALUE_ALIGNMENT, + "shared count must start above the pointer tag bits"); +_Static_assert(_Alignof(pushlock_wait_block_t) >= PUSHLOCK_VALUE_ALIGNMENT, + "pushlock waiters must preserve the low three pointer bits"); -#define PUSHLOCK_GET_SHARED_COUNT(x) \ - (((pushlock_t)(x) & PUSHLOCK_MASK_POINTER_SHARED_COUNT) / PUSHLOCK_VALUE_ALIGNMENT) +static inline pushlock_t pushlock_load(const pushlock_t *pushlock) { + return __atomic_load_n(pushlock, __ATOMIC_ACQUIRE); +} -#define PUSHLOCK_INCREMENT_SHARED_COUNT(x) \ - ((x) + PUSHLOCK_VALUE_ALIGNMENT) +static inline bool pushlock_cas_acquire(pushlock_t *pushlock, pushlock_t *expected, + pushlock_t desired) { + return __atomic_compare_exchange_n(pushlock, expected, desired, false, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE); +} -#define PUSHLOCK_DECREMENT_SHARED_COUNT(x) \ - ((x) - PUSHLOCK_VALUE_ALIGNMENT) +static inline bool pushlock_cas_acq_rel(pushlock_t *pushlock, pushlock_t *expected, + pushlock_t desired) { + return __atomic_compare_exchange_n(pushlock, expected, desired, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE); +} -bool pushlock_try_acquire_exclusive(pushlock_t *pushlock) { -retry_lock: - pushlock_t saved_value = 0; - if (CAS_ACQUIRE(pushlock, saved_value, PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) - return true; +static inline void pushlock_assert_state(pushlock_t state) { +#ifdef DEBUG_LOCKS + if ((state & PUSHLOCK_FLAGS_CONTENDED) == 0) { + __assert((state & PUSHLOCK_FLAGS_WAKING) == 0); + if ((state & PUSHLOCK_FLAGS_ACQUIRED) == 0) + __assert(state == 0); + } else { + __assert(PUSHLOCK_GET_POINTER(state) != NULL); + } +#else + (void)state; +#endif +} - if (saved_value & PUSHLOCK_FLAGS_ACQUIRED) +// Shared ownership keeps the chain alive. Only read links here as another CPU +// may hold WAKING and be completing the reverse links. +static pushlock_wait_block_t *pushlock_find_oldest(pushlock_wait_block_t *waiter) { + for (;;) { + pushlock_wait_block_t *cached = __atomic_load_n(&waiter->oldest, __ATOMIC_ACQUIRE); + if (cached != NULL) + return cached; + + waiter = waiter->older; + __assert(waiter != NULL); + } +} + +// Called with WAKING held. Stop at the portion already linked by an earlier +// pass, then cache its oldest entry at the current head. +static pushlock_wait_block_t *pushlock_complete_waiter_links(pushlock_wait_block_t *head) { + pushlock_wait_block_t *waiter = head; + + for (;;) { + pushlock_wait_block_t *cached = __atomic_load_n(&waiter->oldest, __ATOMIC_ACQUIRE); + if (cached != NULL) { + if (waiter != head) + __atomic_store_n(&head->oldest, cached, __ATOMIC_RELEASE); + return cached; + } + + pushlock_wait_block_t *older = waiter->older; + __assert(older != NULL); + + older->newer = waiter; + waiter = older; + } +} + +// Called by the thread that acquired WAKING. Clear it while ACQUIRED remains +// set or wake waiters after observing ACQUIRED clear. +static void pushlock_update_waiters(pushlock_t *pushlock, pushlock_t state) { + pushlock_wait_block_t *head; + pushlock_wait_block_t *oldest; + pushlock_wait_block_t *new_oldest; + pushlock_t desired; + pushlock_t previous; + + for (;;) { + pushlock_assert_state(state); + __assert(state & PUSHLOCK_FLAGS_CONTENDED); + __assert(state & PUSHLOCK_FLAGS_WAKING); + + head = PUSHLOCK_GET_POINTER(state); + oldest = pushlock_complete_waiter_links(head); + + if (state & PUSHLOCK_FLAGS_ACQUIRED) { + // Recheck both the head and ACQUIRED. A release may leave wakeup to us + // while a newly inserted head still needs its links completed. + desired = state & ~PUSHLOCK_FLAGS_WAKING; + if (pushlock_cas_acq_rel(pushlock, &state, desired)) + return; + continue; + } + + new_oldest = oldest->newer; + +#ifdef DEBUG_LOCKS + __assert(oldest->older == NULL); + __assert(__atomic_load_n(&oldest->shared_count, __ATOMIC_ACQUIRE) == 0); + __assert((oldest == head) == (new_oldest == NULL)); +#endif + + if (oldest->exclusive && new_oldest != NULL) { +#ifdef DEBUG_LOCKS + __assert(new_oldest->older == oldest); +#endif + // No shared releaser remains after ACQUIRED clears. Cut the old + // tail and replace before the waiter can leave its stack. + new_oldest->older = NULL; + __atomic_store_n(&new_oldest->oldest, new_oldest, __ATOMIC_RELEASE); + + if (head != new_oldest) + __atomic_store_n(&head->oldest, new_oldest, __ATOMIC_RELEASE); + + oldest->newer = NULL; + + // Preserve a concurrently inserted head or exclusive acquisition + previous = __atomic_fetch_and(pushlock, ~PUSHLOCK_FLAGS_WAKING, + __ATOMIC_RELEASE); + (void)previous; +#ifdef DEBUG_LOCKS + __assert(previous & PUSHLOCK_FLAGS_WAKING); + __assert(previous & PUSHLOCK_FLAGS_CONTENDED); +#endif + semaphore_signal(&oldest->semaphore); + return; + } + + // Remove the queue only if its head and the unlocked state still match + if (!pushlock_cas_acq_rel(pushlock, &state, 0)) + continue; + + // Read newer before signalling since the waiter may return immediately + while (oldest != NULL) { + new_oldest = oldest->newer; + semaphore_signal(&oldest->semaphore); + oldest = new_oldest; + } + + return; + } +} + +// With CONTENDED set, the high bits hold a waiter pointer instead of a +// shared count. New readers queue rather than becoming an uncounted owner. +static inline bool pushlock_shared_acquire_value(pushlock_t state, pushlock_t *desired) { + if (state & PUSHLOCK_FLAGS_CONTENDED) return false; - if (!CAS_ACQUIRE(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) - goto retry_lock; + if ((state & PUSHLOCK_FLAGS_ACQUIRED) == 0) { + *desired = state | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_SHARED_INC; + return true; + } - return true; + if (PUSHLOCK_GET_SHARED_COUNT(state) != 0) { +#ifdef DEBUG_LOCKS + __assert(state <= UINTPTR_MAX - PUSHLOCK_SHARED_INC); +#endif + *desired = state + PUSHLOCK_SHARED_INC; + return true; + } + + return false; +} + +static pushlock_t pushlock_queue_and_wait(pushlock_t *pushlock, pushlock_t state, bool exclusive) { + pushlock_wait_block_t waiter; + pushlock_t desired; + size_t readers; + bool acquired_waking; + int result; + + waiter.newer = NULL; + waiter.exclusive = exclusive; + // A releaser may signal as soon as the publication CMPXCHG succeeds + SEMAPHORE_INIT(&waiter.semaphore, 0); + +#ifdef DEBUG_LOCKS + __assert(((uintptr_t)&waiter & PUSHLOCK_MASK_FLAGS) == 0); +#endif + + for (;;) { + acquired_waking = false; + + pushlock_assert_state(state); + if (exclusive ? (state & PUSHLOCK_FLAGS_ACQUIRED) == 0 : pushlock_shared_acquire_value(state, &desired)) + return state; + // A failed CMPXCHG may return a different queue head, so rebuild from state + waiter.older = NULL; + __atomic_store_n(&waiter.oldest, NULL, __ATOMIC_RELAXED); + __atomic_store_n(&waiter.shared_count, 0, __ATOMIC_RELAXED); + + if (state & PUSHLOCK_FLAGS_CONTENDED) { + // Link this waiter to the current queue head before publishing it + waiter.older = PUSHLOCK_GET_POINTER(state); + desired = (pushlock_t)(uintptr_t)&waiter | + (state & PUSHLOCK_MASK_FLAGS) | + PUSHLOCK_FLAGS_CONTENDED | PUSHLOCK_FLAGS_WAKING; + acquired_waking = !(state & PUSHLOCK_FLAGS_WAKING); + } else { + readers = PUSHLOCK_GET_SHARED_COUNT(state); +#ifdef DEBUG_LOCKS + if (!exclusive) + __assert(readers == 0); +#endif + + __atomic_store_n(&waiter.oldest, &waiter, __ATOMIC_RELAXED); + desired = (pushlock_t)(uintptr_t)&waiter | + PUSHLOCK_FLAGS_ACQUIRED | + PUSHLOCK_FLAGS_CONTENDED; + + // Multiple existing readers need their count moved into the wait block + if (readers > 1) + __atomic_store_n(&waiter.shared_count, readers, __ATOMIC_RELAXED); + } + + if (!pushlock_cas_acq_rel(pushlock, &state, desired)) + continue; + + if (acquired_waking) + pushlock_update_waiters(pushlock, desired); + + result = semaphore_wait(&waiter.semaphore, false); +#ifdef DEBUG_LOCKS + __assert(result == 0); +#endif + (void)result; + return pushlock_load(pushlock); + } +} + +bool pushlock_try_acquire_exclusive(pushlock_t *pushlock) { + pushlock_t state = 0; + + if (pushlock_cas_acquire(pushlock, &state, PUSHLOCK_FLAGS_ACQUIRED)) + return true; + + for (;;) { + pushlock_assert_state(state); + + if (state & PUSHLOCK_FLAGS_ACQUIRED) + return false; + + pushlock_t desired = state | PUSHLOCK_FLAGS_ACQUIRED; + if (pushlock_cas_acquire(pushlock, &state, desired)) + return true; + } } void pushlock_acquire_exclusive(pushlock_t *pushlock) { -retry_lock: - pushlock_t saved_value = 0; - if (CAS_ACQUIRE(pushlock, saved_value, PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) + pushlock_t state = 0; + + if (pushlock_cas_acquire(pushlock, &state, PUSHLOCK_FLAGS_ACQUIRED)) return; - if ((saved_value & PUSHLOCK_FLAGS_ACQUIRED) == 0) { - if (!CAS_ACQUIRE(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) - goto retry_lock; + for (;;) { + pushlock_assert_state(state); - return; + if ((state & PUSHLOCK_FLAGS_ACQUIRED) == 0) { + pushlock_t desired = state | PUSHLOCK_FLAGS_ACQUIRED; + if (pushlock_cas_acquire(pushlock, &state, desired)) + return; + + continue; + } + + state = pushlock_queue_and_wait(pushlock, state, true); } - - pushlock_wait_block_t wait_block; - SEMAPHORE_INIT(&wait_block.semaphore, 0); - wait_block.exclusive = true; - - if (saved_value & PUSHLOCK_FLAGS_CONTENDED) { - wait_block.next = PUSHLOCK_GET_POINTER(saved_value); - __atomic_store_n(&wait_block.shared_count, 0, __ATOMIC_RELAXED); - } else { - wait_block.next = NULL; - __atomic_store_n(&wait_block.shared_count, PUSHLOCK_GET_SHARED_COUNT(saved_value), __ATOMIC_RELAXED); - } - - if (CAS_RELEASE(pushlock, saved_value, (uintptr_t)&wait_block | (saved_value & PUSHLOCK_MASK_FLAGS) | PUSHLOCK_FLAGS_CONTENDED)) - semaphore_wait(&wait_block.semaphore, false); - - goto retry_lock; } bool pushlock_try_acquire_shared(pushlock_t *pushlock) { -retry_lock: - pushlock_t saved_value = 0; - if (CAS_ACQUIRE(pushlock, saved_value, PUSHLOCK_INCREMENT_SHARED_COUNT(PUSHLOCK_FLAGS_ACQUIRED))) + pushlock_t state = 0; + + if (pushlock_cas_acquire(pushlock, &state, + PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_SHARED_INC)) return true; - if (saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) - return false; + for (;;) { + pushlock_assert_state(state); - if ((saved_value & PUSHLOCK_FLAGS_CONTENDED) == 0) { - if (!CAS_ACQUIRE(pushlock, saved_value, PUSHLOCK_INCREMENT_SHARED_COUNT(saved_value))) - goto retry_lock; + pushlock_t desired; + if (!pushlock_shared_acquire_value(state, &desired)) + return false; - return true; + if (pushlock_cas_acquire(pushlock, &state, desired)) + return true; } - - if (saved_value & PUSHLOCK_FLAGS_ACQUIRED) - return false; - - // The waiter pointer occupies the shared count field, so acquire as a single owner while contended. - if (!CAS_ACQUIRE(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) - goto retry_lock; - - return true; } void pushlock_acquire_shared(pushlock_t *pushlock) { -retry_lock: - pushlock_t saved_value = 0; - if (CAS_ACQUIRE(pushlock, saved_value, PUSHLOCK_INCREMENT_SHARED_COUNT(PUSHLOCK_FLAGS_ACQUIRED))) + pushlock_t state = 0; + + if (pushlock_cas_acquire(pushlock, &state, + PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_SHARED_INC)) return; - if ((saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) || ((saved_value & PUSHLOCK_FLAGS_ACQUIRED) && (saved_value & PUSHLOCK_FLAGS_CONTENDED))) { - pushlock_wait_block_t wait_block; - SEMAPHORE_INIT(&wait_block.semaphore, 0); - wait_block.exclusive = false; - - wait_block.next = PUSHLOCK_GET_POINTER(saved_value); - __atomic_store_n(&wait_block.shared_count, 0, __ATOMIC_RELAXED); - - if (CAS_RELEASE(pushlock, saved_value, (uintptr_t)&wait_block | (saved_value & PUSHLOCK_MASK_FLAGS) | PUSHLOCK_FLAGS_CONTENDED)) - semaphore_wait(&wait_block.semaphore, false); - - goto retry_lock; - } - - if (saved_value & PUSHLOCK_FLAGS_ACQUIRED) { - if (!CAS_ACQUIRE(pushlock, saved_value, PUSHLOCK_INCREMENT_SHARED_COUNT(saved_value))) - goto retry_lock; - - return; - } - - if (!CAS_ACQUIRE(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) - goto retry_lock; -} - -static void pushlock_signal_waiters(pushlock_wait_block_t *wait_block) { - pushlock_wait_block_t *next_wait_block; - pushlock_wait_block_t *previous_wait_block = NULL; - - // Reverse the detached chain to signal older waiters first. - while (wait_block) { - next_wait_block = wait_block->next; - wait_block->next = previous_wait_block; - previous_wait_block = wait_block; - wait_block = next_wait_block; - } - - wait_block = previous_wait_block; - - while (wait_block) { - // The wait block is stack allocated, so save next before waking. - next_wait_block = wait_block->next; - semaphore_signal(&wait_block->semaphore); - wait_block = next_wait_block; - } -} - -static void pushlock_wake_waiters(pushlock_t *pushlock) { - pushlock_t saved_value; - pushlock_t new_value; - pushlock_wait_block_t *head_wait_block; - pushlock_wait_block_t *penultimate_wait_block; - pushlock_wait_block_t *oldest_wait_block; - for (;;) { - // Wait blocks are published with release semantics. Acquire - // the current head before traversing the chain. - saved_value = __atomic_load_n(pushlock, __ATOMIC_ACQUIRE); + pushlock_assert_state(state); + pushlock_t desired; + if (pushlock_shared_acquire_value(state, &desired)) { + if (pushlock_cas_acquire(pushlock, &state, desired)) + return; + + continue; + } + + state = pushlock_queue_and_wait(pushlock, state, false); + } +} + +// Clear ACQUIRED and set WAKING in one CMPXCHG. If WAKING was already set, +// the thread processing the queue will handle the release. +static void pushlock_release_contended(pushlock_t *pushlock, pushlock_t state) { + pushlock_t desired; + bool acquired_waking; + + for (;;) { + pushlock_assert_state(state); #ifdef DEBUG_LOCKS - __assert(saved_value & PUSHLOCK_FLAGS_ACQUIRED); - __assert(saved_value & PUSHLOCK_FLAGS_CONTENDED); + __assert(state & PUSHLOCK_FLAGS_ACQUIRED); + __assert(state & PUSHLOCK_FLAGS_CONTENDED); #endif + acquired_waking = !(state & PUSHLOCK_FLAGS_WAKING); + desired = state & ~PUSHLOCK_FLAGS_ACQUIRED; + if (acquired_waking) + desired |= PUSHLOCK_FLAGS_WAKING; - head_wait_block = PUSHLOCK_GET_POINTER(saved_value); - penultimate_wait_block = NULL; - oldest_wait_block = head_wait_block; + if (!pushlock_cas_acq_rel(pushlock, &state, desired)) + continue; - while (oldest_wait_block->next) { - penultimate_wait_block = oldest_wait_block; - oldest_wait_block = oldest_wait_block->next; - } - - // Wake an exclusive oldest waiter alone, leaving newer waiters queued. - if (oldest_wait_block->exclusive && penultimate_wait_block != NULL) { - // New waiters can only be inserted at the head, so they - // can't modify this tail link. - penultimate_wait_block->next = NULL; - - for (;;) { - new_value = - (saved_value & PUSHLOCK_MASK_POINTER_SHARED_COUNT) | - PUSHLOCK_FLAGS_CONTENDED; - - if (CAS_RELEASE(pushlock, saved_value, new_value)) { - break; - } - - // A waiter was inserted at the head. The failed CAS - // updated saved_value, while the detached tail stays - // unchanged. - } - - semaphore_signal(&oldest_wait_block->semaphore); - return; - } - - // A single waiter, or a chain whose oldest waiter is shared, can be - // detached completely. Woken waiters retry acquisition without handoff. - if (CAS_RELEASE(pushlock, saved_value, 0)) { - pushlock_signal_waiters(head_wait_block); - return; - } - - // The chain changed before it could be detached. Restart with - // an acquire load before traversing it again. + if (acquired_waking) + pushlock_update_waiters(pushlock, desired); + return; } } void pushlock_release_exclusive(pushlock_t *pushlock) { - pushlock_t saved_value; + pushlock_t state = PUSHLOCK_FLAGS_ACQUIRED; - saved_value = PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE; - if (CAS_RELEASE(pushlock, saved_value, 0)) + if (pushlock_cas_acq_rel(pushlock, &state, 0)) return; -#ifdef DEBUG_LOCKS - __assert(saved_value & PUSHLOCK_FLAGS_ACQUIRED); - __assert(saved_value & PUSHLOCK_FLAGS_EXCLUSIVE); -#endif - - pushlock_wake_waiters(pushlock); -} - -static void pushlock_release_shared_contended(pushlock_t *pushlock, pushlock_t saved_value) { - pushlock_wait_block_t *oldest_wait_block; - - oldest_wait_block = PUSHLOCK_GET_POINTER(saved_value); - - while (oldest_wait_block->next) - oldest_wait_block = oldest_wait_block->next; - - if (__atomic_sub_fetch(&oldest_wait_block->shared_count, 1, __ATOMIC_RELEASE)) - return; - - __atomic_thread_fence(__ATOMIC_ACQUIRE); - - pushlock_wake_waiters(pushlock); + pushlock_release_contended(pushlock, state); } void pushlock_release_shared(pushlock_t *pushlock) { -retry_fast_path: - pushlock_t saved_value = PUSHLOCK_INCREMENT_SHARED_COUNT(PUSHLOCK_FLAGS_ACQUIRED); - if (CAS_RELEASE(pushlock, saved_value, 0)) + pushlock_t state = PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_SHARED_INC; + + if (pushlock_cas_acq_rel(pushlock, &state, 0)) return; + for (;;) { + pushlock_assert_state(state); #ifdef DEBUG_LOCKS - __assert(saved_value & PUSHLOCK_FLAGS_ACQUIRED); + __assert(state & PUSHLOCK_FLAGS_ACQUIRED); #endif - if (saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) { - pushlock_release_exclusive(pushlock); + if ((state & PUSHLOCK_FLAGS_CONTENDED) == 0) { + size_t readers = PUSHLOCK_GET_SHARED_COUNT(state); +#ifdef DEBUG_LOCKS + __assert(readers != 0); +#endif + pushlock_t desired = readers > 1 ? state - PUSHLOCK_SHARED_INC : 0; + if (pushlock_cas_acq_rel(pushlock, &state, desired)) + return; + continue; + } + + // The oldest wait block stays valid until this reader drops its count + pushlock_wait_block_t *oldest = pushlock_find_oldest(PUSHLOCK_GET_POINTER(state)); + size_t displaced = __atomic_load_n(&oldest->shared_count, __ATOMIC_ACQUIRE); + + if (displaced != 0) { + size_t previous = __atomic_fetch_sub(&oldest->shared_count, 1, __ATOMIC_ACQ_REL); + +#ifdef DEBUG_LOCKS + __assert(previous != 0); +#endif + + if (previous != 1) + return; + + state = pushlock_load(pushlock); + } + + pushlock_release_contended(pushlock, state); return; } - - if ((saved_value & PUSHLOCK_FLAGS_CONTENDED)) { - __atomic_thread_fence(__ATOMIC_ACQUIRE); - pushlock_release_shared_contended(pushlock, saved_value); - return; - } - - if (!CAS_RELEASE(pushlock, saved_value, PUSHLOCK_DECREMENT_SHARED_COUNT(saved_value))) - goto retry_fast_path; }