From c47f0e5ea7da2ac010d347650a4dca32553a36fa Mon Sep 17 00:00:00 2001 From: Matheus Date: Sat, 28 Dec 2024 00:16:42 -0300 Subject: [PATCH] pushlock: initial implementation --- kernel-src/include/mutex.h | 17 ++-- kernel-src/include/pushlock.h | 31 +++++++ kernel-src/io/acpi/glue.c | 10 ++- kernel-src/lib/pushlock.c | 157 ++++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 12 deletions(-) create mode 100644 kernel-src/include/pushlock.h create mode 100644 kernel-src/lib/pushlock.c diff --git a/kernel-src/include/mutex.h b/kernel-src/include/mutex.h index 1e513a3f..4f777af0 100644 --- a/kernel-src/include/mutex.h +++ b/kernel-src/include/mutex.h @@ -1,25 +1,22 @@ #ifndef _MUTEX_H #define _MUTEX_H -#include +#include -typedef semaphore_t mutex_t; +typedef pushlock_t mutex_t; #define MUTEX_INIT(m) \ - SEMAPHORE_INIT(m, 1); + *m = 0; #define MUTEX_ACQUIRE(m) \ - semaphore_wait(m, false) - -#define MUTEX_ACQUIRE_TIMED(m, t) \ - semaphore_timedwait(m, t, false) + pushlock_acquire_exclusive(m) #define MUTEX_RELEASE(m) \ - semaphore_signal(m) + pushlock_release(m) #define MUTEX_TRY(m) \ - semaphore_test(m) + pushlock_try_acquire_exclusive(m) -#define MUTEX_DEFINE(x) SEMAPHORE_DEFINE(x, 1) +#define MUTEX_DEFINE(x) pushlock_t x = 0; #endif diff --git a/kernel-src/include/pushlock.h b/kernel-src/include/pushlock.h new file mode 100644 index 00000000..424f3148 --- /dev/null +++ b/kernel-src/include/pushlock.h @@ -0,0 +1,31 @@ +#ifndef _RWLOCK_H +#define _RWLOCK_H + +#include + +typedef struct pushlock_wait_block_t { + struct pushlock_wait_block_t *next; + size_t shared_count; + semaphore_t semaphore; +} __attribute__((aligned(8))) pushlock_wait_block_t; + +typedef uintptr_t pushlock_t; + +#define PUSHLOCK_FLAGS_ACQUIRED 1 +#define PUSHLOCK_FLAGS_EXCLUSIVE 2 +#define PUSHLOCK_FLAGS_CONTENDED 4 +#define PUSHLOCK_MASK_FLAGS 0x7 +#define PUSHLOCK_MASK_POINTER_SHARED_COUNT (0xfffffffffffffff8lu) + +#define PUSHLOCK_GET_POINTER(x) ((pushlock_wait_block_t *)((x) & PUSHLOCK_MASK_POINTER_SHARED_COUNT)) +#define PUSHLOCK_GET_SHARED_COUNT(x) ((uintptr_t)PUSHLOCK_GET_POINTER(x) >> 3) +#define PUSHLOCK_INCREMENT_SHARED_COUNT(x) ((x) + 0x8) +#define PUSHLOCK_DECREMENT_SHARED_COUNT(x) ((x) - 0x8) + +bool pushlock_try_acquire_exclusive(pushlock_t *pushlock); +void pushlock_acquire_exclusive(pushlock_t *pushlock); +bool pushlock_try_acquire_shared(pushlock_t *pushlock); +void pushlock_acquire_shared(pushlock_t *pushlock); +void pushlock_release(pushlock_t *pushlock); + +#endif diff --git a/kernel-src/io/acpi/glue.c b/kernel-src/io/acpi/glue.c index 4cfa8e6f..3049a725 100644 --- a/kernel-src/io/acpi/glue.c +++ b/kernel-src/io/acpi/glue.c @@ -461,8 +461,14 @@ uacpi_status uacpi_kernel_acquire_mutex( return UACPI_STATUS_OK; } - bool didacquire = MUTEX_ACQUIRE_TIMED(mut, timeout * 1000); - return didacquire ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT; + for (int i = 0; i < timeout; ++i) { + if (MUTEX_TRY(mut)) + return UACPI_STATUS_OK; + + sched_sleep_us(1000); + } + + return UACPI_STATUS_TIMEOUT; } void uacpi_kernel_release_mutex(uacpi_handle mut) { diff --git a/kernel-src/lib/pushlock.c b/kernel-src/lib/pushlock.c new file mode 100644 index 00000000..232abe70 --- /dev/null +++ b/kernel-src/lib/pushlock.c @@ -0,0 +1,157 @@ +#include + +#define DO_CAS(ptr, saved, new) \ + __atomic_compare_exchange_n(ptr, &saved, new, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) + +bool pushlock_try_acquire_exclusive(pushlock_t *pushlock) { + pushlock_t saved_value = *pushlock; + for (;;) { + if ((PUSHLOCK_FLAGS_ACQUIRED & saved_value) == 0) { + if (DO_CAS(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) + return true; + + continue; + } + + return false; + } +} + +void pushlock_acquire_exclusive(pushlock_t *pushlock) { + pushlock_t saved_value = *pushlock; + for (;;) { + if ((PUSHLOCK_FLAGS_ACQUIRED & saved_value) == 0) { + if (DO_CAS(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) + return; + + continue; + } + + + pushlock_wait_block_t wait_block; + SEMAPHORE_INIT(&wait_block.semaphore, 0); + + if (saved_value & PUSHLOCK_FLAGS_CONTENDED) { + wait_block.next = PUSHLOCK_GET_POINTER(saved_value); + wait_block.shared_count = 0; + } else { + wait_block.next = NULL; + wait_block.shared_count = PUSHLOCK_GET_SHARED_COUNT(saved_value); + } + + if (DO_CAS(pushlock, saved_value, (uintptr_t)&wait_block | (saved_value & PUSHLOCK_MASK_FLAGS) | PUSHLOCK_FLAGS_CONTENDED)) + semaphore_wait(&wait_block.semaphore, false); + } +} + +bool pushlock_try_acquire_shared(pushlock_t *pushlock) { + pushlock_t saved_value = *pushlock; + for (;;) { + if ((saved_value & PUSHLOCK_FLAGS_ACQUIRED) == 0 || (saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) == 0) { + if (saved_value & PUSHLOCK_FLAGS_CONTENDED) { + // if its contended, we will try to acquire it as an exclusive lock + // (as we have no space for putting the shared count in the pointer) + if (DO_CAS(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) + return true; + } else { + // its not contended and either unlocked or locked shared + // set the appropriate flag and increment the shared count + if (DO_CAS(pushlock, saved_value, PUSHLOCK_INCREMENT_SHARED_COUNT(saved_value) | PUSHLOCK_FLAGS_ACQUIRED)) + return true; + } + + continue; + } + + return false; + } +} + +void pushlock_acquire_shared(pushlock_t *pushlock) { + pushlock_t saved_value = *pushlock; + for (;;) { + if ((saved_value & PUSHLOCK_FLAGS_ACQUIRED) == 0 || (saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) == 0) { + if (saved_value & PUSHLOCK_FLAGS_CONTENDED) { + // if its contended, we will try to acquire it as an exclusive lock + // (as we have no space for putting the shared count in the pointer) + if (DO_CAS(pushlock, saved_value, saved_value | PUSHLOCK_FLAGS_ACQUIRED | PUSHLOCK_FLAGS_EXCLUSIVE)) + return; + } else { + // its not contended and either unlocked or locked shared + // set the appropriate flag and increment the shared count + if (DO_CAS(pushlock, saved_value, PUSHLOCK_INCREMENT_SHARED_COUNT(saved_value) | PUSHLOCK_FLAGS_ACQUIRED)) + return; + } + + continue; + } + + pushlock_wait_block_t wait_block; + SEMAPHORE_INIT(&wait_block.semaphore, 0); + + // lock is being exclusively held, which means the bits for shared_count + // is the pointer to the first wait block, so we can directly get it here + wait_block.next = PUSHLOCK_GET_POINTER(saved_value); + wait_block.shared_count = 0; + + if (DO_CAS(pushlock, saved_value, (uintptr_t)&wait_block | (saved_value & PUSHLOCK_MASK_FLAGS) | PUSHLOCK_FLAGS_CONTENDED)) + semaphore_wait(&wait_block.semaphore, false); + } +} + +void pushlock_release(pushlock_t *pushlock) { + pushlock_t saved_value = *pushlock; + for (;;) { + if ((saved_value & PUSHLOCK_FLAGS_CONTENDED) == 0) { + if ((saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) == 0 && PUSHLOCK_GET_SHARED_COUNT(saved_value) > 1) { + // shared lock with 2 or more holders, decrement shared count + if (DO_CAS(pushlock, saved_value, PUSHLOCK_DECREMENT_SHARED_COUNT(saved_value))) + return; + } else { + // shared lock with 1 holder or exclusive lock, can just set it to 0 + if (DO_CAS(pushlock, saved_value, 0)) + return; + } + + continue; + } + + break; + } + + // lock is contended, we will have to go through the wait blocks and get the + // last and penultimate ones + bool skip_check = false; + for (;;) { + pushlock_wait_block_t *prev = NULL, *last = PUSHLOCK_GET_POINTER(saved_value); + while (last->next != NULL) { + prev = last; + last = last->next; + } + + // return if its shared and we arent the only holder + if (skip_check == false && (saved_value & PUSHLOCK_FLAGS_EXCLUSIVE) == 0 && __atomic_sub_fetch(&last->shared_count, 1, __ATOMIC_SEQ_CST) > 0) + return; + + skip_check = true; + + // here we can be assured we are the only holder + if (prev == NULL) { + // last is being pointed to as the front, we just have to unset it + if (DO_CAS(pushlock, saved_value, 0) == false) + continue; + } else { + // we need to remove it from prev + prev->next = NULL; + // and unlock it properly, keeping in mind it still contended + for (;;) { + if (DO_CAS(pushlock, saved_value, (uintptr_t)PUSHLOCK_GET_POINTER(saved_value) | PUSHLOCK_FLAGS_CONTENDED)) + break; + } + } + + // wake the waiting thread + semaphore_signal(&last->semaphore); + return; + } +}