From 4c781f31ade7a72ce49810966f2e1f28d56a2677 Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Wed, 14 Oct 2026 00:05:32 +0200 Subject: [PATCH] replace stdatomic.h with zinnia/atomic.h --- arch/x86_64/include/x86_64/smp.h | 2 +- arch/x86_64/smp.c | 6 +- core/mem/virt.c | 5 +- core/percpu.c | 4 +- core/sched/sched_dumb.c | 8 +- core/spin.c | 10 +- include/stdatomic.h | 163 ------------------------------- include/zinnia/atomic.h | 67 +++++++++++++ include/zinnia/dpc.h | 2 +- include/zinnia/refcount.h | 10 +- include/zinnia/sched.h | 2 +- include/zinnia/waitq.h | 2 +- 12 files changed, 92 insertions(+), 189 deletions(-) delete mode 100644 include/stdatomic.h create mode 100644 include/zinnia/atomic.h diff --git a/arch/x86_64/include/x86_64/smp.h b/arch/x86_64/include/x86_64/smp.h index 8dd3025..fa8145b 100644 --- a/arch/x86_64/include/x86_64/smp.h +++ b/arch/x86_64/include/x86_64/smp.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include #include #include diff --git a/arch/x86_64/smp.c b/arch/x86_64/smp.c index b1e9d48..ee1d4b0 100644 --- a/arch/x86_64/smp.c +++ b/arch/x86_64/smp.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -172,7 +172,7 @@ static void ap_entry(phys_t info_addr) { // Let the BSP know that we're alive. // `info` must not be touched after this point. - atomic_store_explicit(&info->booted, 1, memory_order_release); + atomic_store(&info->booted, 1, ATOMIC_RELEASE); log("CPU %zu online\n", cpu->id); @@ -231,7 +231,7 @@ static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) { ); clock_spin_ns(10'000'000); - for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) { + for (uint64_t waited = 0; !atomic_load(&info->booted, ATOMIC_ACQUIRE); waited += 1'000'000) { if (waited >= AP_BOOT_TIMEOUT_NS) { // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up. log_err("CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id); diff --git a/core/mem/virt.c b/core/mem/virt.c index 61f9e4d..0a82d78 100644 --- a/core/mem/virt.c +++ b/core/mem/virt.c @@ -1,7 +1,7 @@ #include +#include #include #include -#include #include struct page* vm_pfndb = nullptr; @@ -12,8 +12,7 @@ static _Atomic(size_t) mapping_offset = 0; uintptr_t vm_alloc(size_t length) { uintptr_t base = arch_mem_mapping_addr(); - size_t offset = - atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire); + size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE); return base + offset; } diff --git a/core/percpu.c b/core/percpu.c index 9740583..069f483 100644 --- a/core/percpu.c +++ b/core/percpu.c @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include extern uint8_t __ld_percpu_start[]; extern uint8_t __ld_percpu_end[]; @@ -37,7 +37,7 @@ struct percpu* percpu_new() { struct percpu* cpu = HHDM_PTR(mem); cpu->self = cpu; - cpu->id = atomic_fetch_add_explicit(&num_cpus, 1, memory_order_relaxed); + cpu->id = atomic_fetch_add(&num_cpus, 1, ATOMIC_RELAXED); cpu->online = false; return cpu; } diff --git a/core/sched/sched_dumb.c b/core/sched/sched_dumb.c index 5e525e1..9d9be81 100644 --- a/core/sched/sched_dumb.c +++ b/core/sched/sched_dumb.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include #include -#include #include static void idle_func(uintptr_t) { @@ -113,7 +113,7 @@ void sched_yield(struct scheduler* sched) { void sched_block(struct scheduler* sched) { irq_lock(); - if (atomic_exchange_explicit(&sched->current->wake_pending, false, memory_order_acq_rel)) { + if (atomic_exchange(&sched->current->wake_pending, false, ATOMIC_ACQ_REL)) { irq_unlock(); return; } @@ -126,7 +126,7 @@ void sched_wake(struct task* task) { if (task->state == TASK_STATE_BLOCKED) sched_add_task(&percpu_get()->sched, task); else - atomic_store_explicit(&task->wake_pending, true, memory_order_release); + atomic_store(&task->wake_pending, true, ATOMIC_RELEASE); irq_unlock(); } @@ -139,7 +139,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt return ENOMEM; static atomic_size_t last_tid = 0; - new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire); + new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE); new_task->space = space; strncpy(new_task->name, name, sizeof(new_task->name)); diff --git a/core/spin.c b/core/spin.c index 6b18418..5e4ec63 100644 --- a/core/spin.c +++ b/core/spin.c @@ -1,6 +1,6 @@ +#include #include #include -#include void spin_init(struct spinlock* spin) { spin->next = 0; @@ -9,15 +9,15 @@ void spin_init(struct spinlock* spin) { void spin_lock(struct spinlock* spin) { irq_lock(); - uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed); + uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED); - while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) { + while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) { asm volatile("pause"); } } void spin_unlock(struct spinlock* spin) { - uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed); - atomic_store_explicit(&spin->owner, val + 1, memory_order_release); + uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED); + atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE); irq_unlock(); } diff --git a/include/stdatomic.h b/include/stdatomic.h deleted file mode 100644 index 29c2046..0000000 --- a/include/stdatomic.h +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-License-Identifier: ISC -#pragma once - -#include -#include - -typedef enum memory_order { - memory_order_relaxed = __ATOMIC_RELAXED, - memory_order_consume = __ATOMIC_CONSUME, - memory_order_acquire = __ATOMIC_ACQUIRE, - memory_order_release = __ATOMIC_RELEASE, - memory_order_acq_rel = __ATOMIC_ACQ_REL, - memory_order_seq_cst = __ATOMIC_SEQ_CST, -} memory_order; - -#define atomic_thread_fence(order) __atomic_thread_fence(order) -#define atomic_signal_fence(order) __atomic_signal_fence(order) - -#ifdef __clang__ - -#define atomic_init(obj, value) __c11_atomic_init(obj, value) - -#define atomic_store_explicit(obj, val, order) __c11_atomic_store(obj, val, order) -#define atomic_load_explicit(obj, order) __c11_atomic_load(obj, order) - -#define atomic_exchange_explicit(obj, val, order) __c11_atomic_exchange(obj, val, order) - -#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ - __c11_atomic_compare_exchange_strong(obj, expected, desired, succ, fail) -#define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \ - __c11_atomic_compare_exchange_weak(obj, expected, desired, succ, fail) - -#define atomic_fetch_add_explicit(obj, arg, order) __c11_atomic_fetch_add(obj, arg, order) -#define atomic_fetch_sub_explicit(obj, arg, order) __c11_atomic_fetch_sub(obj, arg, order) -#define atomic_fetch_or_explicit(obj, arg, order) __c11_atomic_fetch_or(obj, arg, order) -#define atomic_fetch_xor_explicit(obj, arg, order) __c11_atomic_fetch_xor(obj, arg, order) -#define atomic_fetch_and_explicit(obj, arg, order) __c11_atomic_fetch_and(obj, arg, order) - -#define atomic_is_lock_free(obj) __c11_atomic_is_lock_free(sizeof(*(obj))) - -#else /* GCC */ - -#define atomic_init(obj, value) \ - do { \ - *(obj) = (value); \ - } while (0) - -#define atomic_store_explicit(obj, val, order) \ - ({ \ - auto __ao = (obj); \ - typeof(*__ao) __av = (val); \ - __atomic_store(__ao, &__av, order); \ - }) - -#define atomic_load_explicit(obj, order) \ - ({ \ - auto __ao = (obj); \ - typeof(*__ao) __ar; \ - __atomic_load(__ao, &__ar, order); \ - __ar; \ - }) - -#define atomic_exchange_explicit(obj, val, order) \ - ({ \ - auto __ao = (obj); \ - typeof(*__ao) __av = (val); \ - typeof(*__ao) __ar; \ - __atomic_exchange(__ao, &__av, &__ar, order); \ - __ar; \ - }) - -#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ - ({ \ - auto __ao = (obj); \ - auto __ae = (expected); \ - typeof(*__ao) __ad = (desired); \ - __atomic_compare_exchange(__ao, __ae, &__ad, 0, succ, fail); \ - }) - -#define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \ - ({ \ - auto __ao = (obj); \ - auto __ae = (expected); \ - typeof(*__ao) __ad = (desired); \ - __atomic_compare_exchange(__ao, __ae, &__ad, 1, succ, fail); \ - }) - -#define atomic_fetch_add_explicit(obj, arg, order) __atomic_fetch_add(obj, arg, order) -#define atomic_fetch_sub_explicit(obj, arg, order) __atomic_fetch_sub(obj, arg, order) -#define atomic_fetch_or_explicit(obj, arg, order) __atomic_fetch_or(obj, arg, order) -#define atomic_fetch_xor_explicit(obj, arg, order) __atomic_fetch_xor(obj, arg, order) -#define atomic_fetch_and_explicit(obj, arg, order) __atomic_fetch_and(obj, arg, order) - -#define atomic_is_lock_free(obj) __atomic_is_lock_free(sizeof(*(obj)), (obj)) - -#endif - -#define atomic_store(obj, val) atomic_store_explicit(obj, val, memory_order_seq_cst) -#define atomic_load(obj) atomic_load_explicit(obj, memory_order_seq_cst) - -#define atomic_exchange(obj, val) atomic_exchange_explicit(obj, val, memory_order_seq_cst) - -#define atomic_compare_exchange_strong(obj, expected, desired) \ - atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst) -#define atomic_compare_exchange_weak(obj, expected, desired) \ - atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst) - -#define atomic_fetch_add(obj, arg) atomic_fetch_add_explicit(obj, arg, memory_order_seq_cst) -#define atomic_fetch_sub(obj, arg) atomic_fetch_sub_explicit(obj, arg, memory_order_seq_cst) -#define atomic_fetch_or(obj, arg) atomic_fetch_or_explicit(obj, arg, memory_order_seq_cst) -#define atomic_fetch_xor(obj, arg) atomic_fetch_xor_explicit(obj, arg, memory_order_seq_cst) -#define atomic_fetch_and(obj, arg) atomic_fetch_and_explicit(obj, arg, memory_order_seq_cst) - -// atomic_flag -typedef struct atomic_flag { - _Atomic(bool) __val; -} atomic_flag; - -#define ATOMIC_FLAG_INIT {0} - -#define atomic_flag_test_and_set_explicit(obj, order) atomic_exchange_explicit(&(obj)->__val, 1, order) -#define atomic_flag_test_and_set(obj) atomic_flag_test_and_set_explicit(obj, memory_order_seq_cst) -#define atomic_flag_clear_explicit(obj, order) atomic_store_explicit(&(obj)->__val, 0, order) -#define atomic_flag_clear(obj) atomic_flag_clear_explicit(obj, memory_order_seq_cst) - -typedef _Atomic(bool) atomic_bool; -typedef _Atomic(char) atomic_char; -typedef _Atomic(signed char) atomic_schar; -typedef _Atomic(unsigned char) atomic_uchar; -typedef _Atomic(short) atomic_short; -typedef _Atomic(unsigned short) atomic_ushort; -typedef _Atomic(int) atomic_int; -typedef _Atomic(unsigned int) atomic_uint; -typedef _Atomic(long) atomic_long; -typedef _Atomic(unsigned long) atomic_ulong; -typedef _Atomic(long long) atomic_llong; -typedef _Atomic(unsigned long long) atomic_ullong; -typedef _Atomic(__CHAR16_TYPE__) atomic_char16_t; -typedef _Atomic(__CHAR32_TYPE__) atomic_char32_t; -typedef _Atomic(__WCHAR_TYPE__) atomic_wchar_t; - -typedef _Atomic(int_least8_t) atomic_int_least8_t; -typedef _Atomic(uint_least8_t) atomic_uint_least8_t; -typedef _Atomic(int_least16_t) atomic_int_least16_t; -typedef _Atomic(uint_least16_t) atomic_uint_least16_t; -typedef _Atomic(int_least32_t) atomic_int_least32_t; -typedef _Atomic(uint_least32_t) atomic_uint_least32_t; -typedef _Atomic(int_least64_t) atomic_int_least64_t; -typedef _Atomic(uint_least64_t) atomic_uint_least64_t; -typedef _Atomic(int_fast8_t) atomic_int_fast8_t; -typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t; -typedef _Atomic(int_fast16_t) atomic_int_fast16_t; -typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t; -typedef _Atomic(int_fast32_t) atomic_int_fast32_t; -typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t; -typedef _Atomic(int_fast64_t) atomic_int_fast64_t; -typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t; -typedef _Atomic(intptr_t) atomic_intptr_t; -typedef _Atomic(uintptr_t) atomic_uintptr_t; -typedef _Atomic(size_t) atomic_size_t; -typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t; -typedef _Atomic(intmax_t) atomic_intmax_t; -typedef _Atomic(uintmax_t) atomic_uintmax_t; diff --git a/include/zinnia/atomic.h b/include/zinnia/atomic.h new file mode 100644 index 0000000..87aef4c --- /dev/null +++ b/include/zinnia/atomic.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include + +enum atomic_order { + ATOMIC_RELAXED = __ATOMIC_RELAXED, + ATOMIC_ACQUIRE = __ATOMIC_ACQUIRE, + ATOMIC_RELEASE = __ATOMIC_RELEASE, + ATOMIC_ACQ_REL = __ATOMIC_ACQ_REL, + ATOMIC_SEQ_CST = __ATOMIC_SEQ_CST, +}; + +typedef _Atomic(bool) atomic_bool; +typedef _Atomic(int) atomic_int; +typedef _Atomic(unsigned int) atomic_uint; +typedef _Atomic(long) atomic_long; +typedef _Atomic(unsigned long) atomic_ulong; +typedef _Atomic(intptr_t) atomic_intptr_t; +typedef _Atomic(uintptr_t) atomic_uintptr_t; +typedef _Atomic(size_t) atomic_size_t; + +#ifdef __clang__ + +#define atomic_init(obj, value) __c11_atomic_init(obj, value) + +#define atomic_load(obj, order) __c11_atomic_load(obj, order) +#define atomic_store(obj, val, order) __c11_atomic_store(obj, val, order) +#define atomic_exchange(obj, val, order) __c11_atomic_exchange(obj, val, order) + +#define atomic_compare_exchange(obj, expected, desired, success, failure) \ + __c11_atomic_compare_exchange_strong(obj, expected, desired, success, failure) +#define atomic_compare_exchange_weak(obj, expected, desired, success, failure) \ + __c11_atomic_compare_exchange_weak(obj, expected, desired, success, failure) + +#define atomic_fetch_add(obj, arg, order) __c11_atomic_fetch_add(obj, arg, order) +#define atomic_fetch_sub(obj, arg, order) __c11_atomic_fetch_sub(obj, arg, order) +#define atomic_fetch_and(obj, arg, order) __c11_atomic_fetch_and(obj, arg, order) +#define atomic_fetch_or(obj, arg, order) __c11_atomic_fetch_or(obj, arg, order) +#define atomic_fetch_xor(obj, arg, order) __c11_atomic_fetch_xor(obj, arg, order) + +#else + +#define atomic_init(obj, value) \ + do { \ + *(obj) = (value); \ + } while (0) + +#define atomic_load(obj, order) __atomic_load_n(obj, order) +#define atomic_store(obj, val, order) __atomic_store_n(obj, val, order) +#define atomic_exchange(obj, val, order) __atomic_exchange_n(obj, val, order) + +#define atomic_compare_exchange(obj, expected, desired, success, failure) \ + __atomic_compare_exchange_n(obj, expected, desired, false, success, failure) +#define atomic_compare_exchange_weak(obj, expected, desired, success, failure) \ + __atomic_compare_exchange_n(obj, expected, desired, true, success, failure) + +#define atomic_fetch_add(obj, arg, order) __atomic_fetch_add(obj, arg, order) +#define atomic_fetch_sub(obj, arg, order) __atomic_fetch_sub(obj, arg, order) +#define atomic_fetch_and(obj, arg, order) __atomic_fetch_and(obj, arg, order) +#define atomic_fetch_or(obj, arg, order) __atomic_fetch_or(obj, arg, order) +#define atomic_fetch_xor(obj, arg, order) __atomic_fetch_xor(obj, arg, order) + +#endif + +#define atomic_fence(order) __atomic_thread_fence(order) +#define atomic_compiler_fence(order) __atomic_signal_fence(order) diff --git a/include/zinnia/dpc.h b/include/zinnia/dpc.h index c9e95cd..627d1a2 100644 --- a/include/zinnia/dpc.h +++ b/include/zinnia/dpc.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include struct dpc { void (*callback)(struct dpc* dpc, void* data); diff --git a/include/zinnia/refcount.h b/include/zinnia/refcount.h index 8bd6d23..d1ede0f 100644 --- a/include/zinnia/refcount.h +++ b/include/zinnia/refcount.h @@ -1,7 +1,7 @@ #pragma once +#include #include -#include #include struct refcount { @@ -9,16 +9,16 @@ struct refcount { }; static inline void refcount_init(struct refcount* rc) { - atomic_store_explicit(&rc->count, 1, memory_order_relaxed); + atomic_store(&rc->count, 1, ATOMIC_RELAXED); } static inline void refcount_inc(struct refcount* rc) { - atomic_fetch_add_explicit(&rc->count, 1, memory_order_relaxed); + atomic_fetch_add(&rc->count, 1, ATOMIC_RELAXED); } static inline bool refcount_dec(struct refcount* rc) { - if (atomic_fetch_sub_explicit(&rc->count, 1, memory_order_release) != 1) + if (atomic_fetch_sub(&rc->count, 1, ATOMIC_RELEASE) != 1) return false; - atomic_thread_fence(memory_order_acquire); + atomic_fence(ATOMIC_ACQUIRE); return true; } diff --git a/include/zinnia/sched.h b/include/zinnia/sched.h index fb31b23..e576120 100644 --- a/include/zinnia/sched.h +++ b/include/zinnia/sched.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -7,7 +8,6 @@ #include #include #include -#include #include #include diff --git a/include/zinnia/waitq.h b/include/zinnia/waitq.h index dc7c901..77e7f88 100644 --- a/include/zinnia/waitq.h +++ b/include/zinnia/waitq.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include #include -#include // A queue of tasks waiting for an event. struct waitq {