diff --git a/fs/refs.c b/fs/refs.c index 94f4ace..4b45fd9 100644 --- a/fs/refs.c +++ b/fs/refs.c @@ -224,6 +224,10 @@ void vfs_file_ref(struct vfs_file* file) { refcount_inc(&file->rc); } +static void file_free(struct epoch_head* head) { + mem_free(containerof(head, struct vfs_file, epoch)); +} + void vfs_file_unref(struct vfs_file* file) { if (!refcount_dec(&file->rc)) return; @@ -234,7 +238,7 @@ void vfs_file_unref(struct vfs_file* file) { vfs_path_drop(file->path); vfs_node_unref(file->vnode); mem_free(file->absolute_path); - mem_free(file); + epoch_call(&file->epoch, file_free); } struct vfs_path vfs_path_clone(struct vfs_path path) { diff --git a/include/zinnia/eplist.h b/include/zinnia/eplist.h new file mode 100644 index 0000000..1b5afbb --- /dev/null +++ b/include/zinnia/eplist.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include + +#define eplist_head(type) \ + struct { \ + _Atomic(type*) eph_first; \ + } + +#define eplist_link(type) \ + struct { \ + _Atomic(type*) epl_next; \ + _Atomic(type*)* epl_prev; \ + } + +#define eplist_init(head) atomic_init(&(head)->eph_first, nullptr) + +#define eplist_first(head) epoch_deref(&(head)->eph_first) + +#define eplist_next(elm, field) epoch_deref(&(elm)->field.epl_next) + +#define eplist_empty(head) (atomic_load(&(head)->eph_first, ATOMIC_RELAXED) == nullptr) + +#define eplist_foreach(var, head, field) for ((var) = eplist_first(head); (var); (var) = eplist_next(var, field)) + +#define eplist_insert_head(head, elm, field) \ + do { \ + typeof(elm) _epl_first = atomic_load(&(head)->eph_first, ATOMIC_RELAXED); \ + atomic_store(&(elm)->field.epl_next, _epl_first, ATOMIC_RELAXED); \ + (elm)->field.epl_prev = &(head)->eph_first; \ + if (_epl_first) \ + _epl_first->field.epl_prev = &(elm)->field.epl_next; \ + epoch_publish(&(head)->eph_first, (elm)); \ + } while (0) + +#define eplist_insert_after(listelm, elm, field) \ + do { \ + typeof(elm) _epl_next = atomic_load(&(listelm)->field.epl_next, ATOMIC_RELAXED); \ + atomic_store(&(elm)->field.epl_next, _epl_next, ATOMIC_RELAXED); \ + (elm)->field.epl_prev = &(listelm)->field.epl_next; \ + if (_epl_next) \ + _epl_next->field.epl_prev = &(elm)->field.epl_next; \ + epoch_publish(&(listelm)->field.epl_next, (elm)); \ + } while (0) + +#define eplist_insert_before(listelm, elm, field) \ + do { \ + atomic_store(&(elm)->field.epl_next, (listelm), ATOMIC_RELAXED); \ + (elm)->field.epl_prev = (listelm)->field.epl_prev; \ + epoch_publish((listelm)->field.epl_prev, (elm)); \ + (listelm)->field.epl_prev = &(elm)->field.epl_next; \ + } while (0) + +#define eplist_remove(elm, field) \ + do { \ + typeof(elm) _epl_next = atomic_load(&(elm)->field.epl_next, ATOMIC_RELAXED); \ + if (_epl_next) \ + _epl_next->field.epl_prev = (elm)->field.epl_prev; \ + epoch_publish((elm)->field.epl_prev, _epl_next); \ + } while (0) diff --git a/include/zinnia/epoch.h b/include/zinnia/epoch.h new file mode 100644 index 0000000..3f1e011 --- /dev/null +++ b/include/zinnia/epoch.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include + +struct epoch_head { + struct epoch_head* next; + void (*func)(struct epoch_head* head); +}; + +// Enters a read-side critical section. +// Disables preemption, so the caller must not sleep. +// A reader may call this function even while already in an epoch section. +void epoch_enter(); + +// Leaves a read-side critical section. +void epoch_exit(); + +// Returns true if the current CPU is inside a critical section. +bool epoch_active(); + +// Runs `func` once all readers that could observe `head` have left. +void epoch_call(struct epoch_head* head, void (*func)(struct epoch_head* head)); + +// Advances the global epoch if all active readers have observed it. +bool epoch_try_advance(); + +// Returns the current global epoch. +uint64_t epoch_current(); + +// Waits until all readers active at the time of the call have left. +void epoch_synchronize(); + +// Waits until all previously queued callbacks have run. +void epoch_barrier(); + +// Loads a pointer protected by an epoch critical section. +#define epoch_deref(ptr) atomic_load(ptr, ATOMIC_ACQUIRE) + +// Publishes a pointer to epoch readers. +#define epoch_publish(ptr, val) atomic_store(ptr, val, ATOMIC_RELEASE) + +INIT_EXTERN(epoch_stage); diff --git a/include/zinnia/lfhash.h b/include/zinnia/lfhash.h new file mode 100644 index 0000000..c9160c1 --- /dev/null +++ b/include/zinnia/lfhash.h @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include +#include + +constexpr size_t LFHASH_SEGMENT_BUCKETS = 512; +constexpr size_t LFHASH_SEGMENTS = 1024; +constexpr size_t LFHASH_MIN_BUCKETS = 16; +constexpr size_t LFHASH_LOAD_FACTOR = 2; + +struct lfhash_node { + _Atomic(uintptr_t) next; + uint64_t key; +}; + +struct lfhash_bucket; + +struct lfhash { + _Atomic(struct lfhash_bucket*)* segments; + atomic_size_t size; + atomic_size_t count; +}; + +typedef bool (*lfhash_eq_fn)(const struct lfhash_node* node, const void* key); + +errno_t lfhash_init(struct lfhash* table); +void lfhash_destroy(struct lfhash* table); +size_t lfhash_count(struct lfhash* table); + +errno_t lfhash_insert(struct lfhash* table, uint64_t hash, struct lfhash_node* node, lfhash_eq_fn eq, const void* key); +struct lfhash_node* lfhash_lookup(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key); +struct lfhash_node* lfhash_remove(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key); + +struct lfhash_node* lfhash_first(struct lfhash* table); +struct lfhash_node* lfhash_next(struct lfhash_node* node); + +#define lfhash_foreach(table, var) for ((var) = lfhash_first(table); (var); (var) = lfhash_next(var)) diff --git a/include/zinnia/lfqueue.h b/include/zinnia/lfqueue.h new file mode 100644 index 0000000..3408c78 --- /dev/null +++ b/include/zinnia/lfqueue.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include + +struct lfqueue_node; + +struct lfqueue { + _Atomic(struct lfqueue_node*) head; + _Atomic(struct lfqueue_node*) tail; +}; + +errno_t lfqueue_init(struct lfqueue* queue); +void lfqueue_destroy(struct lfqueue* queue); +bool lfqueue_empty(struct lfqueue* queue); +errno_t lfqueue_push(struct lfqueue* queue, void* value); +bool lfqueue_pop(struct lfqueue* queue, void** out); diff --git a/include/zinnia/lfstack.h b/include/zinnia/lfstack.h new file mode 100644 index 0000000..2bad9d7 --- /dev/null +++ b/include/zinnia/lfstack.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include + +struct lfstack_node { + _Atomic(struct lfstack_node*) next; +}; + +struct lfstack { + _Atomic(struct lfstack_node*) top; +}; + +void lfstack_init(struct lfstack* stack); +bool lfstack_empty(struct lfstack* stack); +void lfstack_push(struct lfstack* stack, struct lfstack_node* node); +struct lfstack_node* lfstack_pop(struct lfstack* stack); +struct lfstack_node* lfstack_pop_all(struct lfstack* stack); diff --git a/include/zinnia/process.h b/include/zinnia/process.h index 68b7a81..a4b2634 100644 --- a/include/zinnia/process.h +++ b/include/zinnia/process.h @@ -25,8 +25,10 @@ struct process_rlimit { rlim_t max; }; +struct fd_files; + struct fd_table { - struct vfs_file** files; + _Atomic(struct fd_files*) files; uint64_t* open; uint64_t* cloexec; size_t capacity; diff --git a/include/zinnia/radix.h b/include/zinnia/radix.h index aa12044..eaee86b 100644 --- a/include/zinnia/radix.h +++ b/include/zinnia/radix.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include #include @@ -20,12 +21,13 @@ struct radix_node { uint8_t count; uint8_t offset; struct radix_node* parent; - uint64_t tags[RADIX_NUM_TAGS]; - void* slots[RADIX_SLOTS]; + _Atomic(uint64_t) tags[RADIX_NUM_TAGS]; + _Atomic(void*) slots[RADIX_SLOTS]; + struct epoch_head epoch; }; struct radix_tree { - struct radix_node* root; + _Atomic(struct radix_node*) root; }; void radix_init(struct radix_tree* tree); @@ -35,7 +37,7 @@ bool radix_empty(struct radix_tree* tree); void* radix_lookup(struct radix_tree* tree, uint64_t index); errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value); void* radix_remove(struct radix_tree* tree, uint64_t index); -void** radix_slot(struct radix_tree* tree, uint64_t index); +void* radix_replace(struct radix_tree* tree, uint64_t index, void* value); void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag); void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag); diff --git a/include/zinnia/rcu.h b/include/zinnia/rcu.h deleted file mode 100644 index 8e0ab78..0000000 --- a/include/zinnia/rcu.h +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: ISC - -#pragma once - -void rcu_read_lock(); -void rcu_read_unlock(); diff --git a/include/zinnia/sched.h b/include/zinnia/sched.h index ac8c10a..7933451 100644 --- a/include/zinnia/sched.h +++ b/include/zinnia/sched.h @@ -135,6 +135,9 @@ void sched_notify(struct task* task); void sched_sleep_until(uint64_t deadline_ns); +void sched_preempt_disable(); +void sched_preempt_enable(); + [[noreturn]] void sched_exit(); diff --git a/include/zinnia/vfs.h b/include/zinnia/vfs.h index e0f1132..3048a5b 100644 --- a/include/zinnia/vfs.h +++ b/include/zinnia/vfs.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include #include @@ -356,6 +357,8 @@ struct vfs_file { // Driver data. void* priv; + + struct epoch_head epoch; }; // A file system driver. diff --git a/include/zinnia/virt.h b/include/zinnia/virt.h index c0fa552..a00c2b2 100644 --- a/include/zinnia/virt.h +++ b/include/zinnia/virt.h @@ -47,6 +47,7 @@ struct page* vm_page_from_addr(phys_t addr); errno_t page_alloc(enum alloc_flags flags, struct page** out); void page_hold(struct page* page); +bool page_try_hold(struct page* page); void page_unhold(struct page* page); bool page_try_busy(struct page* page); void page_unbusy(struct page* page); diff --git a/lib/epoch.c b/lib/epoch.c new file mode 100644 index 0000000..ece9671 --- /dev/null +++ b/lib/epoch.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr uint64_t EPOCH_NEST_BITS = 16; +constexpr uint64_t EPOCH_NEST_MASK = ((uint64_t)1 << EPOCH_NEST_BITS) - 1; +constexpr size_t EPOCH_BUCKETS = 3; +constexpr size_t EPOCH_BATCH = 64; +constexpr size_t EPOCH_SYNC_SPINS = 64; +constexpr uint64_t EPOCH_POLL_NS = 10'000'000; +constexpr uint64_t EPOCH_SYNC_POLL_NS = 1'000'000; + +struct epoch_list { + struct epoch_head* head; + struct epoch_head* tail; +}; + +struct epoch_bucket { + uint64_t epoch; + struct epoch_list list; +}; + +struct epoch_cpu { + _Atomic(uint64_t) local; + struct spinlock lock; + struct epoch_bucket buckets[EPOCH_BUCKETS]; + struct epoch_list ready; +}; + +[[__percpu]] +static struct epoch_cpu epoch_data; + +static _Atomic(uint64_t) global_epoch = 0; +static atomic_size_t pending = 0; +static mutex_t reclaim_lock; +static struct timer poll_timer; +static _Atomic(struct task*) reclaimer = nullptr; +static atomic_bool reclaimer_idle = false; + +void epoch_enter() { + sched_preempt_disable(); + _Atomic(uint64_t)* local = &percpu_ref(epoch_data)->local; + const uint64_t value = atomic_load(local, ATOMIC_RELAXED); + + // If we already entered the critical section before, just increment the nest counter. + if (value & EPOCH_NEST_MASK) { + assert((value & EPOCH_NEST_MASK) != EPOCH_NEST_MASK, "Epoch nesting overflow"); + atomic_store(local, value + 1, ATOMIC_RELAXED); + return; + } + + // Store the current epoch in the upper bits, and mark this CPU as active. + atomic_store(local, (atomic_load(&global_epoch, ATOMIC_RELAXED) << EPOCH_NEST_BITS) | 1, ATOMIC_RELAXED); + atomic_fence(ATOMIC_SEQ_CST); +} + +void epoch_exit() { + _Atomic(uint64_t)* local = &percpu_ref(epoch_data)->local; + const uint64_t value = atomic_load(local, ATOMIC_RELAXED); + assert(value & EPOCH_NEST_MASK, "Tried to exit an epoch section without entering one"); + + // If we didn't nest, we can clear the local epoch. + // Otherwise, decrease the nest counter. + if ((value & EPOCH_NEST_MASK) == 1) + atomic_store(local, 0, ATOMIC_RELEASE); + else + atomic_store(local, value - 1, ATOMIC_RELAXED); + + sched_preempt_enable(); +} + +bool epoch_active() { + sched_preempt_disable(); + const bool active = atomic_load(&percpu_ref(epoch_data)->local, ATOMIC_RELAXED) & EPOCH_NEST_MASK; + sched_preempt_enable(); + return active; +} + +static void list_push(struct epoch_list* list, struct epoch_head* head) { + head->next = list->head; + if (!list->head) + list->tail = head; + list->head = head; +} + +static void list_splice(struct epoch_list* dst, struct epoch_list* src) { + if (!src->head) + return; + src->tail->next = dst->head; + if (!dst->head) + dst->tail = src->tail; + dst->head = src->head; + *src = (struct epoch_list){}; +} + +static void wake_reclaimer() { + struct task* task = atomic_load(&reclaimer, ATOMIC_ACQUIRE); + if (task && atomic_exchange(&reclaimer_idle, false, ATOMIC_SEQ_CST)) + sched_wake(task); +} + +void epoch_call(struct epoch_head* head, void (*func)(struct epoch_head* head)) { + head->func = func; + const size_t old = atomic_fetch_add(&pending, 1, ATOMIC_SEQ_CST); + atomic_fence(ATOMIC_SEQ_CST); + + irq_lock(); + struct epoch_cpu* cpu = percpu_ref(epoch_data); + spin_lock(&cpu->lock); + + // Store the object head in the bucket of this epoch. + // We only need to store 3 buckets, since only these epochs are relevant: + // - Current epoch, may have readers + // - Previous epoch, may still have readers + // - Safe epoch, has no readers remaining + const uint64_t now = atomic_load(&global_epoch, ATOMIC_RELAXED); + struct epoch_bucket* bucket = &cpu->buckets[now % EPOCH_BUCKETS]; + if (bucket->epoch != now) { + // Combine both epoch lists. + list_splice(&cpu->ready, &bucket->list); + bucket->epoch = now; + } + list_push(&bucket->list, head); + + spin_unlock(&cpu->lock); + irq_unlock(); + + if (old == 0 || old % EPOCH_BATCH == 0) + wake_reclaimer(); +} + +uint64_t epoch_current() { + return atomic_load(&global_epoch, ATOMIC_RELAXED); +} + +bool epoch_try_advance() { + uint64_t now = atomic_load(&global_epoch, ATOMIC_RELAXED); + atomic_fence(ATOMIC_SEQ_CST); + const size_t count = percpu_count(); + + // Go through all CPUs. + for (size_t i = 0; i < count; i++) { + const uint64_t local = atomic_load(&percpu_ref_on(percpu_cpu(i), epoch_data)->local, ATOMIC_ACQUIRE); + + // If any of the CPUs are in a critical section, and their local epoch + // is not the same as the current global epoch, we can't advance. + if ((local & EPOCH_NEST_MASK) && (local >> EPOCH_NEST_BITS) != now) + return false; + } + + // At this point all readers agree on the epoch, it's safe to increment now. + atomic_compare_exchange(&global_epoch, &now, now + 1, ATOMIC_SEQ_CST, ATOMIC_RELAXED); + return true; +} + +static struct epoch_list collect(struct epoch_cpu* cpu, uint64_t now) { + spin_lock(&cpu->lock); + for (size_t i = 0; i < EPOCH_BUCKETS; i++) { + struct epoch_bucket* bucket = &cpu->buckets[i]; + if (bucket->epoch + 2 <= now) + list_splice(&cpu->ready, &bucket->list); + } + struct epoch_list out = cpu->ready; + cpu->ready = (struct epoch_list){}; + spin_unlock(&cpu->lock); + return out; +} + +static void run(struct epoch_head* head) { + size_t count = 0; + while (head) { + struct epoch_head* next = head->next; + head->func(head); + head = next; + count++; + } + if (count) + atomic_fetch_sub(&pending, count, ATOMIC_RELAXED); +} + +static void reclaim() { + mutex_lock(&reclaim_lock); + const uint64_t now = atomic_load(&global_epoch, ATOMIC_ACQUIRE); + const size_t count = percpu_count(); + for (size_t i = 0; i < count; i++) + run(collect(percpu_ref_on(percpu_cpu(i), epoch_data), now).head); + mutex_unlock(&reclaim_lock); +} + +void epoch_synchronize() { + // We can't synchronize a write while being a reader ourselves. + assert(!epoch_active(), "Tried to synchronize inside an epoch section"); + + atomic_fence(ATOMIC_SEQ_CST); + const uint64_t target = atomic_load(&global_epoch, ATOMIC_RELAXED) + 2; + for (size_t spins = 0; atomic_load(&global_epoch, ATOMIC_ACQUIRE) < target; spins++) { + if (epoch_try_advance()) + continue; + if (spins < EPOCH_SYNC_SPINS) + arch_cpu_pause(); + else + sched_sleep_until(clock_get_elapsed_ns() + EPOCH_SYNC_POLL_NS); + } +} + +void epoch_barrier() { + epoch_synchronize(); + reclaim(); +} + +static void poll_tick(struct timer*) { + wake_reclaimer(); +} + +static void reclaimer_fn(uintptr_t) { + while (true) { + if (atomic_load(&pending, ATOMIC_RELAXED)) { + epoch_try_advance(); + reclaim(); + } + + atomic_store(&reclaimer_idle, true, ATOMIC_SEQ_CST); + if (atomic_load(&pending, ATOMIC_SEQ_CST)) + timer_arm(&poll_timer, clock_get_elapsed_ns() + EPOCH_POLL_NS); + sched_block(&percpu_get()->sched); + } +} + +static void epoch_stage_init(struct boot_info*) { + mutex_init(&reclaim_lock); + timer_init(&poll_timer, poll_tick); + + struct task* task; + errno_t status = task_create("epoch", process_kernel(), reclaimer_fn, 0, &task); + assert(status == 0, "Unable to create the epoch reclaimer"); + sched_set_priority(task, SCHED_PRIO_WORK_HIGH); + atomic_store(&reclaimer, task, ATOMIC_RELEASE); + sched_add_task(&percpu_get()->sched, task); +} +INIT_TASK(epoch_stage, "core.epoch", epoch_stage_init); +INIT_DEPENDS(epoch_stage, sched_stage); diff --git a/lib/lfhash.c b/lib/lfhash.c new file mode 100644 index 0000000..0df052d --- /dev/null +++ b/lib/lfhash.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include + +constexpr uintptr_t LFHASH_MARK = 1; +constexpr uint64_t LFHASH_MSB = (uint64_t)1 << 63; + +enum bucket_state : uint8_t { + BUCKET_UNLINKED, + BUCKET_LINKING, + BUCKET_LINKED, +}; + +struct lfhash_bucket { + struct lfhash_node node; + _Atomic(enum bucket_state) state; +}; + +struct find { + _Atomic(uintptr_t)* prev; + struct lfhash_node* cur; +}; + +static uint64_t bitrev(uint64_t x) { + x = ((x >> 1) & 0x5555555555555555) | ((x & 0x5555555555555555) << 1); + x = ((x >> 2) & 0x3333333333333333) | ((x & 0x3333333333333333) << 2); + x = ((x >> 4) & 0x0F0F0F0F0F0F0F0F) | ((x & 0x0F0F0F0F0F0F0F0F) << 4); + return __builtin_bswap64(x); +} + +static uint64_t regular_key(uint64_t hash) { + return bitrev(hash | LFHASH_MSB); +} + +static uint64_t sentinel_key(size_t bucket) { + return bitrev(bucket); +} + +static size_t parent_of(size_t bucket) { + return bucket & ~((size_t)1 << (63 - __builtin_clzll(bucket))); +} + +static struct lfhash_node* node_of(uintptr_t link) { + return (struct lfhash_node*)(link & ~LFHASH_MARK); +} + +static struct lfhash_bucket* bucket_get(struct lfhash* table, size_t bucket) { + struct lfhash_bucket* segment = atomic_load(&table->segments[bucket / LFHASH_SEGMENT_BUCKETS], ATOMIC_ACQUIRE); + return segment ? &segment[bucket % LFHASH_SEGMENT_BUCKETS] : nullptr; +} + +static struct lfhash_bucket* segment_ensure(struct lfhash* table, size_t bucket) { + _Atomic(struct lfhash_bucket*)* slot = &table->segments[bucket / LFHASH_SEGMENT_BUCKETS]; + struct lfhash_bucket* segment = atomic_load(slot, ATOMIC_ACQUIRE); + if (!segment) { + struct lfhash_bucket* fresh = mem_alloc(sizeof(struct lfhash_bucket) * LFHASH_SEGMENT_BUCKETS, 0); + if (!fresh) + return nullptr; + if (atomic_compare_exchange(slot, &segment, fresh, ATOMIC_ACQ_REL, ATOMIC_ACQUIRE)) + segment = fresh; + else + mem_free(fresh); + } + return &segment[bucket % LFHASH_SEGMENT_BUCKETS]; +} + +static bool find(_Atomic(uintptr_t)* head, uint64_t key, lfhash_eq_fn eq, const void* eq_key, struct find* out) { +again: + _Atomic(uintptr_t)* prev = head; + struct lfhash_node* cur = node_of(atomic_load(prev, ATOMIC_ACQUIRE)); + while (cur) { + const uintptr_t next = atomic_load(&cur->next, ATOMIC_ACQUIRE); + if (next & LFHASH_MARK) { + uintptr_t expected = (uintptr_t)cur; + if (!atomic_compare_exchange(prev, &expected, next & ~LFHASH_MARK, ATOMIC_RELEASE, ATOMIC_RELAXED)) + goto again; + cur = node_of(next); + continue; + } + if (cur->key > key) + break; + if (cur->key == key && (!eq || eq(cur, eq_key))) { + *out = (struct find){.prev = prev, .cur = cur}; + return true; + } + prev = &cur->next; + cur = node_of(next); + } + *out = (struct find){.prev = prev, .cur = cur}; + return false; +} + +static void bucket_link(struct lfhash* table, size_t index, struct lfhash_bucket* bucket) { + enum bucket_state state = BUCKET_UNLINKED; + if (!atomic_compare_exchange(&bucket->state, &state, BUCKET_LINKING, ATOMIC_RELAXED, ATOMIC_RELAXED)) { + while (atomic_load(&bucket->state, ATOMIC_ACQUIRE) != BUCKET_LINKED) + arch_cpu_pause(); + return; + } + + const size_t parent_index = parent_of(index); + struct lfhash_bucket* parent = bucket_get(table, parent_index); + if (atomic_load(&parent->state, ATOMIC_ACQUIRE) != BUCKET_LINKED) + bucket_link(table, parent_index, parent); + + bucket->node.key = sentinel_key(index); + while (true) { + struct find f; + find(&parent->node.next, bucket->node.key, nullptr, nullptr, &f); + atomic_store(&bucket->node.next, (uintptr_t)f.cur, ATOMIC_RELAXED); + uintptr_t expected = (uintptr_t)f.cur; + if (atomic_compare_exchange(f.prev, &expected, (uintptr_t)&bucket->node, ATOMIC_RELEASE, ATOMIC_RELAXED)) + break; + } + atomic_store(&bucket->state, BUCKET_LINKED, ATOMIC_RELEASE); +} + +static bool ensure_segments(struct lfhash* table, size_t index) { + while (index) { + if (!segment_ensure(table, index)) + return false; + index = parent_of(index); + } + return true; +} + +static _Atomic(uintptr_t)* start_of(struct lfhash* table, size_t index) { + while (true) { + struct lfhash_bucket* bucket = bucket_get(table, index); + if (bucket && atomic_load(&bucket->state, ATOMIC_ACQUIRE) == BUCKET_LINKED) + return &bucket->node.next; + index = parent_of(index); + } +} + +static size_t index_of(struct lfhash* table, uint64_t hash) { + return hash & (atomic_load(&table->size, ATOMIC_RELAXED) - 1); +} + +errno_t lfhash_init(struct lfhash* table) { + table->segments = mem_alloc(sizeof(*table->segments) * LFHASH_SEGMENTS, 0); + if (!table->segments) + return ENOMEM; + struct lfhash_bucket* root = segment_ensure(table, 0); + if (!root) { + mem_free(table->segments); + return ENOMEM; + } + root->node.key = sentinel_key(0); + atomic_init(&root->node.next, 0); + atomic_init(&root->state, BUCKET_LINKED); + atomic_init(&table->size, LFHASH_MIN_BUCKETS); + atomic_init(&table->count, 0); + return 0; +} + +void lfhash_destroy(struct lfhash* table) { + for (size_t i = 0; i < LFHASH_SEGMENTS; i++) + mem_free(atomic_load(&table->segments[i], ATOMIC_RELAXED)); + mem_free(table->segments); + table->segments = nullptr; +} + +size_t lfhash_count(struct lfhash* table) { + return atomic_load(&table->count, ATOMIC_RELAXED); +} + +static void maybe_grow(struct lfhash* table, size_t count) { + size_t size = atomic_load(&table->size, ATOMIC_RELAXED); + if (count / size > LFHASH_LOAD_FACTOR && size * 2 <= LFHASH_SEGMENTS * LFHASH_SEGMENT_BUCKETS) + atomic_compare_exchange(&table->size, &size, size * 2, ATOMIC_RELAXED, ATOMIC_RELAXED); +} + +errno_t lfhash_insert(struct lfhash* table, uint64_t hash, struct lfhash_node* node, lfhash_eq_fn eq, const void* key) { + node->key = regular_key(hash); + const size_t index = index_of(table, hash); + const bool linkable = ensure_segments(table, index); + + epoch_enter(); + if (linkable) { + struct lfhash_bucket* bucket = bucket_get(table, index); + if (atomic_load(&bucket->state, ATOMIC_ACQUIRE) != BUCKET_LINKED) + bucket_link(table, index, bucket); + } + + _Atomic(uintptr_t)* start = start_of(table, index); + while (true) { + struct find f; + if (find(start, node->key, eq, key, &f)) { + epoch_exit(); + return EEXIST; + } + atomic_store(&node->next, (uintptr_t)f.cur, ATOMIC_RELAXED); + uintptr_t expected = (uintptr_t)f.cur; + if (atomic_compare_exchange(f.prev, &expected, (uintptr_t)node, ATOMIC_RELEASE, ATOMIC_RELAXED)) + break; + } + epoch_exit(); + + maybe_grow(table, atomic_fetch_add(&table->count, 1, ATOMIC_RELAXED) + 1); + return 0; +} + +struct lfhash_node* lfhash_lookup(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key) { + struct find f; + return find(start_of(table, index_of(table, hash)), regular_key(hash), eq, key, &f) ? f.cur : nullptr; +} + +struct lfhash_node* lfhash_remove(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key) { + const uint64_t so_key = regular_key(hash); + struct lfhash_node* victim = nullptr; + + epoch_enter(); + _Atomic(uintptr_t)* start = start_of(table, index_of(table, hash)); + while (true) { + struct find f; + if (!find(start, so_key, eq, key, &f)) + break; + uintptr_t next = atomic_load(&f.cur->next, ATOMIC_ACQUIRE); + if (next & LFHASH_MARK) + continue; + if (!atomic_compare_exchange(&f.cur->next, &next, next | LFHASH_MARK, ATOMIC_RELAXED, ATOMIC_RELAXED)) + continue; + victim = f.cur; + uintptr_t expected = (uintptr_t)victim; + if (!atomic_compare_exchange(f.prev, &expected, next, ATOMIC_RELEASE, ATOMIC_RELAXED)) + find(start, so_key, eq, key, &f); + break; + } + epoch_exit(); + + if (victim) + atomic_fetch_sub(&table->count, 1, ATOMIC_RELAXED); + return victim; +} + +static struct lfhash_node* skip(uintptr_t link) { + struct lfhash_node* node = node_of(link); + while (node) { + const uintptr_t next = atomic_load(&node->next, ATOMIC_ACQUIRE); + if (!(next & LFHASH_MARK) && (node->key & 1)) + return node; + node = node_of(next); + } + return nullptr; +} + +struct lfhash_node* lfhash_first(struct lfhash* table) { + return skip(atomic_load(&bucket_get(table, 0)->node.next, ATOMIC_ACQUIRE)); +} + +struct lfhash_node* lfhash_next(struct lfhash_node* node) { + return skip(atomic_load(&node->next, ATOMIC_ACQUIRE)); +} diff --git a/lib/lfqueue.c b/lib/lfqueue.c new file mode 100644 index 0000000..b341e76 --- /dev/null +++ b/lib/lfqueue.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include +#include +#include + +struct lfqueue_node { + _Atomic(struct lfqueue_node*) next; + void* value; +}; + +static struct slab_cache* node_cache; + +static void lfqueue_stage_init(struct boot_info*) { + node_cache = + slab_cache_new("lfqueue_node", sizeof(struct lfqueue_node), alignof(struct lfqueue_node), SLAB_EPOCH_SAFE); + assert(node_cache, "Unable to create the lfqueue node cache"); +} +INIT_TASK(lfqueue_stage, "core.lfqueue", lfqueue_stage_init); +INIT_DEPENDS(lfqueue_stage, mem_stage); + +static struct lfqueue_node* node_alloc(void* value) { + struct lfqueue_node* node = slab_cache_alloc(node_cache, 0); + if (node) + node->value = value; + return node; +} + +errno_t lfqueue_init(struct lfqueue* queue) { + struct lfqueue_node* dummy = node_alloc(nullptr); + if (!dummy) + return ENOMEM; + atomic_init(&queue->head, dummy); + atomic_init(&queue->tail, dummy); + return 0; +} + +void lfqueue_destroy(struct lfqueue* queue) { + struct lfqueue_node* node = atomic_load(&queue->head, ATOMIC_ACQUIRE); + while (node) { + struct lfqueue_node* next = atomic_load(&node->next, ATOMIC_RELAXED); + slab_cache_free(node_cache, node); + node = next; + } + atomic_store(&queue->head, nullptr, ATOMIC_RELAXED); + atomic_store(&queue->tail, nullptr, ATOMIC_RELAXED); +} + +bool lfqueue_empty(struct lfqueue* queue) { + epoch_enter(); + struct lfqueue_node* head = epoch_deref(&queue->head); + const bool empty = atomic_load(&head->next, ATOMIC_RELAXED) == nullptr; + epoch_exit(); + return empty; +} + +errno_t lfqueue_push(struct lfqueue* queue, void* value) { + struct lfqueue_node* node = node_alloc(value); + if (!node) + return ENOMEM; + + epoch_enter(); + while (true) { + struct lfqueue_node* tail = epoch_deref(&queue->tail); + struct lfqueue_node* next = epoch_deref(&tail->next); + if (tail != atomic_load(&queue->tail, ATOMIC_RELAXED)) + continue; + if (next) { + atomic_compare_exchange(&queue->tail, &tail, next, ATOMIC_RELEASE, ATOMIC_RELAXED); + continue; + } + if (atomic_compare_exchange(&tail->next, &next, node, ATOMIC_RELEASE, ATOMIC_RELAXED)) { + atomic_compare_exchange(&queue->tail, &tail, node, ATOMIC_RELEASE, ATOMIC_RELAXED); + break; + } + } + epoch_exit(); + return 0; +} + +bool lfqueue_pop(struct lfqueue* queue, void** out) { + epoch_enter(); + while (true) { + struct lfqueue_node* head = epoch_deref(&queue->head); + struct lfqueue_node* tail = atomic_load(&queue->tail, ATOMIC_RELAXED); + struct lfqueue_node* next = epoch_deref(&head->next); + if (head != atomic_load(&queue->head, ATOMIC_RELAXED)) + continue; + if (!next) { + epoch_exit(); + return false; + } + if (head == tail) { + atomic_compare_exchange(&queue->tail, &tail, next, ATOMIC_RELEASE, ATOMIC_RELAXED); + continue; + } + void* value = next->value; + if (atomic_compare_exchange(&queue->head, &head, next, ATOMIC_RELEASE, ATOMIC_RELAXED)) { + slab_cache_free(node_cache, head); + epoch_exit(); + *out = value; + return true; + } + } +} diff --git a/lib/lfstack.c b/lib/lfstack.c new file mode 100644 index 0000000..f5ab293 --- /dev/null +++ b/lib/lfstack.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: ISC + +#include +#include + +void lfstack_init(struct lfstack* stack) { + atomic_init(&stack->top, nullptr); +} + +bool lfstack_empty(struct lfstack* stack) { + return atomic_load(&stack->top, ATOMIC_RELAXED) == nullptr; +} + +void lfstack_push(struct lfstack* stack, struct lfstack_node* node) { + struct lfstack_node* top = atomic_load(&stack->top, ATOMIC_RELAXED); + do { + atomic_store(&node->next, top, ATOMIC_RELAXED); + } while (!atomic_compare_exchange_weak(&stack->top, &top, node, ATOMIC_RELEASE, ATOMIC_RELAXED)); +} + +struct lfstack_node* lfstack_pop(struct lfstack* stack) { + epoch_enter(); + struct lfstack_node* top = epoch_deref(&stack->top); + while (top) { + struct lfstack_node* next = atomic_load(&top->next, ATOMIC_RELAXED); + if (atomic_compare_exchange_weak(&stack->top, &top, next, ATOMIC_ACQUIRE, ATOMIC_ACQUIRE)) + break; + } + epoch_exit(); + return top; +} + +struct lfstack_node* lfstack_pop_all(struct lfstack* stack) { + return atomic_exchange(&stack->top, nullptr, ATOMIC_ACQUIRE); +} diff --git a/lib/meson.build b/lib/meson.build index f28f2b1..5bb4e9c 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -4,12 +4,16 @@ kernel_sources += files( 'condvar.c', 'console.c', 'entry.c', + 'epoch.c', 'event.c', 'guid.c', 'initgraph.c', 'iovec.c', 'irq.c', 'ksyms.c', + 'lfhash.c', + 'lfqueue.c', + 'lfstack.c', 'main.c', 'module.c', 'mutex.c', diff --git a/lib/process.c b/lib/process.c index 062ae46..1536c78 100644 --- a/lib/process.c +++ b/lib/process.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -18,19 +19,39 @@ static struct process kernel_process; static atomic_bool kernel_process_ready; static atomic_int next_pid = 1; +struct fd_files { + size_t capacity; + struct epoch_head epoch; + _Atomic(struct vfs_file*) slots[]; +}; + static size_t fd_words(size_t capacity) { return capacity / 64; } +static void fd_files_free(struct epoch_head* head) { + mem_free(containerof(head, struct fd_files, epoch)); +} + +static _Atomic(struct vfs_file*)* fd_slot(struct fd_table* t, size_t fd) { + return &atomic_load(&t->files, ATOMIC_RELAXED)->slots[fd]; +} + static void fd_table_destroy(struct fd_table* t) { - mem_free(t->files); + struct fd_files* files = atomic_load(&t->files, ATOMIC_RELAXED); + atomic_store(&t->files, nullptr, ATOMIC_RELAXED); + if (files) + epoch_call(&files->epoch, fd_files_free); mem_free(t->open); mem_free(t->cloexec); - *t = (struct fd_table){}; + t->open = nullptr; + t->cloexec = nullptr; + t->capacity = 0; + t->next_free = 0; } static errno_t fd_table_resize(struct fd_table* t, size_t capacity) { - struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0); + struct fd_files* files = mem_alloc(sizeof(*files) + capacity * sizeof(files->slots[0]), 0); uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0); uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0); if (!files || !open || !cloexec) { @@ -40,19 +61,23 @@ static errno_t fd_table_resize(struct fd_table* t, size_t capacity) { return ENOMEM; } - if (t->capacity) { - memcpy(files, t->files, t->capacity * sizeof(*files)); + files->capacity = capacity; + struct fd_files* old = atomic_load(&t->files, ATOMIC_RELAXED); + if (old) { + for (size_t i = 0; i < t->capacity; i++) + atomic_store(&files->slots[i], atomic_load(&old->slots[i], ATOMIC_RELAXED), ATOMIC_RELAXED); memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open)); memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec)); } - mem_free(t->files); mem_free(t->open); mem_free(t->cloexec); - t->files = files; t->open = open; t->cloexec = cloexec; t->capacity = capacity; + epoch_publish(&t->files, files); + if (old) + epoch_call(&old->epoch, fd_files_free); return 0; } @@ -76,19 +101,23 @@ static size_t fd_find_free(struct fd_table* t, size_t start) { return MAX(start, t->capacity); } -static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) { +static void fd_set_cloexec(struct fd_table* t, size_t fd, bool cloexec) { const uint64_t bit = 1ull << (fd % 64); - t->files[fd] = file; - t->open[fd / 64] |= bit; if (cloexec) t->cloexec[fd / 64] |= bit; else t->cloexec[fd / 64] &= ~bit; } +static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) { + t->open[fd / 64] |= 1ull << (fd % 64); + fd_set_cloexec(t, fd, cloexec); + epoch_publish(fd_slot(t, fd), file); +} + static void fd_clear(struct fd_table* t, size_t fd) { const uint64_t bit = 1ull << (fd % 64); - t->files[fd] = nullptr; + atomic_store(fd_slot(t, fd), nullptr, ATOMIC_RELAXED); t->open[fd / 64] &= ~bit; t->cloexec[fd / 64] &= ~bit; if (fd < t->next_free) @@ -98,7 +127,7 @@ static void fd_clear(struct fd_table* t, size_t fd) { static struct vfs_file* fd_lookup(struct process* proc, int fd) { if (fd < 0 || (size_t)fd >= proc->files.capacity) return nullptr; - return proc->files.files[fd]; + return atomic_load(fd_slot(&proc->files, fd), ATOMIC_RELAXED); } static void process_init_common(struct process* proc) { @@ -125,7 +154,7 @@ static void close_files(struct process* proc) { mutex_lock(&proc->files_lock); for (size_t w = 0; w < fd_words(t->capacity); w++) { for (uint64_t bits = t->open[w]; bits; bits &= bits - 1) - vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]); + vfs_file_unref(atomic_load(fd_slot(t, w * 64 + __builtin_ctzll(bits)), ATOMIC_RELAXED)); } fd_table_destroy(t); mutex_unlock(&proc->files_lock); @@ -313,7 +342,7 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, mutex_unlock(&proc->files_lock); return status; } - struct vfs_file* displaced = t->files[fd]; + struct vfs_file* displaced = fd_lookup(proc, fd); fd_install(t, fd, file, cloexec); vfs_file_ref(file); mutex_unlock(&proc->files_lock); @@ -339,7 +368,7 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) { mutex_lock(&proc->files_lock); struct vfs_file* file = fd_lookup(proc, fd); if (file) { - fd_install(&proc->files, fd, file, cloexec); + fd_set_cloexec(&proc->files, fd, cloexec); status = 0; } mutex_unlock(&proc->files_lock); @@ -347,11 +376,18 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) { } struct vfs_file* process_fd_get(struct process* proc, int fd) { - mutex_lock(&proc->files_lock); - struct vfs_file* file = fd_lookup(proc, fd); - if (file) - vfs_file_ref(file); - mutex_unlock(&proc->files_lock); + if (fd < 0) + return nullptr; + + struct vfs_file* file = nullptr; + epoch_enter(); + struct fd_files* files = epoch_deref(&proc->files.files); + if (files && (size_t)fd < files->capacity) { + file = epoch_deref(&files->slots[fd]); + if (file && !refcount_inc_not_zero(&file->rc)) + file = nullptr; + } + epoch_exit(); return file; } diff --git a/lib/radix.c b/lib/radix.c index cd6e6bb..ef2a4e2 100644 --- a/lib/radix.c +++ b/lib/radix.c @@ -1,5 +1,7 @@ #include +#include #include +#include static bool covers(const struct radix_node* node, uint64_t index) { return node->shift + RADIX_BITS >= 64 || (index >> (node->shift + RADIX_BITS)) == 0; @@ -9,6 +11,10 @@ static size_t slot_of(const struct radix_node* node, uint64_t index) { return (index >> node->shift) & (RADIX_SLOTS - 1); } +static uint64_t bit(size_t slot) { + return (uint64_t)1 << slot; +} + static struct radix_node* node_alloc(uint8_t shift, struct radix_node* parent, uint8_t offset) { struct radix_node* node = mem_alloc(sizeof(struct radix_node), 0); if (!node) @@ -19,36 +25,46 @@ static struct radix_node* node_alloc(uint8_t shift, struct radix_node* parent, u return node; } +static void node_free(struct epoch_head* head) { + mem_free(containerof(head, struct radix_node, epoch)); +} + +static void node_retire(struct radix_node* node) { + epoch_call(&node->epoch, node_free); +} + static void free_subtree(struct radix_node* node) { if (node->shift > 0) { for (size_t i = 0; i < RADIX_SLOTS; i++) { - if (node->slots[i]) - free_subtree(node->slots[i]); + struct radix_node* child = atomic_load(&node->slots[i], ATOMIC_RELAXED); + if (child) + free_subtree(child); } } mem_free(node); } void radix_init(struct radix_tree* tree) { - tree->root = nullptr; + atomic_init(&tree->root, nullptr); } void radix_destroy(struct radix_tree* tree) { - if (tree->root) - free_subtree(tree->root); - tree->root = nullptr; + struct radix_node* root = atomic_load(&tree->root, ATOMIC_RELAXED); + if (root) + free_subtree(root); + atomic_store(&tree->root, nullptr, ATOMIC_RELAXED); } bool radix_empty(struct radix_tree* tree) { - return tree->root == nullptr; + return atomic_load(&tree->root, ATOMIC_RELAXED) == nullptr; } static struct radix_node* leaf_of(struct radix_tree* tree, uint64_t index) { - struct radix_node* node = tree->root; + struct radix_node* node = epoch_deref(&tree->root); if (!node || !covers(node, index)) return nullptr; while (node->shift > 0) { - node = node->slots[slot_of(node, index)]; + node = epoch_deref(&node->slots[slot_of(node, index)]); if (!node) return nullptr; } @@ -57,37 +73,48 @@ static struct radix_node* leaf_of(struct radix_tree* tree, uint64_t index) { void* radix_lookup(struct radix_tree* tree, uint64_t index) { struct radix_node* leaf = leaf_of(tree, index); - return leaf ? leaf->slots[slot_of(leaf, index)] : nullptr; + return leaf ? epoch_deref(&leaf->slots[slot_of(leaf, index)]) : nullptr; } -void** radix_slot(struct radix_tree* tree, uint64_t index) { +void* radix_replace(struct radix_tree* tree, uint64_t index, void* value) { + assert(value, "Tried to replace a radix slot with nullptr"); struct radix_node* leaf = leaf_of(tree, index); - return leaf ? &leaf->slots[slot_of(leaf, index)] : nullptr; + if (!leaf) + return nullptr; + _Atomic(void*)* slot = &leaf->slots[slot_of(leaf, index)]; + void* old = atomic_load(slot, ATOMIC_RELAXED); + if (old) + epoch_publish(slot, value); + return old; } static errno_t grow(struct radix_tree* tree, uint64_t index) { - if (!tree->root) { + struct radix_node* root = atomic_load(&tree->root, ATOMIC_RELAXED); + if (!root) { uint8_t shift = 0; while (shift + RADIX_BITS < 64 && (index >> (shift + RADIX_BITS)) != 0) shift += RADIX_BITS; - tree->root = node_alloc(shift, nullptr, 0); - return tree->root ? 0 : ENOMEM; + root = node_alloc(shift, nullptr, 0); + if (!root) + return ENOMEM; + epoch_publish(&tree->root, root); + return 0; } - while (!covers(tree->root, index)) { - struct radix_node* old = tree->root; - struct radix_node* top = node_alloc(old->shift + RADIX_BITS, nullptr, 0); + while (!covers(root, index)) { + struct radix_node* top = node_alloc(root->shift + RADIX_BITS, nullptr, 0); if (!top) return ENOMEM; - top->slots[0] = old; + atomic_store(&top->slots[0], root, ATOMIC_RELAXED); top->count = 1; for (size_t t = 0; t < RADIX_NUM_TAGS; t++) { - if (old->tags[t]) - top->tags[t] = 1; + if (atomic_load(&root->tags[t], ATOMIC_RELAXED)) + atomic_store(&top->tags[t], 1, ATOMIC_RELAXED); } - old->parent = top; - old->offset = 0; - tree->root = top; + root->parent = top; + root->offset = 0; + epoch_publish(&tree->root, top); + root = top; } return 0; } @@ -97,32 +124,33 @@ errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value) { if (status) return status; - struct radix_node* node = tree->root; + struct radix_node* node = atomic_load(&tree->root, ATOMIC_RELAXED); while (node->shift > 0) { const size_t slot = slot_of(node, index); - struct radix_node* child = node->slots[slot]; + struct radix_node* child = atomic_load(&node->slots[slot], ATOMIC_RELAXED); if (!child) { child = node_alloc(node->shift - RADIX_BITS, node, slot); if (!child) return ENOMEM; - node->slots[slot] = child; + epoch_publish(&node->slots[slot], child); node->count++; } node = child; } const size_t slot = slot_of(node, index); - if (node->slots[slot]) + if (atomic_load(&node->slots[slot], ATOMIC_RELAXED)) return EEXIST; - node->slots[slot] = value; + epoch_publish(&node->slots[slot], value); node->count++; return 0; } static void clear_tag_upwards(struct radix_node* node, size_t slot, enum radix_tag tag) { while (node) { - node->tags[tag] &= ~((uint64_t)1 << slot); - if (node->tags[tag]) + const uint64_t tags = atomic_load(&node->tags[tag], ATOMIC_RELAXED) & ~bit(slot); + atomic_store(&node->tags[tag], tags, ATOMIC_RELAXED); + if (tags) return; slot = node->offset; node = node->parent; @@ -135,28 +163,28 @@ void* radix_remove(struct radix_tree* tree, uint64_t index) { return nullptr; size_t slot = slot_of(node, index); - void* value = node->slots[slot]; + void* value = atomic_load(&node->slots[slot], ATOMIC_RELAXED); if (!value) return nullptr; for (size_t t = 0; t < RADIX_NUM_TAGS; t++) { - if (node->tags[t] & ((uint64_t)1 << slot)) + if (atomic_load(&node->tags[t], ATOMIC_RELAXED) & bit(slot)) clear_tag_upwards(node, slot, t); } - node->slots[slot] = nullptr; + atomic_store(&node->slots[slot], nullptr, ATOMIC_RELAXED); node->count--; while (node->count == 0) { struct radix_node* parent = node->parent; if (!parent) { - tree->root = nullptr; - mem_free(node); + atomic_store(&tree->root, nullptr, ATOMIC_RELAXED); + node_retire(node); break; } slot = node->offset; - parent->slots[slot] = nullptr; + atomic_store(&parent->slots[slot], nullptr, ATOMIC_RELAXED); parent->count--; - mem_free(node); + node_retire(node); node = parent; } return value; @@ -164,14 +192,16 @@ void* radix_remove(struct radix_tree* tree, uint64_t index) { void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag) { struct radix_node* node = leaf_of(tree, index); - if (!node || !node->slots[slot_of(node, index)]) + if (!node) return; size_t slot = slot_of(node, index); + if (!atomic_load(&node->slots[slot], ATOMIC_RELAXED)) + return; while (node) { - const uint64_t bit = (uint64_t)1 << slot; - if (node->tags[tag] & bit) + const uint64_t tags = atomic_load(&node->tags[tag], ATOMIC_RELAXED); + if (tags & bit(slot)) return; - node->tags[tag] |= bit; + atomic_store(&node->tags[tag], tags | bit(slot), ATOMIC_RELAXED); slot = node->offset; node = node->parent; } @@ -182,17 +212,18 @@ void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag if (!node) return; const size_t slot = slot_of(node, index); - if (node->tags[tag] & ((uint64_t)1 << slot)) + if (atomic_load(&node->tags[tag], ATOMIC_RELAXED) & bit(slot)) clear_tag_upwards(node, slot, tag); } bool radix_tag_get(struct radix_tree* tree, uint64_t index, enum radix_tag tag) { struct radix_node* node = leaf_of(tree, index); - return node && (node->tags[tag] & ((uint64_t)1 << slot_of(node, index))); + return node && (atomic_load(&node->tags[tag], ATOMIC_RELAXED) & bit(slot_of(node, index))); } bool radix_tagged(struct radix_tree* tree, enum radix_tag tag) { - return tree->root && tree->root->tags[tag]; + struct radix_node* root = epoch_deref(&tree->root); + return root && atomic_load(&root->tags[tag], ATOMIC_RELAXED); } struct gang { @@ -212,26 +243,28 @@ static void gang_walk(struct radix_node* node, uint64_t base, uint64_t first, st start = delta; } + const uint64_t tags = g->tag >= 0 ? atomic_load(&node->tags[g->tag], ATOMIC_RELAXED) : UINT64_MAX; for (size_t i = start; i < RADIX_SLOTS && g->found < g->max; i++) { - if (!node->slots[i]) + if (!(tags & bit(i))) continue; - if (g->tag >= 0 && !(node->tags[g->tag] & ((uint64_t)1 << i))) + void* entry = epoch_deref(&node->slots[i]); + if (!entry) continue; const uint64_t index = base + ((uint64_t)i << node->shift); if (node->shift == 0) { if (g->out) - g->out[g->found] = node->slots[i]; + g->out[g->found] = entry; if (g->out_index) g->out_index[g->found] = index; g->found++; } else { - gang_walk(node->slots[i], index, first > index ? first : index, g); + gang_walk(entry, index, first > index ? first : index, g); } } } static size_t gang_lookup(struct radix_tree* tree, uint64_t first, int tag, size_t max, void** out, uint64_t* idx) { - struct radix_node* root = tree->root; + struct radix_node* root = epoch_deref(&tree->root); if (!root || max == 0 || !covers(root, first)) return 0; struct gang g = {.max = max, .out = out, .out_index = idx, .tag = tag}; diff --git a/mem/page.c b/mem/page.c index 26160ee..c6f6f33 100644 --- a/mem/page.c +++ b/mem/page.c @@ -29,6 +29,15 @@ void page_hold(struct page* page) { atomic_fetch_add(&page->holds, 1, ATOMIC_RELAXED); } +bool page_try_hold(struct page* page) { + size_t holds = atomic_load(&page->holds, ATOMIC_RELAXED); + while (holds) { + if (atomic_compare_exchange_weak(&page->holds, &holds, holds + 1, ATOMIC_ACQUIRE, ATOMIC_RELAXED)) + return true; + } + return false; +} + void page_unhold(struct page* page) { if (atomic_fetch_sub(&page->holds, 1, ATOMIC_RELEASE) != 1) return; diff --git a/mem/vma.c b/mem/vma.c index 276c247..0c6cf09 100644 --- a/mem/vma.c +++ b/mem/vma.c @@ -445,7 +445,7 @@ static errno_t fault_anon( errno_t status = anon_new(anon->page, ©); if (status) return status; - *radix_slot(&m->amap->anons, slot) = copy; + radix_replace(&m->amap->anons, slot, copy); anon_unref(anon); anon = copy; writable = true; diff --git a/mem/vmo.c b/mem/vmo.c index 15573aa..aea9091 100644 --- a/mem/vmo.c +++ b/mem/vmo.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: ISC #include +#include #include #include #include @@ -143,10 +144,34 @@ static errno_t fill(struct vmo* vmo, uint64_t index, enum vmo_access access, str return 0; } +static struct page* lookup_fast(struct vmo* vmo, uint64_t index) { + epoch_enter(); + struct page* page = radix_lookup(&vmo->pages, index); + if (page && !page_try_hold(page)) + page = nullptr; + const bool valid = + page && radix_lookup(&vmo->pages, index) == page && !(atomic_load(&page->flags, ATOMIC_ACQUIRE) & PAGE_BUSY); + epoch_exit(); + + if (page && !valid) { + page_unhold(page); + return nullptr; + } + return page; +} + errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) { if (vmo->pager->lookup) return ENOTSUP; + if (access == VMO_READ) { + struct page* page = lookup_fast(vmo, index); + if (page) { + *out = page; + return 0; + } + } + mutex_lock(&vmo->lock); while (true) { struct page* page = radix_lookup(&vmo->pages, index); diff --git a/sched/core.c b/sched/core.c index 41ad4c6..c8b4438 100644 --- a/sched/core.c +++ b/sched/core.c @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -163,3 +165,16 @@ void sched_sleep_until(uint64_t deadline_ns) { const uint8_t key = 0; park_timeout(&key, nullptr, nullptr, nullptr, 0, &deadline_ns); } + +void sched_preempt_disable() { + arch_sched_preempt_disable(); +} + +void sched_preempt_enable() { + if (!arch_sched_preempt_enable()) + return; + struct scheduler* sched = &percpu_get()->sched; + if ((sched->owe_preempt || sched->need_resched) && percpu_ref(irq_data)->depth == 0 && irq_get_state() && + sched->current) + sched_reschedule(sched); +} diff --git a/syscalls/system.c b/syscalls/system.c index 714e6a0..a0025b5 100644 --- a/syscalls/system.c +++ b/syscalls/system.c @@ -93,8 +93,12 @@ struct syscall_ret sys_setuname(const __user struct __utsname* ptr) { struct syscall_ret sys_syslog(enum syslog_kind kind, __user char* message, size_t len) { struct syscall_ret ret = {}; - const size_t buf_len = MIN(len + 1, 256); + const size_t buf_len = MIN(len, 255) + 1; char* buf = mem_alloc(buf_len, ALLOC_NOZERO); + if (!buf) { + ret.error = ENOMEM; + return ret; + } if (!usercopy_read(buf, message, buf_len - 1)) { mem_free(buf);