lib: Add epoch-based SMR

fs/refs.c

@@ -224,6 +224,10 @@ void vfs_file_ref(struct vfs_file* file) {
     refcount_inc(&file->rc);
 }
 
+static void file_free(struct epoch_head* head) {
+    mem_free(containerof(head, struct vfs_file, epoch));
+}
+
 void vfs_file_unref(struct vfs_file* file) {
     if (!refcount_dec(&file->rc))
         return;
@@ -234,7 +238,7 @@ void vfs_file_unref(struct vfs_file* file) {
         vfs_path_drop(file->path);
     vfs_node_unref(file->vnode);
     mem_free(file->absolute_path);
-    mem_free(file);
+    epoch_call(&file->epoch, file_free);
 }
 
 struct vfs_path vfs_path_clone(struct vfs_path path) {

include/zinnia/eplist.h

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/epoch.h>
+
+#define eplist_head(type) \
+    struct { \
+        _Atomic(type*) eph_first; \
+    }
+
+#define eplist_link(type) \
+    struct { \
+        _Atomic(type*) epl_next; \
+        _Atomic(type*)* epl_prev; \
+    }
+
+#define eplist_init(head) atomic_init(&(head)->eph_first, nullptr)
+
+#define eplist_first(head) epoch_deref(&(head)->eph_first)
+
+#define eplist_next(elm, field) epoch_deref(&(elm)->field.epl_next)
+
+#define eplist_empty(head) (atomic_load(&(head)->eph_first, ATOMIC_RELAXED) == nullptr)
+
+#define eplist_foreach(var, head, field) for ((var) = eplist_first(head); (var); (var) = eplist_next(var, field))
+
+#define eplist_insert_head(head, elm, field) \
+    do { \
+        typeof(elm) _epl_first = atomic_load(&(head)->eph_first, ATOMIC_RELAXED); \
+        atomic_store(&(elm)->field.epl_next, _epl_first, ATOMIC_RELAXED); \
+        (elm)->field.epl_prev = &(head)->eph_first; \
+        if (_epl_first) \
+            _epl_first->field.epl_prev = &(elm)->field.epl_next; \
+        epoch_publish(&(head)->eph_first, (elm)); \
+    } while (0)
+
+#define eplist_insert_after(listelm, elm, field) \
+    do { \
+        typeof(elm) _epl_next = atomic_load(&(listelm)->field.epl_next, ATOMIC_RELAXED); \
+        atomic_store(&(elm)->field.epl_next, _epl_next, ATOMIC_RELAXED); \
+        (elm)->field.epl_prev = &(listelm)->field.epl_next; \
+        if (_epl_next) \
+            _epl_next->field.epl_prev = &(elm)->field.epl_next; \
+        epoch_publish(&(listelm)->field.epl_next, (elm)); \
+    } while (0)
+
+#define eplist_insert_before(listelm, elm, field) \
+    do { \
+        atomic_store(&(elm)->field.epl_next, (listelm), ATOMIC_RELAXED); \
+        (elm)->field.epl_prev = (listelm)->field.epl_prev; \
+        epoch_publish((listelm)->field.epl_prev, (elm)); \
+        (listelm)->field.epl_prev = &(elm)->field.epl_next; \
+    } while (0)
+
+#define eplist_remove(elm, field) \
+    do { \
+        typeof(elm) _epl_next = atomic_load(&(elm)->field.epl_next, ATOMIC_RELAXED); \
+        if (_epl_next) \
+            _epl_next->field.epl_prev = (elm)->field.epl_prev; \
+        epoch_publish((elm)->field.epl_prev, _epl_next); \
+    } while (0)

include/zinnia/epoch.h

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/initgraph.h>
+#include <stdint.h>
+
+struct epoch_head {
+    struct epoch_head* next;
+    void (*func)(struct epoch_head* head);
+};
+
+// Enters a read-side critical section.
+// Disables preemption, so the caller must not sleep.
+// A reader may call this function even while already in an epoch section.
+void epoch_enter();
+
+// Leaves a read-side critical section.
+void epoch_exit();
+
+// Returns true if the current CPU is inside a critical section.
+bool epoch_active();
+
+// Runs `func` once all readers that could observe `head` have left.
+void epoch_call(struct epoch_head* head, void (*func)(struct epoch_head* head));
+
+// Advances the global epoch if all active readers have observed it.
+bool epoch_try_advance();
+
+// Returns the current global epoch.
+uint64_t epoch_current();
+
+// Waits until all readers active at the time of the call have left.
+void epoch_synchronize();
+
+// Waits until all previously queued callbacks have run.
+void epoch_barrier();
+
+// Loads a pointer protected by an epoch critical section.
+#define epoch_deref(ptr) atomic_load(ptr, ATOMIC_ACQUIRE)
+
+// Publishes a pointer to epoch readers.
+#define epoch_publish(ptr, val) atomic_store(ptr, val, ATOMIC_RELEASE)
+
+INIT_EXTERN(epoch_stage);

include/zinnia/lfhash.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr size_t LFHASH_SEGMENT_BUCKETS = 512;
+constexpr size_t LFHASH_SEGMENTS = 1024;
+constexpr size_t LFHASH_MIN_BUCKETS = 16;
+constexpr size_t LFHASH_LOAD_FACTOR = 2;
+
+struct lfhash_node {
+    _Atomic(uintptr_t) next;
+    uint64_t key;
+};
+
+struct lfhash_bucket;
+
+struct lfhash {
+    _Atomic(struct lfhash_bucket*)* segments;
+    atomic_size_t size;
+    atomic_size_t count;
+};
+
+typedef bool (*lfhash_eq_fn)(const struct lfhash_node* node, const void* key);
+
+errno_t lfhash_init(struct lfhash* table);
+void lfhash_destroy(struct lfhash* table);
+size_t lfhash_count(struct lfhash* table);
+
+errno_t lfhash_insert(struct lfhash* table, uint64_t hash, struct lfhash_node* node, lfhash_eq_fn eq, const void* key);
+struct lfhash_node* lfhash_lookup(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key);
+struct lfhash_node* lfhash_remove(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key);
+
+struct lfhash_node* lfhash_first(struct lfhash* table);
+struct lfhash_node* lfhash_next(struct lfhash_node* node);
+
+#define lfhash_foreach(table, var) for ((var) = lfhash_first(table); (var); (var) = lfhash_next(var))

include/zinnia/lfqueue.h

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+
+struct lfqueue_node;
+
+struct lfqueue {
+    _Atomic(struct lfqueue_node*) head;
+    _Atomic(struct lfqueue_node*) tail;
+};
+
+errno_t lfqueue_init(struct lfqueue* queue);
+void lfqueue_destroy(struct lfqueue* queue);
+bool lfqueue_empty(struct lfqueue* queue);
+errno_t lfqueue_push(struct lfqueue* queue, void* value);
+bool lfqueue_pop(struct lfqueue* queue, void** out);

include/zinnia/lfstack.h

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+
+struct lfstack_node {
+    _Atomic(struct lfstack_node*) next;
+};
+
+struct lfstack {
+    _Atomic(struct lfstack_node*) top;
+};
+
+void lfstack_init(struct lfstack* stack);
+bool lfstack_empty(struct lfstack* stack);
+void lfstack_push(struct lfstack* stack, struct lfstack_node* node);
+struct lfstack_node* lfstack_pop(struct lfstack* stack);
+struct lfstack_node* lfstack_pop_all(struct lfstack* stack);

include/zinnia/process.h

@@ -25,8 +25,10 @@ struct process_rlimit {
     rlim_t max;
 };
 
+struct fd_files;
+
 struct fd_table {
-    struct vfs_file** files;
+    _Atomic(struct fd_files*) files;
     uint64_t* open;
     uint64_t* cloexec;
     size_t capacity;

include/zinnia/radix.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zinnia/epoch.h>
 #include <zinnia/errno.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -20,12 +21,13 @@ struct radix_node {
     uint8_t count;
     uint8_t offset;
     struct radix_node* parent;
-    uint64_t tags[RADIX_NUM_TAGS];
+    _Atomic(uint64_t) tags[RADIX_NUM_TAGS];
-    void* slots[RADIX_SLOTS];
+    _Atomic(void*) slots[RADIX_SLOTS];
+    struct epoch_head epoch;
 };
 
 struct radix_tree {
-    struct radix_node* root;
+    _Atomic(struct radix_node*) root;
 };
 
 void radix_init(struct radix_tree* tree);
@@ -35,7 +37,7 @@ bool radix_empty(struct radix_tree* tree);
 void* radix_lookup(struct radix_tree* tree, uint64_t index);
 errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value);
 void* radix_remove(struct radix_tree* tree, uint64_t index);
-void** radix_slot(struct radix_tree* tree, uint64_t index);
+void* radix_replace(struct radix_tree* tree, uint64_t index, void* value);
 
 void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag);
 void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag);

include/zinnia/rcu.h

@@ -1,6 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-void rcu_read_lock();
-void rcu_read_unlock();

include/zinnia/sched.h

@@ -135,6 +135,9 @@ void sched_notify(struct task* task);
 
 void sched_sleep_until(uint64_t deadline_ns);
 
+void sched_preempt_disable();
+void sched_preempt_enable();
+
 [[noreturn]]
 void sched_exit();
 

include/zinnia/vfs.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zinnia/epoch.h>
 #include <zinnia/errno.h>
 #include <zinnia/ident.h>
 #include <zinnia/initgraph.h>
@@ -356,6 +357,8 @@ struct vfs_file {
 
     // Driver data.
     void* priv;
+
+    struct epoch_head epoch;
 };
 
 // A file system driver.

include/zinnia/virt.h

@@ -47,6 +47,7 @@ struct page* vm_page_from_addr(phys_t addr);
 
 errno_t page_alloc(enum alloc_flags flags, struct page** out);
 void page_hold(struct page* page);
+bool page_try_hold(struct page* page);
 void page_unhold(struct page* page);
 bool page_try_busy(struct page* page);
 void page_unbusy(struct page* page);

lib/epoch.c

@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
+#include <zinnia/epoch.h>
+#include <zinnia/irq.h>
+#include <zinnia/mutex.h>
+#include <zinnia/percpu.h>
+#include <zinnia/process.h>
+#include <zinnia/sched.h>
+#include <zinnia/spin.h>
+#include <zinnia/timer.h>
+#include <arch/cpu.h>
+
+constexpr uint64_t EPOCH_NEST_BITS = 16;
+constexpr uint64_t EPOCH_NEST_MASK = ((uint64_t)1 << EPOCH_NEST_BITS) - 1;
+constexpr size_t EPOCH_BUCKETS = 3;
+constexpr size_t EPOCH_BATCH = 64;
+constexpr size_t EPOCH_SYNC_SPINS = 64;
+constexpr uint64_t EPOCH_POLL_NS = 10'000'000;
+constexpr uint64_t EPOCH_SYNC_POLL_NS = 1'000'000;
+
+struct epoch_list {
+    struct epoch_head* head;
+    struct epoch_head* tail;
+};
+
+struct epoch_bucket {
+    uint64_t epoch;
+    struct epoch_list list;
+};
+
+struct epoch_cpu {
+    _Atomic(uint64_t) local;
+    struct spinlock lock;
+    struct epoch_bucket buckets[EPOCH_BUCKETS];
+    struct epoch_list ready;
+};
+
+[[__percpu]]
+static struct epoch_cpu epoch_data;
+
+static _Atomic(uint64_t) global_epoch = 0;
+static atomic_size_t pending = 0;
+static mutex_t reclaim_lock;
+static struct timer poll_timer;
+static _Atomic(struct task*) reclaimer = nullptr;
+static atomic_bool reclaimer_idle = false;
+
+void epoch_enter() {
+    sched_preempt_disable();
+    _Atomic(uint64_t)* local = &percpu_ref(epoch_data)->local;
+    const uint64_t value = atomic_load(local, ATOMIC_RELAXED);
+
+    // If we already entered the critical section before, just increment the nest counter.
+    if (value & EPOCH_NEST_MASK) {
+        assert((value & EPOCH_NEST_MASK) != EPOCH_NEST_MASK, "Epoch nesting overflow");
+        atomic_store(local, value + 1, ATOMIC_RELAXED);
+        return;
+    }
+
+    // Store the current epoch in the upper bits, and mark this CPU as active.
+    atomic_store(local, (atomic_load(&global_epoch, ATOMIC_RELAXED) << EPOCH_NEST_BITS) | 1, ATOMIC_RELAXED);
+    atomic_fence(ATOMIC_SEQ_CST);
+}
+
+void epoch_exit() {
+    _Atomic(uint64_t)* local = &percpu_ref(epoch_data)->local;
+    const uint64_t value = atomic_load(local, ATOMIC_RELAXED);
+    assert(value & EPOCH_NEST_MASK, "Tried to exit an epoch section without entering one");
+
+    // If we didn't nest, we can clear the local epoch.
+    // Otherwise, decrease the nest counter.
+    if ((value & EPOCH_NEST_MASK) == 1)
+        atomic_store(local, 0, ATOMIC_RELEASE);
+    else
+        atomic_store(local, value - 1, ATOMIC_RELAXED);
+
+    sched_preempt_enable();
+}
+
+bool epoch_active() {
+    sched_preempt_disable();
+    const bool active = atomic_load(&percpu_ref(epoch_data)->local, ATOMIC_RELAXED) & EPOCH_NEST_MASK;
+    sched_preempt_enable();
+    return active;
+}
+
+static void list_push(struct epoch_list* list, struct epoch_head* head) {
+    head->next = list->head;
+    if (!list->head)
+        list->tail = head;
+    list->head = head;
+}
+
+static void list_splice(struct epoch_list* dst, struct epoch_list* src) {
+    if (!src->head)
+        return;
+    src->tail->next = dst->head;
+    if (!dst->head)
+        dst->tail = src->tail;
+    dst->head = src->head;
+    *src = (struct epoch_list){};
+}
+
+static void wake_reclaimer() {
+    struct task* task = atomic_load(&reclaimer, ATOMIC_ACQUIRE);
+    if (task && atomic_exchange(&reclaimer_idle, false, ATOMIC_SEQ_CST))
+        sched_wake(task);
+}
+
+void epoch_call(struct epoch_head* head, void (*func)(struct epoch_head* head)) {
+    head->func = func;
+    const size_t old = atomic_fetch_add(&pending, 1, ATOMIC_SEQ_CST);
+    atomic_fence(ATOMIC_SEQ_CST);
+
+    irq_lock();
+    struct epoch_cpu* cpu = percpu_ref(epoch_data);
+    spin_lock(&cpu->lock);
+
+    // Store the object head in the bucket of this epoch.
+    // We only need to store 3 buckets, since only these epochs are relevant:
+    // - Current epoch, may have readers
+    // - Previous epoch, may still have readers
+    // - Safe epoch, has no readers remaining
+    const uint64_t now = atomic_load(&global_epoch, ATOMIC_RELAXED);
+    struct epoch_bucket* bucket = &cpu->buckets[now % EPOCH_BUCKETS];
+    if (bucket->epoch != now) {
+        // Combine both epoch lists.
+        list_splice(&cpu->ready, &bucket->list);
+        bucket->epoch = now;
+    }
+    list_push(&bucket->list, head);
+
+    spin_unlock(&cpu->lock);
+    irq_unlock();
+
+    if (old == 0 || old % EPOCH_BATCH == 0)
+        wake_reclaimer();
+}
+
+uint64_t epoch_current() {
+    return atomic_load(&global_epoch, ATOMIC_RELAXED);
+}
+
+bool epoch_try_advance() {
+    uint64_t now = atomic_load(&global_epoch, ATOMIC_RELAXED);
+    atomic_fence(ATOMIC_SEQ_CST);
+    const size_t count = percpu_count();
+
+    // Go through all CPUs.
+    for (size_t i = 0; i < count; i++) {
+        const uint64_t local = atomic_load(&percpu_ref_on(percpu_cpu(i), epoch_data)->local, ATOMIC_ACQUIRE);
+
+        // If any of the CPUs are in a critical section, and their local epoch
+        // is not the same as the current global epoch, we can't advance.
+        if ((local & EPOCH_NEST_MASK) && (local >> EPOCH_NEST_BITS) != now)
+            return false;
+    }
+
+    // At this point all readers agree on the epoch, it's safe to increment now.
+    atomic_compare_exchange(&global_epoch, &now, now + 1, ATOMIC_SEQ_CST, ATOMIC_RELAXED);
+    return true;
+}
+
+static struct epoch_list collect(struct epoch_cpu* cpu, uint64_t now) {
+    spin_lock(&cpu->lock);
+    for (size_t i = 0; i < EPOCH_BUCKETS; i++) {
+        struct epoch_bucket* bucket = &cpu->buckets[i];
+        if (bucket->epoch + 2 <= now)
+            list_splice(&cpu->ready, &bucket->list);
+    }
+    struct epoch_list out = cpu->ready;
+    cpu->ready = (struct epoch_list){};
+    spin_unlock(&cpu->lock);
+    return out;
+}
+
+static void run(struct epoch_head* head) {
+    size_t count = 0;
+    while (head) {
+        struct epoch_head* next = head->next;
+        head->func(head);
+        head = next;
+        count++;
+    }
+    if (count)
+        atomic_fetch_sub(&pending, count, ATOMIC_RELAXED);
+}
+
+static void reclaim() {
+    mutex_lock(&reclaim_lock);
+    const uint64_t now = atomic_load(&global_epoch, ATOMIC_ACQUIRE);
+    const size_t count = percpu_count();
+    for (size_t i = 0; i < count; i++)
+        run(collect(percpu_ref_on(percpu_cpu(i), epoch_data), now).head);
+    mutex_unlock(&reclaim_lock);
+}
+
+void epoch_synchronize() {
+    // We can't synchronize a write while being a reader ourselves.
+    assert(!epoch_active(), "Tried to synchronize inside an epoch section");
+
+    atomic_fence(ATOMIC_SEQ_CST);
+    const uint64_t target = atomic_load(&global_epoch, ATOMIC_RELAXED) + 2;
+    for (size_t spins = 0; atomic_load(&global_epoch, ATOMIC_ACQUIRE) < target; spins++) {
+        if (epoch_try_advance())
+            continue;
+        if (spins < EPOCH_SYNC_SPINS)
+            arch_cpu_pause();
+        else
+            sched_sleep_until(clock_get_elapsed_ns() + EPOCH_SYNC_POLL_NS);
+    }
+}
+
+void epoch_barrier() {
+    epoch_synchronize();
+    reclaim();
+}
+
+static void poll_tick(struct timer*) {
+    wake_reclaimer();
+}
+
+static void reclaimer_fn(uintptr_t) {
+    while (true) {
+        if (atomic_load(&pending, ATOMIC_RELAXED)) {
+            epoch_try_advance();
+            reclaim();
+        }
+
+        atomic_store(&reclaimer_idle, true, ATOMIC_SEQ_CST);
+        if (atomic_load(&pending, ATOMIC_SEQ_CST))
+            timer_arm(&poll_timer, clock_get_elapsed_ns() + EPOCH_POLL_NS);
+        sched_block(&percpu_get()->sched);
+    }
+}
+
+static void epoch_stage_init(struct boot_info*) {
+    mutex_init(&reclaim_lock);
+    timer_init(&poll_timer, poll_tick);
+
+    struct task* task;
+    errno_t status = task_create("epoch", process_kernel(), reclaimer_fn, 0, &task);
+    assert(status == 0, "Unable to create the epoch reclaimer");
+    sched_set_priority(task, SCHED_PRIO_WORK_HIGH);
+    atomic_store(&reclaimer, task, ATOMIC_RELEASE);
+    sched_add_task(&percpu_get()->sched, task);
+}
+INIT_TASK(epoch_stage, "core.epoch", epoch_stage_init);
+INIT_DEPENDS(epoch_stage, sched_stage);

lib/lfhash.c

@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/epoch.h>
+#include <zinnia/lfhash.h>
+#include <arch/cpu.h>
+
+constexpr uintptr_t LFHASH_MARK = 1;
+constexpr uint64_t LFHASH_MSB = (uint64_t)1 << 63;
+
+enum bucket_state : uint8_t {
+    BUCKET_UNLINKED,
+    BUCKET_LINKING,
+    BUCKET_LINKED,
+};
+
+struct lfhash_bucket {
+    struct lfhash_node node;
+    _Atomic(enum bucket_state) state;
+};
+
+struct find {
+    _Atomic(uintptr_t)* prev;
+    struct lfhash_node* cur;
+};
+
+static uint64_t bitrev(uint64_t x) {
+    x = ((x >> 1) & 0x5555555555555555) | ((x & 0x5555555555555555) << 1);
+    x = ((x >> 2) & 0x3333333333333333) | ((x & 0x3333333333333333) << 2);
+    x = ((x >> 4) & 0x0F0F0F0F0F0F0F0F) | ((x & 0x0F0F0F0F0F0F0F0F) << 4);
+    return __builtin_bswap64(x);
+}
+
+static uint64_t regular_key(uint64_t hash) {
+    return bitrev(hash | LFHASH_MSB);
+}
+
+static uint64_t sentinel_key(size_t bucket) {
+    return bitrev(bucket);
+}
+
+static size_t parent_of(size_t bucket) {
+    return bucket & ~((size_t)1 << (63 - __builtin_clzll(bucket)));
+}
+
+static struct lfhash_node* node_of(uintptr_t link) {
+    return (struct lfhash_node*)(link & ~LFHASH_MARK);
+}
+
+static struct lfhash_bucket* bucket_get(struct lfhash* table, size_t bucket) {
+    struct lfhash_bucket* segment = atomic_load(&table->segments[bucket / LFHASH_SEGMENT_BUCKETS], ATOMIC_ACQUIRE);
+    return segment ? &segment[bucket % LFHASH_SEGMENT_BUCKETS] : nullptr;
+}
+
+static struct lfhash_bucket* segment_ensure(struct lfhash* table, size_t bucket) {
+    _Atomic(struct lfhash_bucket*)* slot = &table->segments[bucket / LFHASH_SEGMENT_BUCKETS];
+    struct lfhash_bucket* segment = atomic_load(slot, ATOMIC_ACQUIRE);
+    if (!segment) {
+        struct lfhash_bucket* fresh = mem_alloc(sizeof(struct lfhash_bucket) * LFHASH_SEGMENT_BUCKETS, 0);
+        if (!fresh)
+            return nullptr;
+        if (atomic_compare_exchange(slot, &segment, fresh, ATOMIC_ACQ_REL, ATOMIC_ACQUIRE))
+            segment = fresh;
+        else
+            mem_free(fresh);
+    }
+    return &segment[bucket % LFHASH_SEGMENT_BUCKETS];
+}
+
+static bool find(_Atomic(uintptr_t)* head, uint64_t key, lfhash_eq_fn eq, const void* eq_key, struct find* out) {
+again:
+    _Atomic(uintptr_t)* prev = head;
+    struct lfhash_node* cur = node_of(atomic_load(prev, ATOMIC_ACQUIRE));
+    while (cur) {
+        const uintptr_t next = atomic_load(&cur->next, ATOMIC_ACQUIRE);
+        if (next & LFHASH_MARK) {
+            uintptr_t expected = (uintptr_t)cur;
+            if (!atomic_compare_exchange(prev, &expected, next & ~LFHASH_MARK, ATOMIC_RELEASE, ATOMIC_RELAXED))
+                goto again;
+            cur = node_of(next);
+            continue;
+        }
+        if (cur->key > key)
+            break;
+        if (cur->key == key && (!eq || eq(cur, eq_key))) {
+            *out = (struct find){.prev = prev, .cur = cur};
+            return true;
+        }
+        prev = &cur->next;
+        cur = node_of(next);
+    }
+    *out = (struct find){.prev = prev, .cur = cur};
+    return false;
+}
+
+static void bucket_link(struct lfhash* table, size_t index, struct lfhash_bucket* bucket) {
+    enum bucket_state state = BUCKET_UNLINKED;
+    if (!atomic_compare_exchange(&bucket->state, &state, BUCKET_LINKING, ATOMIC_RELAXED, ATOMIC_RELAXED)) {
+        while (atomic_load(&bucket->state, ATOMIC_ACQUIRE) != BUCKET_LINKED)
+            arch_cpu_pause();
+        return;
+    }
+
+    const size_t parent_index = parent_of(index);
+    struct lfhash_bucket* parent = bucket_get(table, parent_index);
+    if (atomic_load(&parent->state, ATOMIC_ACQUIRE) != BUCKET_LINKED)
+        bucket_link(table, parent_index, parent);
+
+    bucket->node.key = sentinel_key(index);
+    while (true) {
+        struct find f;
+        find(&parent->node.next, bucket->node.key, nullptr, nullptr, &f);
+        atomic_store(&bucket->node.next, (uintptr_t)f.cur, ATOMIC_RELAXED);
+        uintptr_t expected = (uintptr_t)f.cur;
+        if (atomic_compare_exchange(f.prev, &expected, (uintptr_t)&bucket->node, ATOMIC_RELEASE, ATOMIC_RELAXED))
+            break;
+    }
+    atomic_store(&bucket->state, BUCKET_LINKED, ATOMIC_RELEASE);
+}
+
+static bool ensure_segments(struct lfhash* table, size_t index) {
+    while (index) {
+        if (!segment_ensure(table, index))
+            return false;
+        index = parent_of(index);
+    }
+    return true;
+}
+
+static _Atomic(uintptr_t)* start_of(struct lfhash* table, size_t index) {
+    while (true) {
+        struct lfhash_bucket* bucket = bucket_get(table, index);
+        if (bucket && atomic_load(&bucket->state, ATOMIC_ACQUIRE) == BUCKET_LINKED)
+            return &bucket->node.next;
+        index = parent_of(index);
+    }
+}
+
+static size_t index_of(struct lfhash* table, uint64_t hash) {
+    return hash & (atomic_load(&table->size, ATOMIC_RELAXED) - 1);
+}
+
+errno_t lfhash_init(struct lfhash* table) {
+    table->segments = mem_alloc(sizeof(*table->segments) * LFHASH_SEGMENTS, 0);
+    if (!table->segments)
+        return ENOMEM;
+    struct lfhash_bucket* root = segment_ensure(table, 0);
+    if (!root) {
+        mem_free(table->segments);
+        return ENOMEM;
+    }
+    root->node.key = sentinel_key(0);
+    atomic_init(&root->node.next, 0);
+    atomic_init(&root->state, BUCKET_LINKED);
+    atomic_init(&table->size, LFHASH_MIN_BUCKETS);
+    atomic_init(&table->count, 0);
+    return 0;
+}
+
+void lfhash_destroy(struct lfhash* table) {
+    for (size_t i = 0; i < LFHASH_SEGMENTS; i++)
+        mem_free(atomic_load(&table->segments[i], ATOMIC_RELAXED));
+    mem_free(table->segments);
+    table->segments = nullptr;
+}
+
+size_t lfhash_count(struct lfhash* table) {
+    return atomic_load(&table->count, ATOMIC_RELAXED);
+}
+
+static void maybe_grow(struct lfhash* table, size_t count) {
+    size_t size = atomic_load(&table->size, ATOMIC_RELAXED);
+    if (count / size > LFHASH_LOAD_FACTOR && size * 2 <= LFHASH_SEGMENTS * LFHASH_SEGMENT_BUCKETS)
+        atomic_compare_exchange(&table->size, &size, size * 2, ATOMIC_RELAXED, ATOMIC_RELAXED);
+}
+
+errno_t lfhash_insert(struct lfhash* table, uint64_t hash, struct lfhash_node* node, lfhash_eq_fn eq, const void* key) {
+    node->key = regular_key(hash);
+    const size_t index = index_of(table, hash);
+    const bool linkable = ensure_segments(table, index);
+
+    epoch_enter();
+    if (linkable) {
+        struct lfhash_bucket* bucket = bucket_get(table, index);
+        if (atomic_load(&bucket->state, ATOMIC_ACQUIRE) != BUCKET_LINKED)
+            bucket_link(table, index, bucket);
+    }
+
+    _Atomic(uintptr_t)* start = start_of(table, index);
+    while (true) {
+        struct find f;
+        if (find(start, node->key, eq, key, &f)) {
+            epoch_exit();
+            return EEXIST;
+        }
+        atomic_store(&node->next, (uintptr_t)f.cur, ATOMIC_RELAXED);
+        uintptr_t expected = (uintptr_t)f.cur;
+        if (atomic_compare_exchange(f.prev, &expected, (uintptr_t)node, ATOMIC_RELEASE, ATOMIC_RELAXED))
+            break;
+    }
+    epoch_exit();
+
+    maybe_grow(table, atomic_fetch_add(&table->count, 1, ATOMIC_RELAXED) + 1);
+    return 0;
+}
+
+struct lfhash_node* lfhash_lookup(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key) {
+    struct find f;
+    return find(start_of(table, index_of(table, hash)), regular_key(hash), eq, key, &f) ? f.cur : nullptr;
+}
+
+struct lfhash_node* lfhash_remove(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key) {
+    const uint64_t so_key = regular_key(hash);
+    struct lfhash_node* victim = nullptr;
+
+    epoch_enter();
+    _Atomic(uintptr_t)* start = start_of(table, index_of(table, hash));
+    while (true) {
+        struct find f;
+        if (!find(start, so_key, eq, key, &f))
+            break;
+        uintptr_t next = atomic_load(&f.cur->next, ATOMIC_ACQUIRE);
+        if (next & LFHASH_MARK)
+            continue;
+        if (!atomic_compare_exchange(&f.cur->next, &next, next | LFHASH_MARK, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            continue;
+        victim = f.cur;
+        uintptr_t expected = (uintptr_t)victim;
+        if (!atomic_compare_exchange(f.prev, &expected, next, ATOMIC_RELEASE, ATOMIC_RELAXED))
+            find(start, so_key, eq, key, &f);
+        break;
+    }
+    epoch_exit();
+
+    if (victim)
+        atomic_fetch_sub(&table->count, 1, ATOMIC_RELAXED);
+    return victim;
+}
+
+static struct lfhash_node* skip(uintptr_t link) {
+    struct lfhash_node* node = node_of(link);
+    while (node) {
+        const uintptr_t next = atomic_load(&node->next, ATOMIC_ACQUIRE);
+        if (!(next & LFHASH_MARK) && (node->key & 1))
+            return node;
+        node = node_of(next);
+    }
+    return nullptr;
+}
+
+struct lfhash_node* lfhash_first(struct lfhash* table) {
+    return skip(atomic_load(&bucket_get(table, 0)->node.next, ATOMIC_ACQUIRE));
+}
+
+struct lfhash_node* lfhash_next(struct lfhash_node* node) {
+    return skip(atomic_load(&node->next, ATOMIC_ACQUIRE));
+}

lib/lfqueue.c

@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/epoch.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/lfqueue.h>
+#include <zinnia/slab.h>
+
+struct lfqueue_node {
+    _Atomic(struct lfqueue_node*) next;
+    void* value;
+};
+
+static struct slab_cache* node_cache;
+
+static void lfqueue_stage_init(struct boot_info*) {
+    node_cache =
+        slab_cache_new("lfqueue_node", sizeof(struct lfqueue_node), alignof(struct lfqueue_node), SLAB_EPOCH_SAFE);
+    assert(node_cache, "Unable to create the lfqueue node cache");
+}
+INIT_TASK(lfqueue_stage, "core.lfqueue", lfqueue_stage_init);
+INIT_DEPENDS(lfqueue_stage, mem_stage);
+
+static struct lfqueue_node* node_alloc(void* value) {
+    struct lfqueue_node* node = slab_cache_alloc(node_cache, 0);
+    if (node)
+        node->value = value;
+    return node;
+}
+
+errno_t lfqueue_init(struct lfqueue* queue) {
+    struct lfqueue_node* dummy = node_alloc(nullptr);
+    if (!dummy)
+        return ENOMEM;
+    atomic_init(&queue->head, dummy);
+    atomic_init(&queue->tail, dummy);
+    return 0;
+}
+
+void lfqueue_destroy(struct lfqueue* queue) {
+    struct lfqueue_node* node = atomic_load(&queue->head, ATOMIC_ACQUIRE);
+    while (node) {
+        struct lfqueue_node* next = atomic_load(&node->next, ATOMIC_RELAXED);
+        slab_cache_free(node_cache, node);
+        node = next;
+    }
+    atomic_store(&queue->head, nullptr, ATOMIC_RELAXED);
+    atomic_store(&queue->tail, nullptr, ATOMIC_RELAXED);
+}
+
+bool lfqueue_empty(struct lfqueue* queue) {
+    epoch_enter();
+    struct lfqueue_node* head = epoch_deref(&queue->head);
+    const bool empty = atomic_load(&head->next, ATOMIC_RELAXED) == nullptr;
+    epoch_exit();
+    return empty;
+}
+
+errno_t lfqueue_push(struct lfqueue* queue, void* value) {
+    struct lfqueue_node* node = node_alloc(value);
+    if (!node)
+        return ENOMEM;
+
+    epoch_enter();
+    while (true) {
+        struct lfqueue_node* tail = epoch_deref(&queue->tail);
+        struct lfqueue_node* next = epoch_deref(&tail->next);
+        if (tail != atomic_load(&queue->tail, ATOMIC_RELAXED))
+            continue;
+        if (next) {
+            atomic_compare_exchange(&queue->tail, &tail, next, ATOMIC_RELEASE, ATOMIC_RELAXED);
+            continue;
+        }
+        if (atomic_compare_exchange(&tail->next, &next, node, ATOMIC_RELEASE, ATOMIC_RELAXED)) {
+            atomic_compare_exchange(&queue->tail, &tail, node, ATOMIC_RELEASE, ATOMIC_RELAXED);
+            break;
+        }
+    }
+    epoch_exit();
+    return 0;
+}
+
+bool lfqueue_pop(struct lfqueue* queue, void** out) {
+    epoch_enter();
+    while (true) {
+        struct lfqueue_node* head = epoch_deref(&queue->head);
+        struct lfqueue_node* tail = atomic_load(&queue->tail, ATOMIC_RELAXED);
+        struct lfqueue_node* next = epoch_deref(&head->next);
+        if (head != atomic_load(&queue->head, ATOMIC_RELAXED))
+            continue;
+        if (!next) {
+            epoch_exit();
+            return false;
+        }
+        if (head == tail) {
+            atomic_compare_exchange(&queue->tail, &tail, next, ATOMIC_RELEASE, ATOMIC_RELAXED);
+            continue;
+        }
+        void* value = next->value;
+        if (atomic_compare_exchange(&queue->head, &head, next, ATOMIC_RELEASE, ATOMIC_RELAXED)) {
+            slab_cache_free(node_cache, head);
+            epoch_exit();
+            *out = value;
+            return true;
+        }
+    }
+}

lib/lfstack.c

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/epoch.h>
+#include <zinnia/lfstack.h>
+
+void lfstack_init(struct lfstack* stack) {
+    atomic_init(&stack->top, nullptr);
+}
+
+bool lfstack_empty(struct lfstack* stack) {
+    return atomic_load(&stack->top, ATOMIC_RELAXED) == nullptr;
+}
+
+void lfstack_push(struct lfstack* stack, struct lfstack_node* node) {
+    struct lfstack_node* top = atomic_load(&stack->top, ATOMIC_RELAXED);
+    do {
+        atomic_store(&node->next, top, ATOMIC_RELAXED);
+    } while (!atomic_compare_exchange_weak(&stack->top, &top, node, ATOMIC_RELEASE, ATOMIC_RELAXED));
+}
+
+struct lfstack_node* lfstack_pop(struct lfstack* stack) {
+    epoch_enter();
+    struct lfstack_node* top = epoch_deref(&stack->top);
+    while (top) {
+        struct lfstack_node* next = atomic_load(&top->next, ATOMIC_RELAXED);
+        if (atomic_compare_exchange_weak(&stack->top, &top, next, ATOMIC_ACQUIRE, ATOMIC_ACQUIRE))
+            break;
+    }
+    epoch_exit();
+    return top;
+}
+
+struct lfstack_node* lfstack_pop_all(struct lfstack* stack) {
+    return atomic_exchange(&stack->top, nullptr, ATOMIC_ACQUIRE);
+}

lib/meson.build

@@ -4,12 +4,16 @@ kernel_sources += files(
     'condvar.c',
     'console.c',
     'entry.c',
+    'epoch.c',
     'event.c',
     'guid.c',
     'initgraph.c',
     'iovec.c',
     'irq.c',
     'ksyms.c',
+    'lfhash.c',
+    'lfqueue.c',
+    'lfstack.c',
     'main.c',
     'module.c',
     'mutex.c',

lib/process.c

@@ -2,6 +2,7 @@
 
 #include <zinnia/alloc.h>
 #include <zinnia/atomic.h>
+#include <zinnia/epoch.h>
 #include <zinnia/exec.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
@@ -18,19 +19,39 @@ static struct process kernel_process;
 static atomic_bool kernel_process_ready;
 static atomic_int next_pid = 1;
 
+struct fd_files {
+    size_t capacity;
+    struct epoch_head epoch;
+    _Atomic(struct vfs_file*) slots[];
+};
+
 static size_t fd_words(size_t capacity) {
     return capacity / 64;
 }
 
+static void fd_files_free(struct epoch_head* head) {
+    mem_free(containerof(head, struct fd_files, epoch));
+}
+
+static _Atomic(struct vfs_file*)* fd_slot(struct fd_table* t, size_t fd) {
+    return &atomic_load(&t->files, ATOMIC_RELAXED)->slots[fd];
+}
+
 static void fd_table_destroy(struct fd_table* t) {
-    mem_free(t->files);
+    struct fd_files* files = atomic_load(&t->files, ATOMIC_RELAXED);
+    atomic_store(&t->files, nullptr, ATOMIC_RELAXED);
+    if (files)
+        epoch_call(&files->epoch, fd_files_free);
     mem_free(t->open);
     mem_free(t->cloexec);
-    *t = (struct fd_table){};
+    t->open = nullptr;
+    t->cloexec = nullptr;
+    t->capacity = 0;
+    t->next_free = 0;
 }
 
 static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
-    struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0);
+    struct fd_files* files = mem_alloc(sizeof(*files) + capacity * sizeof(files->slots[0]), 0);
     uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0);
     uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0);
     if (!files || !open || !cloexec) {
@@ -40,19 +61,23 @@ static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
         return ENOMEM;
     }
 
-    if (t->capacity) {
+    files->capacity = capacity;
-        memcpy(files, t->files, t->capacity * sizeof(*files));
+    struct fd_files* old = atomic_load(&t->files, ATOMIC_RELAXED);
+    if (old) {
+        for (size_t i = 0; i < t->capacity; i++)
+            atomic_store(&files->slots[i], atomic_load(&old->slots[i], ATOMIC_RELAXED), ATOMIC_RELAXED);
         memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open));
         memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec));
     }
-    mem_free(t->files);
     mem_free(t->open);
     mem_free(t->cloexec);
 
-    t->files = files;
     t->open = open;
     t->cloexec = cloexec;
     t->capacity = capacity;
+    epoch_publish(&t->files, files);
+    if (old)
+        epoch_call(&old->epoch, fd_files_free);
     return 0;
 }
 
@@ -76,19 +101,23 @@ static size_t fd_find_free(struct fd_table* t, size_t start) {
     return MAX(start, t->capacity);
 }
 
-static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+static void fd_set_cloexec(struct fd_table* t, size_t fd, bool cloexec) {
     const uint64_t bit = 1ull << (fd % 64);
-    t->files[fd] = file;
-    t->open[fd / 64] |= bit;
     if (cloexec)
         t->cloexec[fd / 64] |= bit;
     else
         t->cloexec[fd / 64] &= ~bit;
 }
 
+static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+    t->open[fd / 64] |= 1ull << (fd % 64);
+    fd_set_cloexec(t, fd, cloexec);
+    epoch_publish(fd_slot(t, fd), file);
+}
+
 static void fd_clear(struct fd_table* t, size_t fd) {
     const uint64_t bit = 1ull << (fd % 64);
-    t->files[fd] = nullptr;
+    atomic_store(fd_slot(t, fd), nullptr, ATOMIC_RELAXED);
     t->open[fd / 64] &= ~bit;
     t->cloexec[fd / 64] &= ~bit;
     if (fd < t->next_free)
@@ -98,7 +127,7 @@ static void fd_clear(struct fd_table* t, size_t fd) {
 static struct vfs_file* fd_lookup(struct process* proc, int fd) {
     if (fd < 0 || (size_t)fd >= proc->files.capacity)
         return nullptr;
-    return proc->files.files[fd];
+    return atomic_load(fd_slot(&proc->files, fd), ATOMIC_RELAXED);
 }
 
 static void process_init_common(struct process* proc) {
@@ -125,7 +154,7 @@ static void close_files(struct process* proc) {
     mutex_lock(&proc->files_lock);
     for (size_t w = 0; w < fd_words(t->capacity); w++) {
         for (uint64_t bits = t->open[w]; bits; bits &= bits - 1)
-            vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]);
+            vfs_file_unref(atomic_load(fd_slot(t, w * 64 + __builtin_ctzll(bits)), ATOMIC_RELAXED));
     }
     fd_table_destroy(t);
     mutex_unlock(&proc->files_lock);
@@ -313,7 +342,7 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file,
         mutex_unlock(&proc->files_lock);
         return status;
     }
-    struct vfs_file* displaced = t->files[fd];
+    struct vfs_file* displaced = fd_lookup(proc, fd);
     fd_install(t, fd, file, cloexec);
     vfs_file_ref(file);
     mutex_unlock(&proc->files_lock);
@@ -339,7 +368,7 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
     mutex_lock(&proc->files_lock);
     struct vfs_file* file = fd_lookup(proc, fd);
     if (file) {
-        fd_install(&proc->files, fd, file, cloexec);
+        fd_set_cloexec(&proc->files, fd, cloexec);
         status = 0;
     }
     mutex_unlock(&proc->files_lock);
@@ -347,11 +376,18 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
 }
 
 struct vfs_file* process_fd_get(struct process* proc, int fd) {
-    mutex_lock(&proc->files_lock);
+    if (fd < 0)
-    struct vfs_file* file = fd_lookup(proc, fd);
+        return nullptr;
-    if (file)
+
-        vfs_file_ref(file);
+    struct vfs_file* file = nullptr;
-    mutex_unlock(&proc->files_lock);
+    epoch_enter();
+    struct fd_files* files = epoch_deref(&proc->files.files);
+    if (files && (size_t)fd < files->capacity) {
+        file = epoch_deref(&files->slots[fd]);
+        if (file && !refcount_inc_not_zero(&file->rc))
+            file = nullptr;
+    }
+    epoch_exit();
     return file;
 }
 

lib/radix.c

@@ -1,5 +1,7 @@
 #include <zinnia/alloc.h>
+#include <zinnia/assert.h>
 #include <zinnia/radix.h>
+#include <zinnia/utils.h>
 
 static bool covers(const struct radix_node* node, uint64_t index) {
     return node->shift + RADIX_BITS >= 64 || (index >> (node->shift + RADIX_BITS)) == 0;
@@ -9,6 +11,10 @@ static size_t slot_of(const struct radix_node* node, uint64_t index) {
     return (index >> node->shift) & (RADIX_SLOTS - 1);
 }
 
+static uint64_t bit(size_t slot) {
+    return (uint64_t)1 << slot;
+}
+
 static struct radix_node* node_alloc(uint8_t shift, struct radix_node* parent, uint8_t offset) {
     struct radix_node* node = mem_alloc(sizeof(struct radix_node), 0);
     if (!node)
@@ -19,36 +25,46 @@ static struct radix_node* node_alloc(uint8_t shift, struct radix_node* parent, u
     return node;
 }
 
+static void node_free(struct epoch_head* head) {
+    mem_free(containerof(head, struct radix_node, epoch));
+}
+
+static void node_retire(struct radix_node* node) {
+    epoch_call(&node->epoch, node_free);
+}
+
 static void free_subtree(struct radix_node* node) {
     if (node->shift > 0) {
         for (size_t i = 0; i < RADIX_SLOTS; i++) {
-            if (node->slots[i])
+            struct radix_node* child = atomic_load(&node->slots[i], ATOMIC_RELAXED);
-                free_subtree(node->slots[i]);
+            if (child)
+                free_subtree(child);
         }
     }
     mem_free(node);
 }
 
 void radix_init(struct radix_tree* tree) {
-    tree->root = nullptr;
+    atomic_init(&tree->root, nullptr);
 }
 
 void radix_destroy(struct radix_tree* tree) {
-    if (tree->root)
+    struct radix_node* root = atomic_load(&tree->root, ATOMIC_RELAXED);
-        free_subtree(tree->root);
+    if (root)
-    tree->root = nullptr;
+        free_subtree(root);
+    atomic_store(&tree->root, nullptr, ATOMIC_RELAXED);
 }
 
 bool radix_empty(struct radix_tree* tree) {
-    return tree->root == nullptr;
+    return atomic_load(&tree->root, ATOMIC_RELAXED) == nullptr;
 }
 
 static struct radix_node* leaf_of(struct radix_tree* tree, uint64_t index) {
-    struct radix_node* node = tree->root;
+    struct radix_node* node = epoch_deref(&tree->root);
     if (!node || !covers(node, index))
         return nullptr;
     while (node->shift > 0) {
-        node = node->slots[slot_of(node, index)];
+        node = epoch_deref(&node->slots[slot_of(node, index)]);
         if (!node)
             return nullptr;
     }
@@ -57,37 +73,48 @@ static struct radix_node* leaf_of(struct radix_tree* tree, uint64_t index) {
 
 void* radix_lookup(struct radix_tree* tree, uint64_t index) {
     struct radix_node* leaf = leaf_of(tree, index);
-    return leaf ? leaf->slots[slot_of(leaf, index)] : nullptr;
+    return leaf ? epoch_deref(&leaf->slots[slot_of(leaf, index)]) : nullptr;
 }
 
-void** radix_slot(struct radix_tree* tree, uint64_t index) {
+void* radix_replace(struct radix_tree* tree, uint64_t index, void* value) {
+    assert(value, "Tried to replace a radix slot with nullptr");
     struct radix_node* leaf = leaf_of(tree, index);
-    return leaf ? &leaf->slots[slot_of(leaf, index)] : nullptr;
+    if (!leaf)
+        return nullptr;
+    _Atomic(void*)* slot = &leaf->slots[slot_of(leaf, index)];
+    void* old = atomic_load(slot, ATOMIC_RELAXED);
+    if (old)
+        epoch_publish(slot, value);
+    return old;
 }
 
 static errno_t grow(struct radix_tree* tree, uint64_t index) {
-    if (!tree->root) {
+    struct radix_node* root = atomic_load(&tree->root, ATOMIC_RELAXED);
+    if (!root) {
         uint8_t shift = 0;
         while (shift + RADIX_BITS < 64 && (index >> (shift + RADIX_BITS)) != 0)
             shift += RADIX_BITS;
-        tree->root = node_alloc(shift, nullptr, 0);
+        root = node_alloc(shift, nullptr, 0);
-        return tree->root ? 0 : ENOMEM;
+        if (!root)
+            return ENOMEM;
+        epoch_publish(&tree->root, root);
+        return 0;
     }
 
-    while (!covers(tree->root, index)) {
+    while (!covers(root, index)) {
-        struct radix_node* old = tree->root;
+        struct radix_node* top = node_alloc(root->shift + RADIX_BITS, nullptr, 0);
-        struct radix_node* top = node_alloc(old->shift + RADIX_BITS, nullptr, 0);
         if (!top)
             return ENOMEM;
-        top->slots[0] = old;
+        atomic_store(&top->slots[0], root, ATOMIC_RELAXED);
         top->count = 1;
         for (size_t t = 0; t < RADIX_NUM_TAGS; t++) {
-            if (old->tags[t])
+            if (atomic_load(&root->tags[t], ATOMIC_RELAXED))
-                top->tags[t] = 1;
+                atomic_store(&top->tags[t], 1, ATOMIC_RELAXED);
         }
-        old->parent = top;
+        root->parent = top;
-        old->offset = 0;
+        root->offset = 0;
-        tree->root = top;
+        epoch_publish(&tree->root, top);
+        root = top;
     }
     return 0;
 }
@@ -97,32 +124,33 @@ errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value) {
     if (status)
         return status;
 
-    struct radix_node* node = tree->root;
+    struct radix_node* node = atomic_load(&tree->root, ATOMIC_RELAXED);
     while (node->shift > 0) {
         const size_t slot = slot_of(node, index);
-        struct radix_node* child = node->slots[slot];
+        struct radix_node* child = atomic_load(&node->slots[slot], ATOMIC_RELAXED);
         if (!child) {
             child = node_alloc(node->shift - RADIX_BITS, node, slot);
             if (!child)
                 return ENOMEM;
-            node->slots[slot] = child;
+            epoch_publish(&node->slots[slot], child);
             node->count++;
         }
         node = child;
     }
 
     const size_t slot = slot_of(node, index);
-    if (node->slots[slot])
+    if (atomic_load(&node->slots[slot], ATOMIC_RELAXED))
         return EEXIST;
-    node->slots[slot] = value;
+    epoch_publish(&node->slots[slot], value);
     node->count++;
     return 0;
 }
 
 static void clear_tag_upwards(struct radix_node* node, size_t slot, enum radix_tag tag) {
     while (node) {
-        node->tags[tag] &= ~((uint64_t)1 << slot);
+        const uint64_t tags = atomic_load(&node->tags[tag], ATOMIC_RELAXED) & ~bit(slot);
-        if (node->tags[tag])
+        atomic_store(&node->tags[tag], tags, ATOMIC_RELAXED);
+        if (tags)
             return;
         slot = node->offset;
         node = node->parent;
@@ -135,28 +163,28 @@ void* radix_remove(struct radix_tree* tree, uint64_t index) {
         return nullptr;
 
     size_t slot = slot_of(node, index);
-    void* value = node->slots[slot];
+    void* value = atomic_load(&node->slots[slot], ATOMIC_RELAXED);
     if (!value)
         return nullptr;
 
     for (size_t t = 0; t < RADIX_NUM_TAGS; t++) {
-        if (node->tags[t] & ((uint64_t)1 << slot))
+        if (atomic_load(&node->tags[t], ATOMIC_RELAXED) & bit(slot))
             clear_tag_upwards(node, slot, t);
     }
 
-    node->slots[slot] = nullptr;
+    atomic_store(&node->slots[slot], nullptr, ATOMIC_RELAXED);
     node->count--;
     while (node->count == 0) {
         struct radix_node* parent = node->parent;
         if (!parent) {
-            tree->root = nullptr;
+            atomic_store(&tree->root, nullptr, ATOMIC_RELAXED);
-            mem_free(node);
+            node_retire(node);
             break;
         }
         slot = node->offset;
-        parent->slots[slot] = nullptr;
+        atomic_store(&parent->slots[slot], nullptr, ATOMIC_RELAXED);
         parent->count--;
-        mem_free(node);
+        node_retire(node);
         node = parent;
     }
     return value;
@@ -164,14 +192,16 @@ void* radix_remove(struct radix_tree* tree, uint64_t index) {
 
 void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
     struct radix_node* node = leaf_of(tree, index);
-    if (!node || !node->slots[slot_of(node, index)])
+    if (!node)
         return;
     size_t slot = slot_of(node, index);
+    if (!atomic_load(&node->slots[slot], ATOMIC_RELAXED))
+        return;
     while (node) {
-        const uint64_t bit = (uint64_t)1 << slot;
+        const uint64_t tags = atomic_load(&node->tags[tag], ATOMIC_RELAXED);
-        if (node->tags[tag] & bit)
+        if (tags & bit(slot))
             return;
-        node->tags[tag] |= bit;
+        atomic_store(&node->tags[tag], tags | bit(slot), ATOMIC_RELAXED);
         slot = node->offset;
         node = node->parent;
     }
@@ -182,17 +212,18 @@ void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag
     if (!node)
         return;
     const size_t slot = slot_of(node, index);
-    if (node->tags[tag] & ((uint64_t)1 << slot))
+    if (atomic_load(&node->tags[tag], ATOMIC_RELAXED) & bit(slot))
         clear_tag_upwards(node, slot, tag);
 }
 
 bool radix_tag_get(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
     struct radix_node* node = leaf_of(tree, index);
-    return node && (node->tags[tag] & ((uint64_t)1 << slot_of(node, index)));
+    return node && (atomic_load(&node->tags[tag], ATOMIC_RELAXED) & bit(slot_of(node, index)));
 }
 
 bool radix_tagged(struct radix_tree* tree, enum radix_tag tag) {
-    return tree->root && tree->root->tags[tag];
+    struct radix_node* root = epoch_deref(&tree->root);
+    return root && atomic_load(&root->tags[tag], ATOMIC_RELAXED);
 }
 
 struct gang {
@@ -212,26 +243,28 @@ static void gang_walk(struct radix_node* node, uint64_t base, uint64_t first, st
         start = delta;
     }
 
+    const uint64_t tags = g->tag >= 0 ? atomic_load(&node->tags[g->tag], ATOMIC_RELAXED) : UINT64_MAX;
     for (size_t i = start; i < RADIX_SLOTS && g->found < g->max; i++) {
-        if (!node->slots[i])
+        if (!(tags & bit(i)))
             continue;
-        if (g->tag >= 0 && !(node->tags[g->tag] & ((uint64_t)1 << i)))
+        void* entry = epoch_deref(&node->slots[i]);
+        if (!entry)
             continue;
         const uint64_t index = base + ((uint64_t)i << node->shift);
         if (node->shift == 0) {
             if (g->out)
-                g->out[g->found] = node->slots[i];
+                g->out[g->found] = entry;
             if (g->out_index)
                 g->out_index[g->found] = index;
             g->found++;
         } else {
-            gang_walk(node->slots[i], index, first > index ? first : index, g);
+            gang_walk(entry, index, first > index ? first : index, g);
         }
     }
 }
 
 static size_t gang_lookup(struct radix_tree* tree, uint64_t first, int tag, size_t max, void** out, uint64_t* idx) {
-    struct radix_node* root = tree->root;
+    struct radix_node* root = epoch_deref(&tree->root);
     if (!root || max == 0 || !covers(root, first))
         return 0;
     struct gang g = {.max = max, .out = out, .out_index = idx, .tag = tag};

mem/page.c

@@ -29,6 +29,15 @@ void page_hold(struct page* page) {
     atomic_fetch_add(&page->holds, 1, ATOMIC_RELAXED);
 }
 
+bool page_try_hold(struct page* page) {
+    size_t holds = atomic_load(&page->holds, ATOMIC_RELAXED);
+    while (holds) {
+        if (atomic_compare_exchange_weak(&page->holds, &holds, holds + 1, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+            return true;
+    }
+    return false;
+}
+
 void page_unhold(struct page* page) {
     if (atomic_fetch_sub(&page->holds, 1, ATOMIC_RELEASE) != 1)
         return;

mem/vma.c

@@ -445,7 +445,7 @@ static errno_t fault_anon(
         errno_t status = anon_new(anon->page, &copy);
         if (status)
             return status;
-        *radix_slot(&m->amap->anons, slot) = copy;
+        radix_replace(&m->amap->anons, slot, copy);
         anon_unref(anon);
         anon = copy;
         writable = true;

mem/vmo.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: ISC
 
 #include <zinnia/alloc.h>
+#include <zinnia/epoch.h>
 #include <zinnia/errno.h>
 #include <zinnia/print.h>
 #include <zinnia/utils.h>
@@ -143,10 +144,34 @@ static errno_t fill(struct vmo* vmo, uint64_t index, enum vmo_access access, str
     return 0;
 }
 
+static struct page* lookup_fast(struct vmo* vmo, uint64_t index) {
+    epoch_enter();
+    struct page* page = radix_lookup(&vmo->pages, index);
+    if (page && !page_try_hold(page))
+        page = nullptr;
+    const bool valid =
+        page && radix_lookup(&vmo->pages, index) == page && !(atomic_load(&page->flags, ATOMIC_ACQUIRE) & PAGE_BUSY);
+    epoch_exit();
+
+    if (page && !valid) {
+        page_unhold(page);
+        return nullptr;
+    }
+    return page;
+}
+
 errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) {
     if (vmo->pager->lookup)
         return ENOTSUP;
 
+    if (access == VMO_READ) {
+        struct page* page = lookup_fast(vmo, index);
+        if (page) {
+            *out = page;
+            return 0;
+        }
+    }
+
     mutex_lock(&vmo->lock);
     while (true) {
         struct page* page = radix_lookup(&vmo->pages, index);

sched/core.c

@@ -4,7 +4,9 @@
 #include <zinnia/assert.h>
 #include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/park.h>
+#include <zinnia/percpu.h>
 #include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
@@ -163,3 +165,16 @@ void sched_sleep_until(uint64_t deadline_ns) {
     const uint8_t key = 0;
     park_timeout(&key, nullptr, nullptr, nullptr, 0, &deadline_ns);
 }
+
+void sched_preempt_disable() {
+    arch_sched_preempt_disable();
+}
+
+void sched_preempt_enable() {
+    if (!arch_sched_preempt_enable())
+        return;
+    struct scheduler* sched = &percpu_get()->sched;
+    if ((sched->owe_preempt || sched->need_resched) && percpu_ref(irq_data)->depth == 0 && irq_get_state() &&
+        sched->current)
+        sched_reschedule(sched);
+}

syscalls/system.c

@@ -93,8 +93,12 @@ struct syscall_ret sys_setuname(const __user struct __utsname* ptr) {
 struct syscall_ret sys_syslog(enum syslog_kind kind, __user char* message, size_t len) {
     struct syscall_ret ret = {};
 
-    const size_t buf_len = MIN(len + 1, 256);
+    const size_t buf_len = MIN(len, 255) + 1;
     char* buf = mem_alloc(buf_len, ALLOC_NOZERO);
+    if (!buf) {
+        ret.error = ENOMEM;
+        return ret;
+    }
     if (!usercopy_read(buf, message, buf_len - 1)) {
         mem_free(buf);