lib: Add epoch-based SMR

include/zinnia/eplist.h

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/epoch.h>
+
+#define eplist_head(type) \
+    struct { \
+        _Atomic(type*) eph_first; \
+    }
+
+#define eplist_link(type) \
+    struct { \
+        _Atomic(type*) epl_next; \
+        _Atomic(type*)* epl_prev; \
+    }
+
+#define eplist_init(head) atomic_init(&(head)->eph_first, nullptr)
+
+#define eplist_first(head) epoch_deref(&(head)->eph_first)
+
+#define eplist_next(elm, field) epoch_deref(&(elm)->field.epl_next)
+
+#define eplist_empty(head) (atomic_load(&(head)->eph_first, ATOMIC_RELAXED) == nullptr)
+
+#define eplist_foreach(var, head, field) for ((var) = eplist_first(head); (var); (var) = eplist_next(var, field))
+
+#define eplist_insert_head(head, elm, field) \
+    do { \
+        typeof(elm) _epl_first = atomic_load(&(head)->eph_first, ATOMIC_RELAXED); \
+        atomic_store(&(elm)->field.epl_next, _epl_first, ATOMIC_RELAXED); \
+        (elm)->field.epl_prev = &(head)->eph_first; \
+        if (_epl_first) \
+            _epl_first->field.epl_prev = &(elm)->field.epl_next; \
+        epoch_publish(&(head)->eph_first, (elm)); \
+    } while (0)
+
+#define eplist_insert_after(listelm, elm, field) \
+    do { \
+        typeof(elm) _epl_next = atomic_load(&(listelm)->field.epl_next, ATOMIC_RELAXED); \
+        atomic_store(&(elm)->field.epl_next, _epl_next, ATOMIC_RELAXED); \
+        (elm)->field.epl_prev = &(listelm)->field.epl_next; \
+        if (_epl_next) \
+            _epl_next->field.epl_prev = &(elm)->field.epl_next; \
+        epoch_publish(&(listelm)->field.epl_next, (elm)); \
+    } while (0)
+
+#define eplist_insert_before(listelm, elm, field) \
+    do { \
+        atomic_store(&(elm)->field.epl_next, (listelm), ATOMIC_RELAXED); \
+        (elm)->field.epl_prev = (listelm)->field.epl_prev; \
+        epoch_publish((listelm)->field.epl_prev, (elm)); \
+        (listelm)->field.epl_prev = &(elm)->field.epl_next; \
+    } while (0)
+
+#define eplist_remove(elm, field) \
+    do { \
+        typeof(elm) _epl_next = atomic_load(&(elm)->field.epl_next, ATOMIC_RELAXED); \
+        if (_epl_next) \
+            _epl_next->field.epl_prev = (elm)->field.epl_prev; \
+        epoch_publish((elm)->field.epl_prev, _epl_next); \
+    } while (0)

include/zinnia/epoch.h

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/initgraph.h>
+#include <stdint.h>
+
+struct epoch_head {
+    struct epoch_head* next;
+    void (*func)(struct epoch_head* head);
+};
+
+// Enters a read-side critical section.
+// Disables preemption, so the caller must not sleep.
+// A reader may call this function even while already in an epoch section.
+void epoch_enter();
+
+// Leaves a read-side critical section.
+void epoch_exit();
+
+// Returns true if the current CPU is inside a critical section.
+bool epoch_active();
+
+// Runs `func` once all readers that could observe `head` have left.
+void epoch_call(struct epoch_head* head, void (*func)(struct epoch_head* head));
+
+// Advances the global epoch if all active readers have observed it.
+bool epoch_try_advance();
+
+// Returns the current global epoch.
+uint64_t epoch_current();
+
+// Waits until all readers active at the time of the call have left.
+void epoch_synchronize();
+
+// Waits until all previously queued callbacks have run.
+void epoch_barrier();
+
+// Loads a pointer protected by an epoch critical section.
+#define epoch_deref(ptr) atomic_load(ptr, ATOMIC_ACQUIRE)
+
+// Publishes a pointer to epoch readers.
+#define epoch_publish(ptr, val) atomic_store(ptr, val, ATOMIC_RELEASE)
+
+INIT_EXTERN(epoch_stage);

include/zinnia/lfhash.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr size_t LFHASH_SEGMENT_BUCKETS = 512;
+constexpr size_t LFHASH_SEGMENTS = 1024;
+constexpr size_t LFHASH_MIN_BUCKETS = 16;
+constexpr size_t LFHASH_LOAD_FACTOR = 2;
+
+struct lfhash_node {
+    _Atomic(uintptr_t) next;
+    uint64_t key;
+};
+
+struct lfhash_bucket;
+
+struct lfhash {
+    _Atomic(struct lfhash_bucket*)* segments;
+    atomic_size_t size;
+    atomic_size_t count;
+};
+
+typedef bool (*lfhash_eq_fn)(const struct lfhash_node* node, const void* key);
+
+errno_t lfhash_init(struct lfhash* table);
+void lfhash_destroy(struct lfhash* table);
+size_t lfhash_count(struct lfhash* table);
+
+errno_t lfhash_insert(struct lfhash* table, uint64_t hash, struct lfhash_node* node, lfhash_eq_fn eq, const void* key);
+struct lfhash_node* lfhash_lookup(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key);
+struct lfhash_node* lfhash_remove(struct lfhash* table, uint64_t hash, lfhash_eq_fn eq, const void* key);
+
+struct lfhash_node* lfhash_first(struct lfhash* table);
+struct lfhash_node* lfhash_next(struct lfhash_node* node);
+
+#define lfhash_foreach(table, var) for ((var) = lfhash_first(table); (var); (var) = lfhash_next(var))

include/zinnia/lfqueue.h

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+
+struct lfqueue_node;
+
+struct lfqueue {
+    _Atomic(struct lfqueue_node*) head;
+    _Atomic(struct lfqueue_node*) tail;
+};
+
+errno_t lfqueue_init(struct lfqueue* queue);
+void lfqueue_destroy(struct lfqueue* queue);
+bool lfqueue_empty(struct lfqueue* queue);
+errno_t lfqueue_push(struct lfqueue* queue, void* value);
+bool lfqueue_pop(struct lfqueue* queue, void** out);

include/zinnia/lfstack.h

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+
+struct lfstack_node {
+    _Atomic(struct lfstack_node*) next;
+};
+
+struct lfstack {
+    _Atomic(struct lfstack_node*) top;
+};
+
+void lfstack_init(struct lfstack* stack);
+bool lfstack_empty(struct lfstack* stack);
+void lfstack_push(struct lfstack* stack, struct lfstack_node* node);
+struct lfstack_node* lfstack_pop(struct lfstack* stack);
+struct lfstack_node* lfstack_pop_all(struct lfstack* stack);

include/zinnia/process.h

@@ -25,8 +25,10 @@ struct process_rlimit {
     rlim_t max;
 };
 
+struct fd_files;
+
 struct fd_table {
-    struct vfs_file** files;
+    _Atomic(struct fd_files*) files;
     uint64_t* open;
     uint64_t* cloexec;
     size_t capacity;

include/zinnia/radix.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zinnia/epoch.h>
 #include <zinnia/errno.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -20,12 +21,13 @@ struct radix_node {
     uint8_t count;
     uint8_t offset;
     struct radix_node* parent;
-    uint64_t tags[RADIX_NUM_TAGS];
-    void* slots[RADIX_SLOTS];
+    _Atomic(uint64_t) tags[RADIX_NUM_TAGS];
+    _Atomic(void*) slots[RADIX_SLOTS];
+    struct epoch_head epoch;
 };
 
 struct radix_tree {
-    struct radix_node* root;
+    _Atomic(struct radix_node*) root;
 };
 
 void radix_init(struct radix_tree* tree);
@@ -35,7 +37,7 @@ bool radix_empty(struct radix_tree* tree);
 void* radix_lookup(struct radix_tree* tree, uint64_t index);
 errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value);
 void* radix_remove(struct radix_tree* tree, uint64_t index);
-void** radix_slot(struct radix_tree* tree, uint64_t index);
+void* radix_replace(struct radix_tree* tree, uint64_t index, void* value);
 
 void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag);
 void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag);

include/zinnia/rcu.h

@@ -1,6 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-void rcu_read_lock();
-void rcu_read_unlock();

include/zinnia/sched.h

@@ -135,6 +135,9 @@ void sched_notify(struct task* task);
 
 void sched_sleep_until(uint64_t deadline_ns);
 
+void sched_preempt_disable();
+void sched_preempt_enable();
+
 [[noreturn]]
 void sched_exit();
 

include/zinnia/vfs.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zinnia/epoch.h>
 #include <zinnia/errno.h>
 #include <zinnia/ident.h>
 #include <zinnia/initgraph.h>
@@ -356,6 +357,8 @@ struct vfs_file {
 
     // Driver data.
     void* priv;
+
+    struct epoch_head epoch;
 };
 
 // A file system driver.

include/zinnia/virt.h

@@ -47,6 +47,7 @@ struct page* vm_page_from_addr(phys_t addr);
 
 errno_t page_alloc(enum alloc_flags flags, struct page** out);
 void page_hold(struct page* page);
+bool page_try_hold(struct page* page);
 void page_unhold(struct page* page);
 bool page_try_busy(struct page* page);
 void page_unbusy(struct page* page);