lib: Add epoch-based SMR

lib/process.c

@@ -2,6 +2,7 @@
 
 #include <zinnia/alloc.h>
 #include <zinnia/atomic.h>
+#include <zinnia/epoch.h>
 #include <zinnia/exec.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
@@ -18,19 +19,39 @@ static struct process kernel_process;
 static atomic_bool kernel_process_ready;
 static atomic_int next_pid = 1;
 
+struct fd_files {
+    size_t capacity;
+    struct epoch_head epoch;
+    _Atomic(struct vfs_file*) slots[];
+};
+
 static size_t fd_words(size_t capacity) {
     return capacity / 64;
 }
 
+static void fd_files_free(struct epoch_head* head) {
+    mem_free(containerof(head, struct fd_files, epoch));
+}
+
+static _Atomic(struct vfs_file*)* fd_slot(struct fd_table* t, size_t fd) {
+    return &atomic_load(&t->files, ATOMIC_RELAXED)->slots[fd];
+}
+
 static void fd_table_destroy(struct fd_table* t) {
-    mem_free(t->files);
+    struct fd_files* files = atomic_load(&t->files, ATOMIC_RELAXED);
+    atomic_store(&t->files, nullptr, ATOMIC_RELAXED);
+    if (files)
+        epoch_call(&files->epoch, fd_files_free);
     mem_free(t->open);
     mem_free(t->cloexec);
-    *t = (struct fd_table){};
+    t->open = nullptr;
+    t->cloexec = nullptr;
+    t->capacity = 0;
+    t->next_free = 0;
 }
 
 static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
-    struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0);
+    struct fd_files* files = mem_alloc(sizeof(*files) + capacity * sizeof(files->slots[0]), 0);
     uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0);
     uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0);
     if (!files || !open || !cloexec) {
@@ -40,19 +61,23 @@ static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
         return ENOMEM;
     }
 
-    if (t->capacity) {
-        memcpy(files, t->files, t->capacity * sizeof(*files));
+    files->capacity = capacity;
+    struct fd_files* old = atomic_load(&t->files, ATOMIC_RELAXED);
+    if (old) {
+        for (size_t i = 0; i < t->capacity; i++)
+            atomic_store(&files->slots[i], atomic_load(&old->slots[i], ATOMIC_RELAXED), ATOMIC_RELAXED);
         memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open));
         memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec));
     }
-    mem_free(t->files);
     mem_free(t->open);
     mem_free(t->cloexec);
 
-    t->files = files;
     t->open = open;
     t->cloexec = cloexec;
     t->capacity = capacity;
+    epoch_publish(&t->files, files);
+    if (old)
+        epoch_call(&old->epoch, fd_files_free);
     return 0;
 }
 
@@ -76,19 +101,23 @@ static size_t fd_find_free(struct fd_table* t, size_t start) {
     return MAX(start, t->capacity);
 }
 
-static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+static void fd_set_cloexec(struct fd_table* t, size_t fd, bool cloexec) {
     const uint64_t bit = 1ull << (fd % 64);
-    t->files[fd] = file;
-    t->open[fd / 64] |= bit;
     if (cloexec)
         t->cloexec[fd / 64] |= bit;
     else
         t->cloexec[fd / 64] &= ~bit;
 }
 
+static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+    t->open[fd / 64] |= 1ull << (fd % 64);
+    fd_set_cloexec(t, fd, cloexec);
+    epoch_publish(fd_slot(t, fd), file);
+}
+
 static void fd_clear(struct fd_table* t, size_t fd) {
     const uint64_t bit = 1ull << (fd % 64);
-    t->files[fd] = nullptr;
+    atomic_store(fd_slot(t, fd), nullptr, ATOMIC_RELAXED);
     t->open[fd / 64] &= ~bit;
     t->cloexec[fd / 64] &= ~bit;
     if (fd < t->next_free)
@@ -98,7 +127,7 @@ static void fd_clear(struct fd_table* t, size_t fd) {
 static struct vfs_file* fd_lookup(struct process* proc, int fd) {
     if (fd < 0 || (size_t)fd >= proc->files.capacity)
         return nullptr;
-    return proc->files.files[fd];
+    return atomic_load(fd_slot(&proc->files, fd), ATOMIC_RELAXED);
 }
 
 static void process_init_common(struct process* proc) {
@@ -125,7 +154,7 @@ static void close_files(struct process* proc) {
     mutex_lock(&proc->files_lock);
     for (size_t w = 0; w < fd_words(t->capacity); w++) {
         for (uint64_t bits = t->open[w]; bits; bits &= bits - 1)
-            vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]);
+            vfs_file_unref(atomic_load(fd_slot(t, w * 64 + __builtin_ctzll(bits)), ATOMIC_RELAXED));
     }
     fd_table_destroy(t);
     mutex_unlock(&proc->files_lock);
@@ -313,7 +342,7 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file,
         mutex_unlock(&proc->files_lock);
         return status;
     }
-    struct vfs_file* displaced = t->files[fd];
+    struct vfs_file* displaced = fd_lookup(proc, fd);
     fd_install(t, fd, file, cloexec);
     vfs_file_ref(file);
     mutex_unlock(&proc->files_lock);
@@ -339,7 +368,7 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
     mutex_lock(&proc->files_lock);
     struct vfs_file* file = fd_lookup(proc, fd);
     if (file) {
-        fd_install(&proc->files, fd, file, cloexec);
+        fd_set_cloexec(&proc->files, fd, cloexec);
         status = 0;
     }
     mutex_unlock(&proc->files_lock);
@@ -347,11 +376,18 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
 }
 
 struct vfs_file* process_fd_get(struct process* proc, int fd) {
-    mutex_lock(&proc->files_lock);
-    struct vfs_file* file = fd_lookup(proc, fd);
-    if (file)
-        vfs_file_ref(file);
-    mutex_unlock(&proc->files_lock);
+    if (fd < 0)
+        return nullptr;
+
+    struct vfs_file* file = nullptr;
+    epoch_enter();
+    struct fd_files* files = epoch_deref(&proc->files.files);
+    if (files && (size_t)fd < files->capacity) {
+        file = epoch_deref(&files->slots[fd]);
+        if (file && !refcount_inc_not_zero(&file->rc))
+            file = nullptr;
+    }
+    epoch_exit();
     return file;
 }