process: Switch to array + bitmap for fd allocations

include/zinnia/process.h

@@ -4,7 +4,6 @@
 
 #include <zinnia/atomic.h>
 #include <zinnia/errno.h>
-#include <zinnia/hashmap.h>
 #include <zinnia/ident.h>
 #include <zinnia/mutex.h>
 #include <zinnia/refcount.h>
@@ -26,9 +25,12 @@ struct process_rlimit {
     rlim_t max;
 };
 
-struct fd_entry {
-    struct vfs_file* file;
-    bool cloexec;
+struct fd_table {
+    struct vfs_file** files;
+    uint64_t* open;
+    uint64_t* cloexec;
+    size_t capacity;
+    size_t next_free;
 };
 
 struct process {
@@ -50,7 +52,7 @@ struct process {
     struct process_rlimit limit_nofile;
 
     mutex_t files_lock;
-    hashmap(int, struct fd_entry) files;
+    struct fd_table files;
 };
 
 struct process* process_kernel();

lib/process.c

@@ -3,7 +3,6 @@
 #include <zinnia/alloc.h>
 #include <zinnia/atomic.h>
 #include <zinnia/exec.h>
-#include <zinnia/hashmap.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <zinnia/process.h>
@@ -11,18 +10,103 @@
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
 #include <zinnia/vmo.h>
+#include <string.h>
+
+constexpr size_t PROCESS_INITIAL_FDS = 64;
 
 static struct process kernel_process;
 static atomic_bool kernel_process_ready;
 static atomic_int next_pid = 1;
 
+static size_t fd_words(size_t capacity) {
+    return capacity / 64;
+}
+
+static void fd_table_destroy(struct fd_table* t) {
+    mem_free(t->files);
+    mem_free(t->open);
+    mem_free(t->cloexec);
+    *t = (struct fd_table){};
+}
+
+static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
+    struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0);
+    uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0);
+    uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0);
+    if (!files || !open || !cloexec) {
+        mem_free(files);
+        mem_free(open);
+        mem_free(cloexec);
+        return ENOMEM;
+    }
+
+    if (t->capacity) {
+        memcpy(files, t->files, t->capacity * sizeof(*files));
+        memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open));
+        memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec));
+    }
+    mem_free(t->files);
+    mem_free(t->open);
+    mem_free(t->cloexec);
+
+    t->files = files;
+    t->open = open;
+    t->cloexec = cloexec;
+    t->capacity = capacity;
+    return 0;
+}
+
+static errno_t fd_table_grow(struct fd_table* t, size_t fd) {
+    if (fd < t->capacity)
+        return 0;
+    size_t capacity = MAX(PROCESS_INITIAL_FDS, t->capacity);
+    while (capacity <= fd)
+        capacity *= 2;
+    return fd_table_resize(t, MIN(capacity, PROCESS_MAX_FDS));
+}
+
+static size_t fd_find_free(struct fd_table* t, size_t start) {
+    for (size_t w = start / 64; w < fd_words(t->capacity); w++) {
+        uint64_t used = t->open[w];
+        if (w == start / 64)
+            used |= (1ull << (start % 64)) - 1;
+        if (~used)
+            return w * 64 + __builtin_ctzll(~used);
+    }
+    return MAX(start, t->capacity);
+}
+
+static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+    const uint64_t bit = 1ull << (fd % 64);
+    t->files[fd] = file;
+    t->open[fd / 64] |= bit;
+    if (cloexec)
+        t->cloexec[fd / 64] |= bit;
+    else
+        t->cloexec[fd / 64] &= ~bit;
+}
+
+static void fd_clear(struct fd_table* t, size_t fd) {
+    const uint64_t bit = 1ull << (fd % 64);
+    t->files[fd] = nullptr;
+    t->open[fd / 64] &= ~bit;
+    t->cloexec[fd / 64] &= ~bit;
+    if (fd < t->next_free)
+        t->next_free = fd;
+}
+
+static struct vfs_file* fd_lookup(struct process* proc, int fd) {
+    if (fd < 0 || (size_t)fd >= proc->files.capacity)
+        return nullptr;
+    return proc->files.files[fd];
+}
+
 static void process_init_common(struct process* proc) {
     refcount_init(&proc->rc);
     spin_init(&proc->tasks_lock);
     tailq_init(&proc->tasks);
     mutex_init(&proc->lock);
     mutex_init(&proc->files_lock);
-    hashmap_init(&proc->files);
     vec_init(&proc->identity.groups);
 }
 
@@ -37,17 +121,13 @@ struct process* process_kernel() {
 }
 
 static void close_files(struct process* proc) {
+    struct fd_table* t = &proc->files;
     mutex_lock(&proc->files_lock);
-    size_t i = 0;
-    while (i < proc->files.capacity) {
-        struct fd_entry entry = proc->files.values[i];
-        if (proc->files.hashes[i] == 0) {
-            i++;
-            continue;
-        }
-        hashmap_remove(&proc->files, proc->files.keys[i], hashmap_hash_default, hashmap_eq_default);
-        vfs_file_unref(entry.file);
+    for (size_t w = 0; w < fd_words(t->capacity); w++) {
+        for (uint64_t bits = t->open[w]; bits; bits &= bits - 1)
+            vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]);
     }
+    fd_table_destroy(t);
     mutex_unlock(&proc->files_lock);
 }
 
@@ -64,7 +144,6 @@ void process_unref(struct process* proc) {
     if (proc->space)
         vma_unref(proc->space);
     vec_destroy(&proc->identity.groups);
-    hashmap_destroy(&proc->files);
     mem_free(proc);
 }
 
@@ -192,10 +271,6 @@ errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloex
     return process_fd_alloc_min(proc, file, 0, cloexec, out_fd);
 }
 
-static struct fd_entry* fd_lookup(struct process* proc, int fd) {
-    return hashmap_get(&proc->files, fd, hashmap_hash_default, hashmap_eq_default);
-}
-
 size_t process_fd_limit(struct process* proc) {
     return MIN((size_t)PROCESS_MAX_FDS, (size_t)atomic_load(&proc->limit_nofile.cur, ATOMIC_RELAXED));
 }
@@ -204,22 +279,22 @@ errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int mi
     if (min_fd < 0 || (size_t)min_fd >= process_fd_limit(proc))
         return EINVAL;
 
+    struct fd_table* t = &proc->files;
     mutex_lock(&proc->files_lock);
-    int fd = min_fd;
-    while ((size_t)fd < process_fd_limit(proc) && fd_lookup(proc, fd))
-        fd++;
-
-    if ((size_t)fd >= process_fd_limit(proc)) {
+    const size_t fd = fd_find_free(t, MAX((size_t)min_fd, t->next_free));
+    if (fd >= process_fd_limit(proc)) {
         mutex_unlock(&proc->files_lock);
         return EMFILE;
     }
 
-    const struct fd_entry entry = {.file = file, .cloexec = cloexec};
-    errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default);
+    errno_t status = fd_table_grow(t, fd);
     if (status) {
         mutex_unlock(&proc->files_lock);
         return status;
     }
+    fd_install(t, fd, file, cloexec);
+    if ((size_t)min_fd <= t->next_free)
+        t->next_free = fd + 1;
     vfs_file_ref(file);
     mutex_unlock(&proc->files_lock);
 
@@ -231,20 +306,15 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file,
     if (fd < 0 || (size_t)fd >= process_fd_limit(proc))
         return EBADF;
 
-    struct vfs_file* displaced = nullptr;
-    const struct fd_entry entry = {.file = file, .cloexec = cloexec};
+    struct fd_table* t = &proc->files;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* slot = fd_lookup(proc, fd);
-    if (slot) {
-        displaced = slot->file;
-        *slot = entry;
-    } else {
-        errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default);
-        if (status) {
-            mutex_unlock(&proc->files_lock);
-            return status;
-        }
+    errno_t status = fd_table_grow(t, fd);
+    if (status) {
+        mutex_unlock(&proc->files_lock);
+        return status;
     }
+    struct vfs_file* displaced = t->files[fd];
+    fd_install(t, fd, file, cloexec);
     vfs_file_ref(file);
     mutex_unlock(&proc->files_lock);
 
@@ -256,9 +326,8 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file,
 errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) {
     errno_t status = EBADF;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        *out_cloexec = entry->cloexec;
+    if (fd_lookup(proc, fd)) {
+        *out_cloexec = (proc->files.cloexec[fd / 64] >> (fd % 64)) & 1;
         status = 0;
     }
     mutex_unlock(&proc->files_lock);
@@ -268,9 +337,9 @@ errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec)
 errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
     errno_t status = EBADF;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        entry->cloexec = cloexec;
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file) {
+        fd_install(&proc->files, fd, file, cloexec);
         status = 0;
     }
     mutex_unlock(&proc->files_lock);
@@ -278,25 +347,19 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
 }
 
 struct vfs_file* process_fd_get(struct process* proc, int fd) {
-    struct vfs_file* file = nullptr;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        file = entry->file;
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file)
         vfs_file_ref(file);
-    }
     mutex_unlock(&proc->files_lock);
     return file;
 }
 
 errno_t process_fd_close(struct process* proc, int fd) {
-    struct vfs_file* file = nullptr;
     mutex_lock(&proc->files_lock);
-    struct fd_entry* entry = fd_lookup(proc, fd);
-    if (entry) {
-        file = entry->file;
-        hashmap_remove(&proc->files, fd, hashmap_hash_default, hashmap_eq_default);
-    }
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file)
+        fd_clear(&proc->files, fd);
     mutex_unlock(&proc->files_lock);
 
     if (!file)