diff --git a/include/zinnia/process.h b/include/zinnia/process.h index d8dcb9d..68b7a81 100644 --- a/include/zinnia/process.h +++ b/include/zinnia/process.h @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -26,9 +25,12 @@ struct process_rlimit { rlim_t max; }; -struct fd_entry { - struct vfs_file* file; - bool cloexec; +struct fd_table { + struct vfs_file** files; + uint64_t* open; + uint64_t* cloexec; + size_t capacity; + size_t next_free; }; struct process { @@ -50,7 +52,7 @@ struct process { struct process_rlimit limit_nofile; mutex_t files_lock; - hashmap(int, struct fd_entry) files; + struct fd_table files; }; struct process* process_kernel(); diff --git a/lib/process.c b/lib/process.c index 99649ef..062ae46 100644 --- a/lib/process.c +++ b/lib/process.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,18 +10,103 @@ #include #include #include +#include + +constexpr size_t PROCESS_INITIAL_FDS = 64; static struct process kernel_process; static atomic_bool kernel_process_ready; static atomic_int next_pid = 1; +static size_t fd_words(size_t capacity) { + return capacity / 64; +} + +static void fd_table_destroy(struct fd_table* t) { + mem_free(t->files); + mem_free(t->open); + mem_free(t->cloexec); + *t = (struct fd_table){}; +} + +static errno_t fd_table_resize(struct fd_table* t, size_t capacity) { + struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0); + uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0); + uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0); + if (!files || !open || !cloexec) { + mem_free(files); + mem_free(open); + mem_free(cloexec); + return ENOMEM; + } + + if (t->capacity) { + memcpy(files, t->files, t->capacity * sizeof(*files)); + memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open)); + memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec)); + } + mem_free(t->files); + mem_free(t->open); + mem_free(t->cloexec); + + t->files = files; + t->open = open; + t->cloexec = cloexec; + t->capacity = capacity; + return 0; +} + +static errno_t fd_table_grow(struct fd_table* t, size_t fd) { + if (fd < t->capacity) + return 0; + size_t capacity = MAX(PROCESS_INITIAL_FDS, t->capacity); + while (capacity <= fd) + capacity *= 2; + return fd_table_resize(t, MIN(capacity, PROCESS_MAX_FDS)); +} + +static size_t fd_find_free(struct fd_table* t, size_t start) { + for (size_t w = start / 64; w < fd_words(t->capacity); w++) { + uint64_t used = t->open[w]; + if (w == start / 64) + used |= (1ull << (start % 64)) - 1; + if (~used) + return w * 64 + __builtin_ctzll(~used); + } + return MAX(start, t->capacity); +} + +static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) { + const uint64_t bit = 1ull << (fd % 64); + t->files[fd] = file; + t->open[fd / 64] |= bit; + if (cloexec) + t->cloexec[fd / 64] |= bit; + else + t->cloexec[fd / 64] &= ~bit; +} + +static void fd_clear(struct fd_table* t, size_t fd) { + const uint64_t bit = 1ull << (fd % 64); + t->files[fd] = nullptr; + t->open[fd / 64] &= ~bit; + t->cloexec[fd / 64] &= ~bit; + if (fd < t->next_free) + t->next_free = fd; +} + +static struct vfs_file* fd_lookup(struct process* proc, int fd) { + if (fd < 0 || (size_t)fd >= proc->files.capacity) + return nullptr; + return proc->files.files[fd]; +} + static void process_init_common(struct process* proc) { refcount_init(&proc->rc); spin_init(&proc->tasks_lock); tailq_init(&proc->tasks); mutex_init(&proc->lock); mutex_init(&proc->files_lock); - hashmap_init(&proc->files); vec_init(&proc->identity.groups); } @@ -37,17 +121,13 @@ struct process* process_kernel() { } static void close_files(struct process* proc) { + struct fd_table* t = &proc->files; mutex_lock(&proc->files_lock); - size_t i = 0; - while (i < proc->files.capacity) { - struct fd_entry entry = proc->files.values[i]; - if (proc->files.hashes[i] == 0) { - i++; - continue; - } - hashmap_remove(&proc->files, proc->files.keys[i], hashmap_hash_default, hashmap_eq_default); - vfs_file_unref(entry.file); + for (size_t w = 0; w < fd_words(t->capacity); w++) { + for (uint64_t bits = t->open[w]; bits; bits &= bits - 1) + vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]); } + fd_table_destroy(t); mutex_unlock(&proc->files_lock); } @@ -64,7 +144,6 @@ void process_unref(struct process* proc) { if (proc->space) vma_unref(proc->space); vec_destroy(&proc->identity.groups); - hashmap_destroy(&proc->files); mem_free(proc); } @@ -192,10 +271,6 @@ errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloex return process_fd_alloc_min(proc, file, 0, cloexec, out_fd); } -static struct fd_entry* fd_lookup(struct process* proc, int fd) { - return hashmap_get(&proc->files, fd, hashmap_hash_default, hashmap_eq_default); -} - size_t process_fd_limit(struct process* proc) { return MIN((size_t)PROCESS_MAX_FDS, (size_t)atomic_load(&proc->limit_nofile.cur, ATOMIC_RELAXED)); } @@ -204,22 +279,22 @@ errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int mi if (min_fd < 0 || (size_t)min_fd >= process_fd_limit(proc)) return EINVAL; + struct fd_table* t = &proc->files; mutex_lock(&proc->files_lock); - int fd = min_fd; - while ((size_t)fd < process_fd_limit(proc) && fd_lookup(proc, fd)) - fd++; - - if ((size_t)fd >= process_fd_limit(proc)) { + const size_t fd = fd_find_free(t, MAX((size_t)min_fd, t->next_free)); + if (fd >= process_fd_limit(proc)) { mutex_unlock(&proc->files_lock); return EMFILE; } - const struct fd_entry entry = {.file = file, .cloexec = cloexec}; - errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default); + errno_t status = fd_table_grow(t, fd); if (status) { mutex_unlock(&proc->files_lock); return status; } + fd_install(t, fd, file, cloexec); + if ((size_t)min_fd <= t->next_free) + t->next_free = fd + 1; vfs_file_ref(file); mutex_unlock(&proc->files_lock); @@ -231,20 +306,15 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, if (fd < 0 || (size_t)fd >= process_fd_limit(proc)) return EBADF; - struct vfs_file* displaced = nullptr; - const struct fd_entry entry = {.file = file, .cloexec = cloexec}; + struct fd_table* t = &proc->files; mutex_lock(&proc->files_lock); - struct fd_entry* slot = fd_lookup(proc, fd); - if (slot) { - displaced = slot->file; - *slot = entry; - } else { - errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default); - if (status) { - mutex_unlock(&proc->files_lock); - return status; - } + errno_t status = fd_table_grow(t, fd); + if (status) { + mutex_unlock(&proc->files_lock); + return status; } + struct vfs_file* displaced = t->files[fd]; + fd_install(t, fd, file, cloexec); vfs_file_ref(file); mutex_unlock(&proc->files_lock); @@ -256,9 +326,8 @@ errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) { errno_t status = EBADF; mutex_lock(&proc->files_lock); - struct fd_entry* entry = fd_lookup(proc, fd); - if (entry) { - *out_cloexec = entry->cloexec; + if (fd_lookup(proc, fd)) { + *out_cloexec = (proc->files.cloexec[fd / 64] >> (fd % 64)) & 1; status = 0; } mutex_unlock(&proc->files_lock); @@ -268,9 +337,9 @@ errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) { errno_t status = EBADF; mutex_lock(&proc->files_lock); - struct fd_entry* entry = fd_lookup(proc, fd); - if (entry) { - entry->cloexec = cloexec; + struct vfs_file* file = fd_lookup(proc, fd); + if (file) { + fd_install(&proc->files, fd, file, cloexec); status = 0; } mutex_unlock(&proc->files_lock); @@ -278,25 +347,19 @@ errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) { } struct vfs_file* process_fd_get(struct process* proc, int fd) { - struct vfs_file* file = nullptr; mutex_lock(&proc->files_lock); - struct fd_entry* entry = fd_lookup(proc, fd); - if (entry) { - file = entry->file; + struct vfs_file* file = fd_lookup(proc, fd); + if (file) vfs_file_ref(file); - } mutex_unlock(&proc->files_lock); return file; } errno_t process_fd_close(struct process* proc, int fd) { - struct vfs_file* file = nullptr; mutex_lock(&proc->files_lock); - struct fd_entry* entry = fd_lookup(proc, fd); - if (entry) { - file = entry->file; - hashmap_remove(&proc->files, fd, hashmap_hash_default, hashmap_eq_default); - } + struct vfs_file* file = fd_lookup(proc, fd); + if (file) + fd_clear(&proc->files, fd); mutex_unlock(&proc->files_lock); if (!file)