process: Rewrite ELF loading, processes

boot/main.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
@@ -5,15 +7,17 @@
 #include <zinnia/compiler.h>
 #include <zinnia/console.h>
 #include <zinnia/elf.h>
-#include <zinnia/exec.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>
+#include <zinnia/initramfs.h>
 #include <zinnia/irq.h>
 #include <zinnia/module.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/utils.h>
+#include <zinnia/vfs.h>
 #include <zinnia/vma.h>
 #include <zinnia/vmo.h>
 #include <config.h>
@@ -22,52 +26,41 @@
 static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
                                     ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
 
+// Path to the init executable.
+static const char* init_path = "/init";
+static void init_option(const char* value) {
+    init_path = value;
+}
+CMDLINE_OPTION("init", "Path to the init executable", init_option);
+
 static void kernel_main_task(uintptr_t arg0) {
     errno_t status = 0;
     struct boot_info* info = (struct boot_info*)arg0;
 
-    log("main: Hello from main task!\n");
+    log("Hello from main task!\n");
 
     // Finish the initgraph.
     initgraph_run(nullptr, info);
 
+    // After everything is loaded, load all builtin drivers.
     module_init();
 
-    // TODO: Try to run init.
+    // Load all initramfs files.
+    initramfs_load_boot_modules(info);
 
-    assert(info->num_files >= 1, "No init executable provided");
+    const char* init_argv[] = {init_path, nullptr};
+    const char* init_envp[] = {"HOME=/", "TERM=console", nullptr};
 
-    struct vma* init_space;
-    assert(vma_new(&init_space) == 0, "");
+    log("Starting init \"%s\"\n", init_path);
+    log("With arguments:\n");
+    for (const char** arg = init_argv; *arg; arg++)
+        log("    %s\n", *arg);
+    log("With environment:\n");
+    for (const char** env = init_envp; *env; env++)
+        log("    %s\n", *env);
 
-    struct boot_file* file_data = &info->files[0];
-    struct task* init_task = nullptr;
-
-    struct vmo* init_file;
-    status = vmo_new_anon(&init_file);
-    assert(status == 0, "Failed to allocate a memory object");
-
-    vmo_write(init_file, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
-
-    // Load init.
-    const char* init_argv[] = {"init", nullptr};
-    const char* init_envp[] = {nullptr};
-
-    struct exec_info exec = {
-        .space = init_space,
-        .file_obj = init_file,
-        .argv = init_argv,
-        .envp = init_envp,
-    };
-
-    vmo_unref(init_file);
-
-    assert(status == 0, "Failed to load init process (%i)", status);
-    sched_add_task(&percpu_get()->sched, init_task);
-
-    while (true) {
-        irq_wait();
-    }
+    status = process_spawn_init(init_path, init_argv, init_envp);
+    assert(status == 0, "Failed to start init \"%s\" (%i)", init_path, status);
 }
 
 [[__init]]
@@ -87,7 +80,7 @@ void kernel_main(struct boot_info* info) {
     initgraph_run(sched_stage, info);
 
     struct task* main_task;
-    task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
+    task_create("main", process_kernel(), kernel_main_task, (uintptr_t)info, &main_task);
     sched_add_task(&percpu_get()->sched, main_task);
 
     irq_unlock();

fs/exec/elf.c

@@ -1,8 +1,11 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/elf.h>
-#include <zinnia/exec.h>
 #include <zinnia/errno.h>
+#include <zinnia/exec.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/mman.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
@@ -14,13 +17,59 @@
 #include <zinnia/vmo.h>
 #include <bits/elf.h>
 #include <stdckdint.h>
+#include <uapi/limits.h>
 
-errno_t elf_load(struct exec_info* info, struct task** out) {
+constexpr uintptr_t ELF_DYN_BASE = 0x10000;
+
+struct elf_image {
+    uintptr_t base;
+    uintptr_t entry;
+    uintptr_t phdr;
+    size_t phnum;
+    size_t phent;
+};
+
+static bool elf_identify(struct vmo* obj) {
+    elf_ehdr_t ehdr = {};
+    size_t read;
+    if (vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read) || read != sizeof(ehdr))
+        return false;
+
+    return memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) == 0;
+}
+
+static errno_t elf_read_interp(struct vmo* obj, const elf_phdr_t* phdr, char** out) {
+    if (phdr->p_filesz < 2 || phdr->p_filesz > __PATH_MAX)
+        return ENOEXEC;
+
+    char* path = mem_alloc(phdr->p_filesz, ALLOC_NOZERO);
+    if (!path)
+        return ENOMEM;
+
+    size_t read;
+    errno_t err = vmo_read(obj, phdr->p_offset, path, phdr->p_filesz, &read);
+    if (!err && (read != phdr->p_filesz || path[phdr->p_filesz - 1] != '\0'))
+        err = ENOEXEC;
+    if (err) {
+        mem_free(path);
+        return err;
+    }
+    *out = path;
+    return 0;
+}
+
+static errno_t elf_map(
+    struct vma* space,
+    struct vmo* obj,
+    uintptr_t dyn_base,
+    struct elf_image* out,
+    char** out_interp
+) {
     errno_t err;
 
     size_t read;
     elf_ehdr_t ehdr = {};
-    err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
+    err = vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read);
     if (err)
         return err;
     if (read != sizeof(ehdr))
@@ -37,9 +86,12 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
         return EINVAL;
     if (ehdr.e_machine != ELF_ARCH_MACHINE)
         return EINVAL;
-    if (ehdr.e_type != ET_EXEC)
+    if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN)
         return EINVAL;
-    if (!arch_mem_addr_is_user(ehdr.e_entry))
+
+    const uintptr_t base = ehdr.e_type == ET_DYN ? dyn_base : 0;
+    uintptr_t entry;
+    if (ckd_add(&entry, ehdr.e_entry, base) || !arch_mem_addr_is_user(entry))
         return ENOEXEC;
 
     const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
@@ -54,14 +106,22 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
 
     for (size_t i = 0; i < ehdr.e_phnum; i++) {
         elf_phdr_t phdr = {};
-        err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
+        err = vmo_read(obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
         if (err)
             return err;
         if (read != sizeof(phdr))
             return ENOEXEC;
 
         if (phdr.p_type == PT_PHDR)
-            phdr_addr = phdr.p_vaddr;
+            phdr_addr = phdr.p_vaddr + base;
+
+        if (phdr.p_type == PT_INTERP && out_interp) {
+            if (*out_interp)
+                return ENOEXEC;
+            err = elf_read_interp(obj, &phdr, out_interp);
+            if (err)
+                return err;
+        }
 
         if (phdr.p_type == PT_LOAD) {
             if (phdr.p_memsz == 0)
@@ -69,17 +129,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
             if (phdr.p_filesz > phdr.p_memsz)
                 return ENOEXEC;
 
-            uintptr_t seg_end;
-            if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz))
+            uintptr_t vaddr, seg_end;
+            if (ckd_add(&vaddr, phdr.p_vaddr, base) || ckd_add(&seg_end, vaddr, phdr.p_memsz))
                 return ENOEXEC;
-            if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1))
+            if (!arch_mem_addr_is_user(vaddr) || !arch_mem_addr_is_user(seg_end - 1))
                 return ENOEXEC;
 
             if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
                 return ENOEXEC;
 
             if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
-                phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset);
+                phdr_addr = vaddr + (ehdr.e_phoff - phdr.p_offset);
 
             enum prot_flags prot = 0;
             if (phdr.p_flags & PF_R)
@@ -89,44 +149,102 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
             if (phdr.p_flags & PF_X)
                 prot |= PROT_EXEC;
 
-            const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
-            const uintptr_t map_address = phdr.p_vaddr - misalign;
+            const uintptr_t misalign = vaddr & (page_size - 1);
+            const uintptr_t map_address = vaddr - misalign;
             const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
             const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
 
-            // Copy the file data into its own mapping.
-            struct vmo* phdr_obj;
-            err = vmo_new_anon(&phdr_obj);
-            if (err)
-                return err;
-
-            err = vmo_copy(phdr_obj, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
-            if (err) {
-                vmo_unref(phdr_obj);
-                return err;
+            if (backed_map_size) {
+                err = vma_map(space, obj, map_address, backed_map_size, prot, 0, phdr.p_offset - misalign);
+                if (err)
+                    return err;
+            }
+            if (total_map_size > backed_map_size) {
+                err = vma_map(
+                    space,
+                    nullptr,
+                    map_address + backed_map_size,
+                    total_map_size - backed_map_size,
+                    prot,
+                    0,
+                    0
+                );
+                if (err)
+                    return err;
             }
 
-            // We map more than we copied so the rest is filled with zeroed pages.
-            err = vma_map(info->space, phdr_obj, phdr.p_vaddr, phdr.p_memsz, prot, 0, phdr.p_offset);
-            vmo_unref(phdr_obj);
-            if (err)
-                return err;
+            const uintptr_t file_end = vaddr + phdr.p_filesz;
+            if (phdr.p_memsz > phdr.p_filesz && file_end % page_size != 0) {
+                const uintptr_t zero_end = MIN(ALIGN_UP(file_end, page_size), seg_end);
+                err = vma_zero(space, file_end, zero_end - file_end);
+                if (err)
+                    return err;
+            }
         }
     }
 
+    *out = (struct elf_image){
+        .base = base,
+        .entry = entry,
+        .phdr = phdr_addr,
+        .phnum = ehdr.e_phnum,
+        .phent = ehdr.e_phentsize,
+    };
+    return 0;
+}
+
+static errno_t elf_load(struct process* proc, struct exec_info* info, uintptr_t* out_ip, uintptr_t* out_sp) {
+    struct elf_image exe = {};
+    char* interp_path = nullptr;
+    errno_t err = elf_map(info->space, info->file_obj, ELF_DYN_BASE, &exe, &interp_path);
+
+    uintptr_t entry = exe.entry;
+    uintptr_t interp_base = 0;
+    if (!err && interp_path) {
+        struct vmo* interp;
+        err = exec_open(proc, interp_path, &interp);
+        if (!err) {
+            struct elf_image ld;
+            if (!elf_identify(interp))
+                err = ELIBBAD;
+            else
+                err = elf_map(info->space, interp, (uintptr_t)1 << (mem_high_shift() - 2), &ld, nullptr);
+            if (!err) {
+                entry = ld.entry;
+                interp_base = ld.base;
+            }
+            vmo_unref(interp);
+        }
+    }
+    mem_free(interp_path);
+    if (err)
+        return err;
+
+    const size_t page_size = arch_mem_page_size();
+
     const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
     const size_t stack_size = 2 * 1024 * 1024; // 2MiB
     const uintptr_t stack_start = highest - stack_size;
 
-    // Allocate stack.
-    struct vmo* stack;
-    err = vmo_new_anon(&stack);
+    err = vma_map(info->space, nullptr, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0);
     if (err)
         return err;
 
     // Fill in stack info.
     uintptr_t stack_off = stack_size;
 
+    uintptr_t execfn = 0;
+    if (info->exec_path) {
+        const char nul = 0;
+        stack_off -= 1;
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
+
+        const size_t len = strlen(info->exec_path);
+        stack_off -= len;
+        vma_write(info->space, stack_start + stack_off, info->exec_path, len);
+        execfn = stack_start + stack_off;
+    }
+
     // TODO: Instead of allocating, calculate it on the fly.
     size_t num_envp = 0;
     for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
@@ -148,18 +266,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
         mem_free(envp_offsets);
         mem_free(argv_offsets);
-        vmo_unref(stack);
         return ENOMEM;
     }
 
     for (size_t env = 0; env < num_envp; env++) {
         const char nul = 0;
         stack_off -= 1;
-        vmo_write(stack, stack_off, &nul, sizeof(nul), nullptr);
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
 
         const size_t len = strlen(info->envp[env]);
         stack_off -= len;
-        vmo_write(stack, stack_off, info->envp[env], len, nullptr);
+        vma_write(info->space, stack_start + stack_off, info->envp[env], len);
 
         envp_offsets[env] = stack_start + stack_off;
     }
@@ -167,11 +284,11 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     for (size_t arg = 0; arg < num_argv; arg++) {
         const char nul = 0;
         stack_off -= 1;
-        vmo_write(stack, stack_off, &nul, sizeof(nul), nullptr);
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
 
         const size_t len = strlen(info->argv[arg]);
         stack_off -= len;
-        vmo_write(stack, stack_off, info->argv[arg], len, nullptr);
+        vma_write(info->space, stack_start + stack_off, info->argv[arg], len);
 
         argv_offsets[arg] = stack_start + stack_off;
     }
@@ -182,43 +299,48 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     if ((1 + num_argv + num_envp) & 1) {
         stack_off -= sizeof(uintptr_t);
         uintptr_t zero = 0;
-        vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr);
+        vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
     }
 
 #define WRITE_AUXV(auxv, value) \
     do { \
         stack_off -= sizeof(uintptr_t); \
         uintptr_t auxv_val = value; \
-        vmo_write(stack, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
+        vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \
         stack_off -= sizeof(uintptr_t); \
         auxv_val = auxv; \
-        vmo_write(stack, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
+        vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \
     } while (0)
 
     // Write auxiliary values.
     WRITE_AUXV(AT_NULL, 0);
     WRITE_AUXV(AT_SECURE, 0);
-    WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
-    WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
-    WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
-    if (phdr_addr)
-        WRITE_AUXV(AT_PHDR, phdr_addr);
+    WRITE_AUXV(AT_PAGESZ, page_size);
+    WRITE_AUXV(AT_PHNUM, exe.phnum);
+    WRITE_AUXV(AT_PHENT, exe.phent);
+    WRITE_AUXV(AT_ENTRY, exe.entry);
+    if (exe.phdr)
+        WRITE_AUXV(AT_PHDR, exe.phdr);
+    if (interp_base)
+        WRITE_AUXV(AT_BASE, interp_base);
+    if (execfn)
+        WRITE_AUXV(AT_EXECFN, execfn);
 
     // envp pointers.
     stack_off -= sizeof(uintptr_t);
     const uintptr_t zero = 0;
-    vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr);
+    vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
     for (size_t env = num_envp; env > 0; env--) {
         stack_off -= sizeof(uintptr_t);
-        vmo_write(stack, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr);
+        vma_write(info->space, stack_start + stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]));
     }
 
     // argv pointers.
     stack_off -= sizeof(uintptr_t);
-    vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr);
+    vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
     for (size_t arg = num_argv; arg > 0; arg--) {
         stack_off -= sizeof(uintptr_t);
-        vmo_write(stack, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr);
+        vma_write(info->space, stack_start + stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]));
     }
 
     mem_free(envp_offsets);
@@ -227,27 +349,22 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     // argc
     stack_off -= sizeof(uintptr_t);
     uintptr_t argc = num_argv;
-    vmo_write(stack, stack_off, &argc, sizeof(argc), nullptr);
+    vma_write(info->space, stack_start + stack_off, &argc, sizeof(argc));
 
-    err = vma_map(info->space, stack, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0);
-    vmo_unref(stack);
-    if (err)
-        return err;
-
-    struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
-    if (!start_info)
-        return ENOMEM;
-    start_info->ip = ehdr.e_entry;
-    start_info->sp = highest - stack_size + stack_off;
-    start_info->arg = 0;
-
-    struct task* result;
-    err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
-    if (err) {
-        mem_free(start_info);
-        return err;
-    }
-
-    *out = result;
+    *out_ip = entry;
+    *out_sp = highest - stack_size + stack_off;
     return 0;
 }
+
+static struct exec_format elf_format = {
+    .name = "elf",
+    .identify = elf_identify,
+    .load = elf_load,
+};
+
+static void elf_stage_init(struct boot_info*) {
+    exec_register(&elf_format);
+}
+INIT_TASK(elf_stage, "fs.exec.elf", elf_stage_init);
+INIT_DEPENDS(elf_stage, mem_stage);
+INIT_ENTAILS(elf_stage, vfs_stage);

fs/exec/exec.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/exec.h>
+#include <zinnia/mutex.h>
+#include <zinnia/print.h>
+#include <zinnia/process.h>
+#include <zinnia/tailq.h>
+#include <zinnia/vfs.h>
+
+static mutex_t formats_lock;
+static tailq_head(struct exec_format) formats = {nullptr, &formats.tqh_first};
+
+struct exec_format* exec_identify(struct vmo* obj) {
+    mutex_lock(&formats_lock);
+    struct exec_format* format = tailq_first(&formats);
+    while (format != nullptr && !format->identify(obj))
+        format = tailq_next(format, link);
+    mutex_unlock(&formats_lock);
+    return format;
+}
+
+void exec_register(struct exec_format* format) {
+    mutex_lock(&formats_lock);
+    tailq_insert_tail(&formats, format, link);
+    mutex_unlock(&formats_lock);
+
+    log("exec: Registered new exec format \"%s\"\n", format->name);
+}
+
+errno_t exec_open(struct process* proc, const char* path, struct vmo** out) {
+    mutex_lock(&proc->lock);
+    struct vfs_path root = vfs_path_clone(proc->root);
+    struct vfs_path cwd = vfs_path_clone(proc->cwd);
+    mutex_unlock(&proc->lock);
+
+    struct vfs_file* file;
+    errno_t status = vfs_file_open(root, cwd, path, O_RDONLY | O_EXEC, 0, &proc->identity, &file);
+    vfs_path_drop(root);
+    vfs_path_drop(cwd);
+    if (status)
+        return status;
+
+    status = exec_file_object(proc, file, out);
+    vfs_file_unref(file);
+    return status;
+}
+
+errno_t exec_file_object(struct process* proc, struct vfs_file* file, struct vmo** out) {
+    errno_t status = vfs_node_check_exec(file->vnode, &proc->identity);
+    if (!status)
+        status = vfs_file_get_object(file, out);
+    return status;
+}

fs/exec/meson.build

@@ -1,3 +1,4 @@
 kernel_sources += files(
     'elf.c',
+    'exec.c',
 )

include/zinnia/process.h

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/hashmap.h>
+#include <zinnia/ident.h>
+#include <zinnia/mutex.h>
+#include <zinnia/refcount.h>
+#include <zinnia/spin.h>
+#include <zinnia/tailq.h>
+#include <zinnia/types.h>
+#include <zinnia/vfs.h>
+#include <stdint.h>
+#include <uapi/resource.h>
+
+constexpr size_t PROCESS_DEFAULT_FDS = 1024;
+constexpr size_t PROCESS_MAX_FDS = 1 << 20;
+
+struct task;
+struct vma;
+
+struct process_rlimit {
+    _Atomic(rlim_t) cur;
+    rlim_t max;
+};
+
+struct fd_entry {
+    struct vfs_file* file;
+    bool cloexec;
+};
+
+struct process {
+    struct refcount rc;
+    pid_t pid;
+
+    struct spinlock tasks_lock;
+    tailq_head(struct task) tasks;
+    size_t thread_count;
+
+    mutex_t lock;
+    struct vma* space;
+    struct vfs_path root;
+    struct vfs_path cwd;
+    struct ident identity;
+    mode_t umask;
+    int8_t nice;
+
+    struct process_rlimit limit_nofile;
+
+    mutex_t files_lock;
+    hashmap(int, struct fd_entry) files;
+};
+
+struct process* process_kernel();
+void process_ref(struct process* proc);
+void process_unref(struct process* proc);
+
+uint8_t process_priority(struct process* proc);
+void process_attach_task(struct process* proc, struct task* task);
+void process_detach_task(struct process* proc, struct task* task);
+
+errno_t process_spawn_init(const char* path, const char** argv, const char** envp);
+
+size_t process_fd_limit(struct process* proc);
+
+// Allocate a file drescriptor for `file` on `proc`.
+errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloexec, int* out_fd);
+
+// Allocate a file descriptor for `file` on `proc`, starting at at least `min_fd`.
+errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int min_fd, bool cloexec, int* out_fd);
+
+// Replaces the current file indentified by `fd` in `proc` with a new `file`.
+errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, bool cloexec);
+errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec);
+errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec);
+struct vfs_file* process_fd_get(struct process* proc, int fd);
+errno_t process_fd_close(struct process* proc, int fd);

include/zinnia/sched.h

@@ -40,6 +40,7 @@ constexpr size_t SCHED_TIME_SLICE_NS = 10'000'000;
 constexpr uint8_t SCHED_FEEDBACK_DEPTH = 8;
 constexpr uint64_t SCHED_BOOST_INTERVAL_NS = 1'000'000'000;
 
+struct process;
 struct turnstile;
 
 constexpr size_t TASK_NAME_MAX = 128;
@@ -49,6 +50,8 @@ struct task {
     size_t id;
     char name[TASK_NAME_MAX];
     struct vma* space;
+    struct process* process;
+    tailq_link(struct task) process_link;
     enum task_state state;
     struct arch_task_context context;
     struct usercopy_region* usercopy_region;
@@ -105,7 +108,7 @@ struct scheduler {
 
 typedef void (*task_fn_t)(uintptr_t arg0);
 
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out);
+errno_t task_create(const char* name, struct process* proc, task_fn_t entry, uintptr_t arg0, struct task** out);
 void task_destroy(struct task* task);
 void task_lock(struct task* task);
 void task_unlock(struct task* task);
@@ -127,6 +130,8 @@ void sched_wake(struct task* task);
 // Same as sched_wake(), but the task must be locked with task_lock().
 void sched_wake_locked(struct task* task);
 
+void sched_notify(struct task* task);
+
 [[noreturn]]
 void sched_exit();
 

lib/meson.build

@@ -14,6 +14,7 @@ kernel_sources += files(
     'park.c',
     'percpu.c',
     'print.c',
+    'process.c',
     'radix.c',
     'random.c',
     'ringbuf.c',

lib/process.c

@@ -0,0 +1,306 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/exec.h>
+#include <zinnia/hashmap.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <zinnia/process.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+
+static struct process kernel_process;
+static atomic_bool kernel_process_ready;
+static atomic_int next_pid = 1;
+
+static void process_init_common(struct process* proc) {
+    refcount_init(&proc->rc);
+    spin_init(&proc->tasks_lock);
+    tailq_init(&proc->tasks);
+    mutex_init(&proc->lock);
+    mutex_init(&proc->files_lock);
+    hashmap_init(&proc->files);
+    vec_init(&proc->identity.groups);
+}
+
+struct process* process_kernel() {
+    if (!atomic_load(&kernel_process_ready, ATOMIC_ACQUIRE)) {
+        process_init_common(&kernel_process);
+        kernel_process.pid = 0;
+        kernel_process.space = &kernel_vma;
+        atomic_store(&kernel_process_ready, true, ATOMIC_RELEASE);
+    }
+    return &kernel_process;
+}
+
+static void close_files(struct process* proc) {
+    mutex_lock(&proc->files_lock);
+    size_t i = 0;
+    while (i < proc->files.capacity) {
+        struct fd_entry entry = proc->files.values[i];
+        if (proc->files.hashes[i] == 0) {
+            i++;
+            continue;
+        }
+        hashmap_remove(&proc->files, proc->files.keys[i], hashmap_hash_default, hashmap_eq_default);
+        vfs_file_unref(entry.file);
+    }
+    mutex_unlock(&proc->files_lock);
+}
+
+void process_ref(struct process* proc) {
+    refcount_inc(&proc->rc);
+}
+
+void process_unref(struct process* proc) {
+    if (proc == &kernel_process || !refcount_dec(&proc->rc))
+        return;
+    close_files(proc);
+    vfs_path_drop(proc->root);
+    vfs_path_drop(proc->cwd);
+    if (proc->space)
+        vma_unref(proc->space);
+    vec_destroy(&proc->identity.groups);
+    hashmap_destroy(&proc->files);
+    mem_free(proc);
+}
+
+uint8_t process_priority(struct process* proc) {
+    if (proc == &kernel_process)
+        return SCHED_PRIO_DEFAULT;
+
+    // Normalize the priority from "nice" ranges to our priorities.
+    return 16 + (19 - proc->nice) * 15 / 39;
+}
+
+void process_attach_task(struct process* proc, struct task* task) {
+    spin_lock(&proc->tasks_lock);
+    tailq_insert_tail(&proc->tasks, task, process_link);
+    proc->thread_count++;
+    spin_unlock(&proc->tasks_lock);
+}
+
+void process_detach_task(struct process* proc, struct task* task) {
+    spin_lock(&proc->tasks_lock);
+    tailq_remove(&proc->tasks, task, process_link);
+    spin_unlock(&proc->tasks_lock);
+}
+
+static void rlimit_init(struct process_rlimit* limit, rlim_t cur, rlim_t max) {
+    atomic_init(&limit->cur, cur);
+    limit->max = max;
+}
+
+static errno_t process_alloc(struct process** out) {
+    struct process* proc = mem_alloc(sizeof(struct process), 0);
+    if (!proc)
+        return ENOMEM;
+    process_init_common(proc);
+    proc->pid = atomic_fetch_add(&next_pid, 1, ATOMIC_RELAXED);
+    proc->root = vfs_get_root();
+    proc->cwd = vfs_get_root();
+    proc->umask = 022;
+    rlimit_init(&proc->limit_nofile, PROCESS_DEFAULT_FDS, PROCESS_MAX_FDS);
+
+    *out = proc;
+    return 0;
+}
+
+static errno_t load_image(
+    struct process* proc,
+    struct vmo* object,
+    const char* path,
+    const char** argv,
+    const char** envp,
+    struct vma** out_space,
+    uintptr_t* out_ip,
+    uintptr_t* out_sp
+) {
+    struct vma* space;
+    errno_t status = vma_new(&space);
+    if (status)
+        return status;
+
+    struct exec_info info = {
+        .space = space,
+        .file_obj = object,
+        .exec_path = path,
+        .argv = argv,
+        .envp = envp,
+    };
+    struct exec_format* format = exec_identify(object);
+    status = format ? format->load(proc, &info, out_ip, out_sp) : ENOEXEC;
+    if (status) {
+        vma_unref(space);
+        return status;
+    }
+
+    *out_space = space;
+    return 0;
+}
+
+errno_t process_spawn_init(const char* path, const char** argv, const char** envp) {
+    struct process* proc;
+    errno_t status = process_alloc(&proc);
+    if (status)
+        return status;
+
+    struct task_start_info* start = mem_alloc(sizeof(struct task_start_info), 0);
+    if (!start) {
+        process_unref(proc);
+        return ENOMEM;
+    }
+
+    struct vmo* object;
+    status = exec_open(proc, path, &object);
+    if (!status) {
+        status = load_image(proc, object, path, argv, envp, &proc->space, &start->ip, &start->sp);
+        vmo_unref(object);
+    }
+    if (status) {
+        mem_free(start);
+        process_unref(proc);
+        return status;
+    }
+
+    struct vfs_file* console;
+    status = vfs_file_open(proc->root, proc->cwd, "/dev/console", O_RDWR, 0, ident_kernel(), &console);
+    if (status) {
+        log_warn("process: Unable to open /dev/console for init (%i)\n", status);
+    } else {
+        for (int fd = 0; fd <= 2; fd++)
+            process_fd_replace(proc, fd, console, false);
+        vfs_file_unref(console);
+    }
+
+    struct task* task;
+    status = task_create(argv[0], proc, sched_to_user, (uintptr_t)start, &task);
+    process_unref(proc);
+    if (status) {
+        mem_free(start);
+        return status;
+    }
+
+    sched_add_task(&percpu_get()->sched, task);
+    return 0;
+}
+
+errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloexec, int* out_fd) {
+    return process_fd_alloc_min(proc, file, 0, cloexec, out_fd);
+}
+
+static struct fd_entry* fd_lookup(struct process* proc, int fd) {
+    return hashmap_get(&proc->files, fd, hashmap_hash_default, hashmap_eq_default);
+}
+
+size_t process_fd_limit(struct process* proc) {
+    return MIN((size_t)PROCESS_MAX_FDS, (size_t)atomic_load(&proc->limit_nofile.cur, ATOMIC_RELAXED));
+}
+
+errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int min_fd, bool cloexec, int* out_fd) {
+    if (min_fd < 0 || (size_t)min_fd >= process_fd_limit(proc))
+        return EINVAL;
+
+    mutex_lock(&proc->files_lock);
+    int fd = min_fd;
+    while ((size_t)fd < process_fd_limit(proc) && fd_lookup(proc, fd))
+        fd++;
+
+    if ((size_t)fd >= process_fd_limit(proc)) {
+        mutex_unlock(&proc->files_lock);
+        return EMFILE;
+    }
+
+    const struct fd_entry entry = {.file = file, .cloexec = cloexec};
+    errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default);
+    if (status) {
+        mutex_unlock(&proc->files_lock);
+        return status;
+    }
+    vfs_file_ref(file);
+    mutex_unlock(&proc->files_lock);
+
+    *out_fd = fd;
+    return 0;
+}
+
+errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, bool cloexec) {
+    if (fd < 0 || (size_t)fd >= process_fd_limit(proc))
+        return EBADF;
+
+    struct vfs_file* displaced = nullptr;
+    const struct fd_entry entry = {.file = file, .cloexec = cloexec};
+    mutex_lock(&proc->files_lock);
+    struct fd_entry* slot = fd_lookup(proc, fd);
+    if (slot) {
+        displaced = slot->file;
+        *slot = entry;
+    } else {
+        errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default);
+        if (status) {
+            mutex_unlock(&proc->files_lock);
+            return status;
+        }
+    }
+    vfs_file_ref(file);
+    mutex_unlock(&proc->files_lock);
+
+    if (displaced)
+        vfs_file_unref(displaced);
+    return 0;
+}
+
+errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) {
+    errno_t status = EBADF;
+    mutex_lock(&proc->files_lock);
+    struct fd_entry* entry = fd_lookup(proc, fd);
+    if (entry) {
+        *out_cloexec = entry->cloexec;
+        status = 0;
+    }
+    mutex_unlock(&proc->files_lock);
+    return status;
+}
+
+errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
+    errno_t status = EBADF;
+    mutex_lock(&proc->files_lock);
+    struct fd_entry* entry = fd_lookup(proc, fd);
+    if (entry) {
+        entry->cloexec = cloexec;
+        status = 0;
+    }
+    mutex_unlock(&proc->files_lock);
+    return status;
+}
+
+struct vfs_file* process_fd_get(struct process* proc, int fd) {
+    struct vfs_file* file = nullptr;
+    mutex_lock(&proc->files_lock);
+    struct fd_entry* entry = fd_lookup(proc, fd);
+    if (entry) {
+        file = entry->file;
+        vfs_file_ref(file);
+    }
+    mutex_unlock(&proc->files_lock);
+    return file;
+}
+
+errno_t process_fd_close(struct process* proc, int fd) {
+    struct vfs_file* file = nullptr;
+    mutex_lock(&proc->files_lock);
+    struct fd_entry* entry = fd_lookup(proc, fd);
+    if (entry) {
+        file = entry->file;
+        hashmap_remove(&proc->files, fd, hashmap_hash_default, hashmap_eq_default);
+    }
+    mutex_unlock(&proc->files_lock);
+
+    if (!file)
+        return EBADF;
+    vfs_file_unref(file);
+    return 0;
+}

sched/core.c

@@ -4,6 +4,7 @@
 #include <zinnia/assert.h>
 #include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/stack.h>
@@ -25,7 +26,9 @@ void sched_to_user(uintptr_t info_ptr) {
 }
 
 void sched_to_user_context(uintptr_t ctx) {
-    arch_sched_jump_to_context((struct arch_context*)ctx);
+    struct arch_context context = *(struct arch_context*)ctx;
+    mem_free((void*)ctx);
+    arch_sched_jump_to_context(&context);
 }
 
 static struct spinlock task_cache_lock;
@@ -54,20 +57,25 @@ static void task_release(struct task* task) {
     spin_unlock(&task_cache_lock);
 }
 
-static void task_init_common(struct task* task, const char* name, struct vma* space) {
-    task->space = space;
+static void task_init_common(struct task* task, const char* name, struct process* proc) {
+    task->process = proc;
+    task->space = proc->space;
+    process_ref(proc);
+    if (task->space != &kernel_vma)
+        vma_ref(task->space);
     spin_init(&task->lock);
     atomic_store(&task->lockp, &task->lock, ATOMIC_RELAXED);
     strncpy(task->name, name, sizeof(task->name));
-    task->base_prio = SCHED_PRIO_DEFAULT;
+    task->base_prio = process_priority(proc);
     task->time_slice = SCHED_TIME_SLICE_NS;
     task->penalty = 0;
     task->feedback_epoch = 0;
-    atomic_store(&task->priority, SCHED_PRIO_DEFAULT, ATOMIC_RELAXED);
+    atomic_store(&task->priority, task->base_prio, ATOMIC_RELAXED);
     tailq_init(&task->pi_lenders);
+    process_attach_task(proc, task);
 }
 
-errno_t task_create_current(const char* name, struct vma* space, struct task** out) {
+errno_t task_create_current(const char* name, struct process* proc, struct task** out) {
     struct task* task = task_alloc();
     if (task == nullptr)
         return ENOMEM;
@@ -78,14 +86,14 @@ errno_t task_create_current(const char* name, struct vma* space, struct task** o
         return ENOMEM;
     }
 
-    task_init_common(task, name, space);
+    task_init_common(task, name, proc);
     *out = task;
     return 0;
 }
 
 constexpr size_t KERNEL_STACK_SIZE = 16 * 1024;
 
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
+errno_t task_create(const char* name, struct process* proc, task_fn_t entry, uintptr_t arg0, struct task** out) {
     if (!out)
         return EINVAL;
 
@@ -119,7 +127,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
         return s;
     }
 
-    task_init_common(new_task, name, space);
+    task_init_common(new_task, name, proc);
     *out = new_task;
     return 0;
 }
@@ -129,6 +137,10 @@ void task_destroy(struct task* task) {
     turnstile_free(task->turnstile);
     arch_task_fini(&task->context);
     stack_destroy(&task->kernel_stack);
+    process_detach_task(task->process, task);
+    if (task->space != &kernel_vma)
+        vma_unref(task->space);
+    process_unref(task->process);
     task_release(task);
 }
 

sched/internal.h

@@ -4,6 +4,6 @@
 
 #include <zinnia/sched.h>
 
-errno_t task_create_current(const char* name, struct vma* space, struct task** out);
+errno_t task_create_current(const char* name, struct process* proc, struct task** out);
 
 bool sched_update_priority_locked(struct task* task);

sched/sched_dumb.c

@@ -8,6 +8,7 @@
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/stack.h>
@@ -57,12 +58,12 @@ void sched_init(struct scheduler* sched) {
         tailq_init(&sched->run_queue[i]);
 
     struct task* idle_task;
-    assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!");
+    assert(task_create("idle", process_kernel(), idle_func, 0, &idle_task) == 0, "Unable to create idle task!");
     idle_task->last_rq = sched;
     sched->idle_task = idle_task;
 
     struct task* boot_task;
-    assert(task_create_current("boot", &kernel_vma, &boot_task) == 0, "Unable to create boot task!");
+    assert(task_create_current("boot", process_kernel(), &boot_task) == 0, "Unable to create boot task!");
     boot_task->state = TASK_STATE_RUNNING;
     boot_task->last_rq = sched;
     atomic_store(&boot_task->on_cpu, true, ATOMIC_RELAXED);
@@ -326,6 +327,18 @@ void sched_wake(struct task* task) {
     task_unlock(task);
 }
 
+void sched_notify(struct task* task) {
+    task_lock(task);
+    if (task->state == TASK_STATE_BLOCKED) {
+        enqueue(select_cpu(task), task);
+    } else {
+        atomic_store(&task->wake_pending, true, ATOMIC_RELEASE);
+        if (task->state == TASK_STATE_RUNNING && task->last_rq && task->last_rq != &percpu_get()->sched)
+            kick(task->last_rq);
+    }
+    task_unlock(task);
+}
+
 bool sched_update_priority_locked(struct task* task) {
     struct scheduler* sched = task->last_rq;
     if (sched == nullptr) {