*: Add SPDX header to all files

arch/x86_64/idt.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/irq.h>

arch/x86_64/irq.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/percpu.h>

arch/x86_64/serial.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>

boot/main.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
@@ -7,12 +9,15 @@
 #include <zinnia/elf.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>
+#include <zinnia/initramfs.h>
 #include <zinnia/irq.h>
 #include <zinnia/module.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/utils.h>
+#include <zinnia/vfs.h>
 #include <zinnia/vma.h>
 #include <zinnia/vmo.h>
 #include <config.h>
@@ -21,52 +26,41 @@
 static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
                                     ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
 
+// Path to the init executable.
+static const char* init_path = "/init";
+static void init_option(const char* value) {
+    init_path = value;
+}
+CMDLINE_OPTION("init", "Path to the init executable", init_option);
+
 static void kernel_main_task(uintptr_t arg0) {
     errno_t status = 0;
     struct boot_info* info = (struct boot_info*)arg0;
 
-    log("main: Hello from main task!\n");
+    log("Hello from main task!\n");
 
     // Finish the initgraph.
     initgraph_run(nullptr, info);
 
+    // After everything is loaded, load all builtin drivers.
     module_init();
 
-    // TODO: Try to run init.
+    // Load all initramfs files.
+    initramfs_load_boot_modules(info);
 
-    assert(info->num_files >= 1, "No init executable provided");
+    const char* init_argv[] = {init_path, nullptr};
+    const char* init_envp[] = {"HOME=/", "TERM=console", nullptr};
 
-    struct vma* init_space;
-    assert(vma_new(&init_space) == 0, "");
+    log("Starting init \"%s\"\n", init_path);
+    log("With arguments:\n");
+    for (const char** arg = init_argv; *arg; arg++)
+        log("    %s\n", *arg);
+    log("With environment:\n");
+    for (const char** env = init_envp; *env; env++)
+        log("    %s\n", *env);
 
-    struct boot_file* file_data = &info->files[0];
-    struct task* init_task = nullptr;
-
-    struct paged_vmo* init_file;
-    status = vmo_new_phys(&init_file);
-    assert(status == 0, "Failed to allocate a memory object");
-
-    vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
-
-    // Load init.
-    const char* init_argv[] = {"init", nullptr};
-    const char* init_envp[] = {nullptr};
-
-    struct exec_info exec = {
-        .space = init_space,
-        .file_obj = &init_file->object,
-        .argv = init_argv,
-        .envp = init_envp,
-    };
-
-    vmo_unref(&init_file->object);
-
-    assert(status == 0, "Failed to load init process (%i)", status);
-    sched_add_task(&percpu_get()->sched, init_task);
-
-    while (true) {
-        irq_wait();
-    }
+    status = process_spawn_init(init_path, init_argv, init_envp);
+    assert(status == 0, "Failed to start init \"%s\" (%i)", init_path, status);
 }
 
 [[__init]]
@@ -86,7 +80,7 @@ void kernel_main(struct boot_info* info) {
     initgraph_run(sched_stage, info);
 
     struct task* main_task;
-    task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
+    task_create("main", process_kernel(), kernel_main_task, (uintptr_t)info, &main_task);
     sched_add_task(&percpu_get()->sched, main_task);
 
     irq_unlock();

dev/acpi/init.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #define pr_fmt(fmt) "acpi: " fmt
 
 #include <zinnia/acpi.h>

dev/acpi/uacpi_glue.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/acpi.h>
 #include <zinnia/alloc.h>
 #include <zinnia/console.h>

dev/block/bio.c

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/utils.h>
+
+struct bio* bio_alloc(enum bio_op op, size_t num_segs) {
+    struct bio* bio = mem_alloc(sizeof(struct bio) + num_segs * sizeof(struct bio_seg), 0);
+    if (!bio)
+        return nullptr;
+    bio->op = op;
+    bio->num_segs = num_segs;
+    event_init(&bio->event);
+    atomic_init(&bio->pending, 0);
+    atomic_init(&bio->short_offset, UINT64_MAX);
+    atomic_init(&bio->child_status, 0);
+    return bio;
+}
+
+void bio_free(struct bio* bio) {
+    mem_free(bio);
+}
+
+static void record_short(struct bio* parent, uint64_t offset, errno_t status) {
+    uint64_t current = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    while (offset < current) {
+        if (atomic_compare_exchange_weak(&parent->short_offset, &current, offset, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            break;
+    }
+    if (status) {
+        errno_t expected = 0;
+        atomic_compare_exchange(&parent->child_status, &expected, status, ATOMIC_RELAXED, ATOMIC_RELAXED);
+    }
+}
+
+static void parent_put(struct bio* parent) {
+    if (atomic_fetch_sub(&parent->pending, 1, ATOMIC_ACQ_REL) != 1)
+        return;
+    const uint64_t offset = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    const size_t done = offset == UINT64_MAX ? parent->num_lbas : offset;
+    const errno_t status = atomic_load(&parent->child_status, ATOMIC_RELAXED);
+    bio_complete(parent, done == 0 ? status : 0, done);
+}
+
+void bio_complete(struct bio* bio, errno_t status, size_t done_lbas) {
+    bio->status = status;
+    bio->done_lbas = done_lbas;
+
+    struct bio* parent = bio->parent;
+    if (parent) {
+        if (status || done_lbas < bio->num_lbas)
+            record_short(parent, bio->parent_offset + done_lbas, status);
+        bio_free(bio);
+        parent_put(parent);
+        return;
+    }
+
+    if (bio->end)
+        bio->end(bio);
+    else
+        event_set(&bio->event);
+}
+
+errno_t bio_wait(struct bio* bio, size_t* out_lbas) {
+    event_wait(&bio->event);
+    if (out_lbas)
+        *out_lbas = bio->done_lbas;
+    return bio->status;
+}
+
+// Submits one bio and completes early if it failed.
+static void submit_one(struct blockdev* dev, struct bio* bio) {
+    errno_t status = dev->ops->submit(dev, bio);
+    if (status)
+        bio_complete(bio, status, 0);
+}
+
+void bio_submit(struct blockdev* dev, struct bio* bio) {
+    bio->dev = dev;
+    if (bio->op == BIO_FLUSH) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    if (bio->op == BIO_READ) {
+        if (bio->lba >= dev->lba_count) {
+            bio_complete(bio, 0, 0);
+            return;
+        }
+        bio->num_lbas = MIN(bio->num_lbas, dev->lba_count - bio->lba);
+    }
+    if (bio->num_lbas == 0) {
+        bio_complete(bio, 0, 0);
+        return;
+    }
+
+    const size_t lba_size = dev->lba_size;
+    const size_t lbas_per_page = MAX(arch_mem_page_size() / lba_size, 1);
+    size_t max_lbas = dev->limits.max_lbas ? dev->limits.max_lbas : SIZE_MAX;
+    if (max_lbas > lbas_per_page)
+        max_lbas -= max_lbas % lbas_per_page;
+    const size_t max_segs = dev->limits.max_segs ? dev->limits.max_segs : SIZE_MAX;
+
+    if (bio->num_lbas <= max_lbas && bio->num_segs <= max_segs) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    atomic_store(&bio->pending, 1, ATOMIC_RELAXED);
+    atomic_store(&bio->short_offset, UINT64_MAX, ATOMIC_RELAXED);
+    atomic_store(&bio->child_status, 0, ATOMIC_RELAXED);
+
+    size_t seg = 0;
+    size_t seg_off = 0;
+    uint64_t consumed = 0;
+    while (consumed < bio->num_lbas) {
+        const size_t cap = MIN(max_segs, bio->num_segs - seg);
+        struct bio* child = cap ? bio_alloc(bio->op, cap) : nullptr;
+        if (!child) {
+            record_short(bio, consumed, cap ? ENOMEM : EINVAL);
+            break;
+        }
+
+        size_t count = 0;
+        size_t lbas = 0;
+        while (lbas < max_lbas && count < cap && consumed + lbas < bio->num_lbas) {
+            const struct bio_seg* src = &bio->segs[seg];
+            const size_t take =
+                MIN(MIN(src->len - seg_off, (max_lbas - lbas) * lba_size),
+                    (bio->num_lbas - consumed - lbas) * lba_size);
+            if (take < lba_size)
+                break;
+            child->segs[count++] = (struct bio_seg){.addr = src->addr + seg_off, .len = take};
+            lbas += take / lba_size;
+            seg_off += take;
+            if (seg_off == src->len) {
+                seg++;
+                seg_off = 0;
+            }
+        }
+
+        if (lbas == 0) {
+            bio_free(child);
+            record_short(bio, consumed, EINVAL);
+            break;
+        }
+
+        child->num_segs = count;
+        child->num_lbas = lbas;
+        child->lba = bio->lba + consumed;
+        child->parent = bio;
+        child->parent_offset = consumed;
+        child->dev = dev;
+        consumed += lbas;
+        atomic_fetch_add(&bio->pending, 1, ATOMIC_RELAXED);
+        submit_one(dev, child);
+    }
+
+    parent_put(bio);
+}

dev/block/blockdev.c

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "block: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/tailq.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vec.h>
+#include <zinnia/vma.h>
+#include <string.h>
+#include <uapi/block.h>
+
+static size_t lbas_per_page(struct blockdev* dev) {
+    return arch_mem_page_size() / dev->lba_size;
+}
+
+static errno_t cache_get_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t page_size = arch_mem_page_size();
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+
+    if (lba >= dev->lba_count) {
+        *out_valid = 0;
+        return 0;
+    }
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_READ, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+
+    const size_t valid = done / per_page + (done % per_page != 0);
+    if (done % per_page != 0) {
+        const size_t tail = (done % per_page) * dev->lba_size;
+        memset((uint8_t*)HHDM_PTR(vm_page_addr(pages[valid - 1])) + tail, 0, page_size - tail);
+    }
+    *out_valid = valid;
+    return valid == 0 && num_lbas > 0 ? EIO : 0;
+}
+
+static errno_t cache_put_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+    if (lba >= dev->lba_count)
+        return 0;
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_WRITE, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+    return done < num_lbas ? EIO : 0;
+}
+
+static void cache_destroy(struct vmo* vmo) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    if (dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static const struct vmo_pager_ops cache_pager = {
+    .get_pages = cache_get_pages,
+    .put_pages = cache_put_pages,
+    .max_cluster = 64,
+    .destroy = cache_destroy,
+};
+
+void blockdev_init(
+    struct blockdev* dev,
+    const struct blockdev_ops* ops,
+    uint32_t major,
+    uint32_t minor,
+    size_t lba_size,
+    uint64_t lba_count
+) {
+    vmo_init(&dev->cache, &cache_pager);
+    dev->ops = ops;
+    dev->dev = dev_make(major, minor);
+    dev->lba_size = lba_size;
+    dev->lba_count = lba_count;
+    atomic_init(&dev->claimed, false);
+}
+
+void blockdev_ref(struct blockdev* dev) {
+    vmo_ref(&dev->cache);
+}
+
+void blockdev_unref(struct blockdev* dev) {
+    vmo_unref(&dev->cache);
+}
+
+errno_t blockdev_rw(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    const struct bio_seg* segs,
+    size_t num_segs,
+    size_t* out_lbas
+) {
+    struct bio* bio = bio_alloc(op, num_segs);
+    if (!bio)
+        return ENOMEM;
+    memcpy(bio->segs, segs, num_segs * sizeof(struct bio_seg));
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_rw_pages(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    struct page** pages,
+    size_t count,
+    size_t* out_lbas
+) {
+    const size_t page_size = arch_mem_page_size();
+    size_t bytes = num_lbas * dev->lba_size;
+    if (bytes > count * page_size)
+        return EINVAL;
+
+    struct bio* bio = bio_alloc(op, count);
+    if (!bio)
+        return ENOMEM;
+
+    size_t num_segs = 0;
+    for (size_t i = 0; i < count && bytes > 0; i++) {
+        const size_t len = MIN(bytes, page_size);
+        bio->segs[num_segs++] = (struct bio_seg){.addr = vm_page_addr(pages[i]), .len = len};
+        bytes -= len;
+    }
+    bio->num_segs = num_segs;
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_flush(struct blockdev* dev) {
+    struct bio* bio = bio_alloc(BIO_FLUSH, 0);
+    if (!bio)
+        return ENOMEM;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, nullptr);
+    bio_free(bio);
+    return status;
+}
+
+static struct blockdev* file_dev(struct vfs_file* file) {
+    return file->priv;
+}
+
+static uint64_t dev_size(struct blockdev* dev) {
+    return dev->lba_count * dev->lba_size;
+}
+
+static errno_t blockdev_file_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct blockdev* dev = file_dev(file);
+    *out_read = 0;
+    if (offset >= dev_size(dev))
+        return 0;
+
+    size_t progress;
+    errno_t status =
+        vmo_read_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_read = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct blockdev* dev = file_dev(file);
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (offset >= dev_size(dev))
+        return ENOSPC;
+
+    size_t progress;
+    errno_t status =
+        vmo_write_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_written = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    struct blockdev* dev = file_dev(file);
+    switch (request) {
+    case __BLKGETSIZE64: {
+        const uint64_t size = dev_size(dev);
+        if (!usercopy_write((__user void*)arg, &size, sizeof(size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKSSZGET: {
+        const unsigned int lba_size = dev->lba_size;
+        if (!usercopy_write((__user void*)arg, &lba_size, sizeof(lba_size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKRRPART: {
+        errno_t status = blockdev_rescan(dev);
+        if (!status)
+            *out_result = 0;
+        return status;
+    }
+    default:
+        if (dev->ops->ioctl)
+            return dev->ops->ioctl(dev, file, request, arg, out_result);
+        return ENOTTY;
+    }
+}
+
+static errno_t blockdev_file_sync(struct vfs_file* file, bool data_only) {
+    struct blockdev* dev = file_dev(file);
+    errno_t status = vmo_sync(&dev->cache, 0, UINT64_MAX);
+    if (status)
+        return status;
+    return blockdev_flush(dev);
+}
+
+static errno_t blockdev_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    errno_t status =
+        vma_map(vma, &file_dev(file)->cache, addr, len, prot, (flags & MAP_SHARED) ? VMA_SHARED : 0, offset);
+    if (status)
+        return status;
+    *out_addr = addr;
+    return 0;
+}
+
+static void blockdev_file_release(struct vfs_file* file) {
+    blockdev_unref(file_dev(file));
+}
+
+static const struct vfs_file_ops blockdev_file_ops = {
+    .read = blockdev_file_read,
+    .write = blockdev_file_write,
+    .ioctl = blockdev_file_ioctl,
+    .sync = blockdev_file_sync,
+    .mmap = blockdev_file_mmap,
+    .release = blockdev_file_release,
+};
+
+static errno_t blockdev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    file->priv = dev;
+    file->ops = &blockdev_file_ops;
+    return 0;
+}
+
+static void blockdev_node_destroy(struct vfs_node* node) {
+    blockdev_unref(node->priv);
+}
+
+static const struct vfs_node_ops blockdev_node_ops = {
+    .destroy = blockdev_node_destroy,
+    .device = {
+        .open = blockdev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_BLOCKDEV,
+        .ops = &blockdev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    blockdev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        blockdev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out) {
+    if (node->type != VNODE_BLOCKDEV || node->ops != &blockdev_node_ops)
+        return ENOTBLK;
+
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    *out = dev;
+    return 0;
+}
+
+errno_t blockdev_claim(struct blockdev* dev) {
+    bool expected = false;
+    if (!atomic_compare_exchange(&dev->claimed, &expected, true, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return EBUSY;
+    return 0;
+}
+
+void blockdev_release(struct blockdev* dev) {
+    atomic_store(&dev->claimed, false, ATOMIC_RELEASE);
+}
+
+struct disk_node {
+    char name[64];
+    struct blockdev* dev;
+};
+
+struct disk {
+    char name[32];
+    struct blockdev* dev;
+    vec(struct disk_node) nodes;
+    tailq_link(struct disk) link;
+};
+
+static tailq_head(struct disk) disks;
+static mutex_t disks_lock;
+
+static void add_node(struct disk* disk, const char* name, struct blockdev* dev) {
+    struct disk_node node = {.dev = dev};
+    ksnprintf(node.name, sizeof(node.name), "%s", name);
+    if (vec_push(&disk->nodes, node) && dev)
+        blockdev_unref(dev);
+}
+
+static void scan_partitions(struct disk* disk) {
+    struct gpt_partition* parts;
+    size_t count;
+    if (gpt_scan(disk->dev, &parts, &count))
+        return;
+
+    for (size_t i = 0; i < count; i++) {
+        const struct gpt_partition* part = &parts[i];
+        if (part->end_lba < part->start_lba || part->end_lba >= disk->dev->lba_count) {
+            log_warn("Ignoring partition %zu of \"%s\" with invalid range\n", i + 1, disk->name);
+            continue;
+        }
+
+        char part_name[48];
+        ksnprintf(part_name, sizeof(part_name), "%sp%zu", disk->name, i + 1);
+
+        struct blockdev* dev;
+        errno_t status = partition_new(disk->dev, part->start_lba, part->end_lba - part->start_lba + 1, &dev);
+        if (status) {
+            log_warn("Unable to create partition \"%s\" (%i)\n", part_name, status);
+            continue;
+        }
+
+        char path[64];
+        ksnprintf(path, sizeof(path), "block/%s", part_name);
+        status = devtmpfs_add_blockdev(path, dev, 0660);
+        if (status) {
+            blockdev_unref(dev);
+            log_warn("Unable to register \"%s\" (%i)\n", path, status);
+            continue;
+        }
+        add_node(disk, path, dev);
+
+        char type_str[GUID_STR_LEN];
+        char uuid_str[GUID_STR_LEN];
+        guid_format(&part->type_guid, type_str);
+        guid_format(&part->unique_guid, uuid_str);
+
+        ksnprintf(path, sizeof(path), "block/parttype-%s", type_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+        else if (status == EEXIST)
+            log("Partition type %s already claimed, no \"%s\" link for \"%s\"\n", type_str, path, part_name);
+
+        ksnprintf(path, sizeof(path), "block/partuuid-%s", uuid_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+
+        log("Partition %zu: \"%s\" Type: %s UUID: %s\n", i + 1, part_name, type_str, uuid_str);
+    }
+
+    mem_free(parts);
+}
+
+errno_t blockdev_register(const char* name, struct blockdev* dev) {
+    struct disk* disk = mem_alloc(sizeof(struct disk), 0);
+    if (!disk)
+        return ENOMEM;
+    ksnprintf(disk->name, sizeof(disk->name), "%s", name);
+    vec_init(&disk->nodes);
+
+    char path[64];
+    ksnprintf(path, sizeof(path), "block/%s", name);
+    errno_t status = devtmpfs_add_blockdev(path, dev, 0660);
+    if (status) {
+        mem_free(disk);
+        return status;
+    }
+
+    blockdev_ref(dev);
+    disk->dev = dev;
+    log("Registered block device \"%s\"\n", name);
+
+    mutex_lock(&disks_lock);
+    tailq_insert_tail(&disks, disk, link);
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_rescan(struct blockdev* dev) {
+    mutex_lock(&disks_lock);
+    struct disk* disk;
+    for (disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        if (disk->dev == dev)
+            break;
+    }
+    if (!disk) {
+        mutex_unlock(&disks_lock);
+        return ENOTTY;
+    }
+
+    for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+        struct disk_node* node = &vec_elem_get(&disk->nodes, i);
+        errno_t status = devtmpfs_remove(node->name);
+        if (status)
+            log_warn("Unable to remove stale node \"%s\" (%i)\n", node->name, status);
+        if (node->dev)
+            blockdev_unref(node->dev);
+    }
+    disk->nodes.length = 0;
+
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_sync_all() {
+    errno_t result = 0;
+    mutex_lock(&disks_lock);
+    for (struct disk* disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+            struct blockdev* part = vec_elem_get(&disk->nodes, i).dev;
+            errno_t status = part ? vmo_sync(&part->cache, 0, UINT64_MAX) : 0;
+            if (status && !result)
+                result = status;
+        }
+        errno_t status = vmo_sync(&disk->dev->cache, 0, UINT64_MAX);
+        if (!status)
+            status = blockdev_flush(disk->dev);
+        if (status && !result)
+            result = status;
+    }
+    mutex_unlock(&disks_lock);
+    return result;
+}
+
+static void block_stage_init(struct boot_info* info) {
+    tailq_init(&disks);
+    mutex_init(&disks_lock);
+    errno_t status = devtmpfs_mkdir("block", 0755);
+    if (status && status != EEXIST)
+        log_warn("Unable to create /dev/block (%i)\n", status);
+}
+
+INIT_TASK(block_stage, "dev.block", block_stage_init);
+INIT_DEPENDS(block_stage, devtmpfs_stage);

dev/block/gpt.c

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+struct [[__packed]] gpt_header {
+    char signature[8];
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t header_crc32;
+    uint32_t reserved;
+    uint64_t my_lba;
+    uint64_t alternate_lba;
+    uint64_t first_usable_lba;
+    uint64_t last_usable_lba;
+    struct guid disk_guid;
+    uint64_t partition_entry_lba;
+    uint32_t num_partition_entries;
+    uint32_t partition_entry_size;
+    uint32_t partition_entry_array_crc32;
+};
+
+struct [[__packed]] gpt_entry {
+    struct guid type_guid;
+    struct guid unique_guid;
+    uint64_t start_lba;
+    uint64_t end_lba;
+    uint64_t attributes;
+    uint16_t name[36];
+};
+
+constexpr size_t GPT_MAX_ENTRIES = 1024;
+
+errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count) {
+    struct gpt_header header;
+    size_t read;
+    errno_t status = vmo_read(&dev->cache, dev->lba_size, &header, sizeof(header), &read);
+    if (status)
+        return status;
+    if (read != sizeof(header) || memcmp(header.signature, "EFI PART", 8) != 0)
+        return ENODATA;
+    if (header.partition_entry_size < sizeof(struct gpt_entry) || header.num_partition_entries > GPT_MAX_ENTRIES)
+        return EINVAL;
+
+    struct gpt_partition* parts = mem_alloc(MAX(header.num_partition_entries, 1) * sizeof(struct gpt_partition), 0);
+    if (!parts)
+        return ENOMEM;
+
+    size_t count = 0;
+    const uint64_t base = header.partition_entry_lba * dev->lba_size;
+    for (size_t i = 0; i < header.num_partition_entries; i++) {
+        struct gpt_entry entry;
+        status = vmo_read(&dev->cache, base + i * header.partition_entry_size, &entry, sizeof(entry), &read);
+        if (status || read != sizeof(entry))
+            break;
+        if (guid_is_zero(&entry.type_guid))
+            continue;
+
+        struct gpt_partition* part = &parts[count++];
+        part->type_guid = entry.type_guid;
+        part->unique_guid = entry.unique_guid;
+        part->start_lba = entry.start_lba;
+        part->end_lba = entry.end_lba;
+    }
+
+    *out = parts;
+    *out_count = count;
+    return 0;
+}

dev/block/meson.build

@@ -1,3 +1,11 @@
+kernel_sources += files(
+    'bio.c',
+    'blockdev.c',
+    'gpt.c',
+    'partition.c',
+    'ram.c',
+)
+
 if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
     subdir('nvme')
 endif

dev/block/nvme/pci.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/errno.h>
 #include <zinnia/module.h>
 #include <zinnia/pci.h>

dev/block/partition.c

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <stdint.h>
+
+struct partition {
+    struct blockdev blk;
+    struct blockdev* parent;
+    uint64_t start;
+};
+
+static _Atomic(uint32_t) next_minor;
+
+static errno_t partition_submit(struct blockdev* blk, struct bio* bio) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    struct blockdev* parent = part->parent;
+
+    if (bio->op != BIO_FLUSH) {
+        if (bio->lba >= blk->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+        if (bio->op == BIO_WRITE && bio->num_lbas > blk->lba_count - bio->lba) {
+            bio_complete(bio, ENOSPC, 0);
+            return 0;
+        }
+        bio->lba += part->start;
+    }
+    return parent->ops->submit(parent, bio);
+}
+
+static errno_t partition_ioctl(
+    struct blockdev* blk,
+    struct vfs_file* file,
+    uintptr_t request,
+    uintptr_t arg,
+    uintptr_t* out
+) {
+    struct blockdev* parent = containerof(blk, struct partition, blk)->parent;
+    if (!parent->ops->ioctl)
+        return ENOTTY;
+    return parent->ops->ioctl(parent, file, request, arg, out);
+}
+
+static void partition_destroy(struct blockdev* blk) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    blockdev_unref(part->parent);
+    mem_free(part);
+}
+
+static const struct blockdev_ops partition_ops = {
+    .submit = partition_submit,
+    .ioctl = partition_ioctl,
+    .destroy = partition_destroy,
+};
+
+errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out) {
+    struct partition* part = mem_alloc(sizeof(struct partition), 0);
+    if (!part)
+        return ENOMEM;
+
+    blockdev_init(
+        &part->blk,
+        &partition_ops,
+        259,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        parent->lba_size,
+        lba_count
+    );
+    part->blk.limits = parent->limits;
+    blockdev_ref(parent);
+    part->parent = parent;
+    part->start = start_lba;
+    *out = &part->blk;
+    return 0;
+}

dev/block/ram.c

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+constexpr uint32_t RAM_MAJOR = 1;
+constexpr size_t RAM_LBA_SIZE = 512;
+
+struct ramdisk {
+    struct blockdev blk;
+    phys_t base;
+};
+
+static atomic_uint next_minor;
+
+static errno_t ramdisk_submit(struct blockdev* blk, struct bio* bio) {
+    struct ramdisk* ram = containerof(blk, struct ramdisk, blk);
+
+    if (bio->op == BIO_FLUSH) {
+        bio_complete(bio, 0, 0);
+        return 0;
+    }
+    if (bio->lba >= blk->lba_count || bio->num_lbas > blk->lba_count - bio->lba) {
+        bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+        return 0;
+    }
+
+    uint64_t lba = bio->lba;
+    for (size_t i = 0; i < bio->num_segs; i++) {
+        const struct bio_seg* seg = &bio->segs[i];
+        void* store = HHDM_PTR(ram->base + lba * RAM_LBA_SIZE);
+        void* buffer = HHDM_PTR(seg->addr);
+        if (bio->op == BIO_READ)
+            memcpy(buffer, store, seg->len);
+        else
+            memcpy(store, buffer, seg->len);
+        lba += seg->len / RAM_LBA_SIZE;
+    }
+
+    bio_complete(bio, 0, bio->num_lbas);
+    return 0;
+}
+
+static void ramdisk_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct ramdisk, blk));
+}
+
+static const struct blockdev_ops ramdisk_ops = {
+    .submit = ramdisk_submit,
+    .destroy = ramdisk_destroy,
+};
+
+errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out) {
+    struct ramdisk* ram = mem_alloc(sizeof(struct ramdisk), 0);
+    if (!ram)
+        return ENOMEM;
+
+    blockdev_init(
+        &ram->blk,
+        &ramdisk_ops,
+        RAM_MAJOR,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        RAM_LBA_SIZE,
+        length / RAM_LBA_SIZE
+    );
+    ram->base = base;
+    *out = &ram->blk;
+    return 0;
+}

dev/char/chardev.c

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+
+void chardev_init(
+    struct chardev* dev,
+    const struct chardev_ops* ops,
+    const struct vfs_file_ops* file_ops,
+    uint32_t major,
+    uint32_t minor
+) {
+    refcount_init(&dev->rc);
+    dev->ops = ops;
+    dev->file_ops = file_ops;
+    dev->dev = dev_make(major, minor);
+}
+
+void chardev_ref(struct chardev* dev) {
+    refcount_inc(&dev->rc);
+}
+
+void chardev_unref(struct chardev* dev) {
+    if (refcount_dec(&dev->rc) && dev->ops && dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static errno_t chardev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct chardev* dev = node->priv;
+    file->ops = dev->file_ops;
+    if (dev->ops && dev->ops->open)
+        return dev->ops->open(dev, file);
+    return 0;
+}
+
+static void chardev_node_destroy(struct vfs_node* node) {
+    chardev_unref(node->priv);
+}
+
+static const struct vfs_node_ops chardev_node_ops = {
+    .destroy = chardev_node_destroy,
+    .device = {
+        .open = chardev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_chardev(const char* name, struct chardev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_CHARDEV,
+        .ops = &chardev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    chardev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        chardev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t devtmpfs_mkdir(const char* name, mode_t mode) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mkdir(root, root, name, mode, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}
+
+errno_t devtmpfs_remove(const char* name) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_unlink(root, root, name, ident_kernel());
+    vfs_path_drop(root);
+    if (status)
+        return status;
+    devctl_notify_destroy(name);
+    return 0;
+}
+
+errno_t devtmpfs_symlink(const char* name, const char* target) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_symlink(root, root, name, target, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}

dev/char/devctl.c

@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/mutex.h>
+#include <zinnia/poll.h>
+#include <zinnia/tailq.h>
+#include <string.h>
+
+constexpr size_t DEVCTL_QUEUE_CAPACITY = 256;
+constexpr size_t DEVCTL_BACKLOG_CAPACITY = 256;
+
+struct devctl_line {
+    tailq_link(struct devctl_line) link;
+    size_t len;
+    char data[];
+};
+
+struct devctl_queue {
+    tailq_head(struct devctl_line) lines;
+    size_t count;
+};
+
+struct devctl_reader {
+    tailq_link(struct devctl_reader) link;
+    struct devctl_queue queue;
+};
+
+static mutex_t devctl_lock;
+static tailq_head(struct devctl_reader) devctl_readers;
+static struct devctl_queue devctl_backlog;
+static bool devctl_had_reader;
+static struct poll_event devctl_event;
+static atomic_bool devctl_ready;
+static struct chardev devctl_dev;
+
+static void queue_init(struct devctl_queue* queue) {
+    tailq_init(&queue->lines);
+    queue->count = 0;
+}
+
+static struct devctl_line* queue_pop(struct devctl_queue* queue) {
+    struct devctl_line* line = tailq_first(&queue->lines);
+    if (line) {
+        tailq_remove(&queue->lines, line, link);
+        queue->count--;
+    }
+    return line;
+}
+
+static void queue_push(struct devctl_queue* queue, struct devctl_line* line, size_t capacity) {
+    if (queue->count >= capacity)
+        mem_free(queue_pop(queue));
+    tailq_insert_tail(&queue->lines, line, link);
+    queue->count++;
+}
+
+static void queue_clear(struct devctl_queue* queue) {
+    struct devctl_line* line;
+    while ((line = queue_pop(queue)))
+        mem_free(line);
+}
+
+static struct devctl_line* line_new(const char* data, size_t len) {
+    struct devctl_line* line = mem_alloc(sizeof(*line) + len, 0);
+    if (!line)
+        return nullptr;
+    line->len = len;
+    memcpy(line->data, data, len);
+    return line;
+}
+
+static void devctl_notify(const char* type, const char* name) {
+    if (!atomic_load(&devctl_ready, ATOMIC_ACQUIRE))
+        return;
+
+    while (*name == '/')
+        name++;
+
+    const char prefix[] = "!system=DEVFS subsystem=CDEV type=";
+    const char middle[] = " cdev=";
+    const size_t prefix_len = sizeof(prefix) - 1;
+    const size_t type_len = strlen(type);
+    const size_t middle_len = sizeof(middle) - 1;
+    const size_t name_len = strlen(name);
+    const size_t len = prefix_len + type_len + middle_len + name_len + 1;
+
+    char* buf = mem_alloc(len, 0);
+    if (!buf)
+        return;
+    char* pos = buf;
+    memcpy(pos, prefix, prefix_len);
+    pos += prefix_len;
+    memcpy(pos, type, type_len);
+    pos += type_len;
+    memcpy(pos, middle, middle_len);
+    pos += middle_len;
+    memcpy(pos, name, name_len);
+    pos += name_len;
+    *pos = '\n';
+
+    mutex_lock(&devctl_lock);
+    if (tailq_empty(&devctl_readers)) {
+        if (!devctl_had_reader) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&devctl_backlog, line, DEVCTL_BACKLOG_CAPACITY);
+        }
+    } else {
+        for (struct devctl_reader* reader = tailq_first(&devctl_readers); reader; reader = tailq_next(reader, link)) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    mem_free(buf);
+
+    poll_event_notify(&devctl_event);
+}
+
+void devctl_notify_create(const char* name) {
+    devctl_notify("CREATE", name);
+}
+
+void devctl_notify_destroy(const char* name) {
+    devctl_notify("DESTROY", name);
+}
+
+static bool file_nonblocking(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_NONBLOCK;
+}
+
+static errno_t drain_lines(struct devctl_queue* queue, iovec_iter_t* buffer, size_t* out_total) {
+    size_t total = 0;
+    errno_t status = 0;
+    struct devctl_line* line;
+    while ((line = tailq_first(&queue->lines))) {
+        if (line->len > iovec_iter_len(buffer)) {
+            if (total == 0)
+                status = EINVAL;
+            break;
+        }
+        queue_pop(queue);
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, line->data, line->len, &copied);
+        mem_free(line);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_total = total;
+    return status;
+}
+
+static errno_t devctl_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct devctl_reader* reader = file->priv;
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &devctl_event);
+
+    errno_t status = 0;
+    size_t read = 0;
+    mutex_lock(&devctl_lock);
+    while (true) {
+        if (reader->queue.count) {
+            status = drain_lines(&reader->queue, buffer, &read);
+            break;
+        }
+        if (file_nonblocking(file)) {
+            status = EAGAIN;
+            break;
+        }
+        if (poll_table_sleep(&table, &devctl_lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    poll_table_fini(&table);
+
+    *out_read = read;
+    return status;
+}
+
+static errno_t devctl_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    return EBADF;
+}
+
+static enum poll_flags devctl_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    struct devctl_reader* reader = file->priv;
+
+    if (events & (POLLIN | POLLRDNORM))
+        poll_wait(table, &devctl_event);
+
+    enum poll_flags revents = 0;
+    mutex_lock(&devctl_lock);
+    if (reader->queue.count)
+        revents |= POLLIN | POLLRDNORM;
+    mutex_unlock(&devctl_lock);
+    return revents;
+}
+
+static void devctl_release(struct vfs_file* file) {
+    struct devctl_reader* reader = file->priv;
+
+    mutex_lock(&devctl_lock);
+    tailq_remove(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    queue_clear(&reader->queue);
+    mem_free(reader);
+}
+
+static const struct vfs_file_ops devctl_file_ops = {
+    .read = devctl_read,
+    .write = devctl_write,
+    .poll = devctl_poll,
+    .release = devctl_release,
+};
+
+static errno_t devctl_open(struct chardev* dev, struct vfs_file* file) {
+    struct devctl_reader* reader = mem_alloc(sizeof(*reader), 0);
+    if (!reader)
+        return ENOMEM;
+    queue_init(&reader->queue);
+
+    mutex_lock(&devctl_lock);
+    if (!devctl_had_reader) {
+        devctl_had_reader = true;
+        struct devctl_line* line;
+        while ((line = queue_pop(&devctl_backlog)))
+            queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+    }
+    tailq_insert_tail(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    file->priv = reader;
+    vfs_stream_open(file);
+    return 0;
+}
+
+static const struct chardev_ops devctl_dev_ops = {.open = devctl_open};
+
+static void devctl_stage_init(struct boot_info* info) {
+    mutex_init(&devctl_lock);
+    tailq_init(&devctl_readers);
+    queue_init(&devctl_backlog);
+    poll_event_init(&devctl_event);
+    atomic_store(&devctl_ready, true, ATOMIC_RELEASE);
+
+    chardev_init(&devctl_dev, &devctl_dev_ops, &devctl_file_ops, 5, 3);
+    errno_t status = devtmpfs_add_chardev("devctl", &devctl_dev, 0600);
+    assert(status == 0, "Unable to create /dev/devctl (%i)", status);
+}
+
+INIT_TASK(devctl_stage, "dev.devctl", devctl_stage_init);
+INIT_DEPENDS(devctl_stage, devtmpfs_stage);

dev/char/memfiles.c

@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/chardev.h>
+#include <zinnia/random.h>
+#include <zinnia/utils.h>
+
+constexpr size_t MEMFILE_CHUNK = 64;
+
+static const uint8_t zero_chunk[MEMFILE_CHUNK];
+
+static errno_t memfile_discard(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = iovec_iter_len(buffer);
+    return 0;
+}
+
+static errno_t memfile_read_zero(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, zero_chunk, MIN(iovec_iter_len(buffer), MEMFILE_CHUNK), &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static errno_t null_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    *out_read = 0;
+    return 0;
+}
+
+static const struct vfs_file_ops null_file_ops = {
+    .read = null_read,
+    .write = memfile_discard,
+};
+
+static const struct vfs_file_ops zero_file_ops = {
+    .read = memfile_read_zero,
+    .write = memfile_discard,
+};
+
+static errno_t full_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = 0;
+    return ENOSPC;
+}
+
+static const struct vfs_file_ops full_file_ops = {
+    .read = memfile_read_zero,
+    .write = full_write,
+};
+
+static errno_t random_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    uint8_t chunk[MEMFILE_CHUNK];
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        const size_t len = MIN(iovec_iter_len(buffer), sizeof(chunk));
+
+        // TODO: This function sucks and is probably not useful for cryptography.
+        random_fill(chunk, len);
+
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, chunk, len, &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static const struct vfs_file_ops random_file_ops = {
+    .read = random_read,
+    .write = memfile_discard,
+};
+
+static struct chardev null_dev;
+static struct chardev zero_dev;
+static struct chardev full_dev;
+static struct chardev random_dev;
+static struct chardev urandom_dev;
+
+static void memfiles_add(const char* name, struct chardev* dev, const struct vfs_file_ops* ops, uint32_t minor) {
+    chardev_init(dev, nullptr, ops, 1, minor);
+    errno_t status = devtmpfs_add_chardev(name, dev, 0666);
+    assert(status == 0, "Unable to create /dev/%s (%i)", name, status);
+}
+
+static void memfiles_stage_init(struct boot_info* info) {
+    memfiles_add("null", &null_dev, &null_file_ops, 3);
+    memfiles_add("zero", &zero_dev, &zero_file_ops, 5);
+    memfiles_add("full", &full_dev, &full_file_ops, 7);
+    memfiles_add("random", &random_dev, &random_file_ops, 8);
+    memfiles_add("urandom", &urandom_dev, &random_file_ops, 9);
+}
+
+INIT_TASK(memfiles_stage, "dev.memfiles", memfiles_stage_init);
+INIT_DEPENDS(memfiles_stage, devtmpfs_stage, random_stage);

dev/char/meson.build

@@ -1,3 +1,5 @@
 kernel_sources += files(
+    'chardev.c',
+    'devctl.c',
     'memfiles.c',
 )

dev/common/memfiles.c

@@ -1 +0,0 @@
-

dev/meson.build

@@ -1,5 +1,5 @@
-subdir('common')
 subdir('block')
+subdir('char')
 subdir('dt')
 subdir('video')
 

dev/pci/core.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #define pr_fmt(fmt) "pci: " fmt
 
 #include <zinnia/alloc.h>

dev/video/fbcon/fbcon.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/console.h>

fs/anon.c

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: ISC
+
+// anonfs is a virtual file system. It backs files that don't live in any other
+// file system, like pipes or unbound sockets.
+
+#include <zinnia/atomic.h>
+#include <zinnia/vfs.h>
+
+static const struct vfs_super_block_ops anon_super_ops = {};
+
+static struct vfs_super_block anon_super = {
+    .rc = {.count = 1}, // There's always just this exact super block.
+    .ops = &anon_super_ops,
+};
+
+static const struct vfs_node_ops anon_node_ops = {};
+
+static _Atomic(ino_t) anon_inode_counter = 1;
+
+errno_t vfs_create_anon_inode(
+    const struct vfs_file_ops* file_ops,
+    enum vfs_node_type type,
+    mode_t mode,
+    const struct ident* identity,
+    struct vfs_node** out
+) {
+    struct vfs_node* node;
+    errno_t status = vfs_node_new(&anon_super, type, &anon_node_ops, mode, &node);
+    if (status)
+        return status;
+
+    node->file_ops = file_ops;
+    // Every anon node gets its own inode counter.
+    node->id = atomic_fetch_add(&anon_inode_counter, 1, ATOMIC_RELAXED);
+    node->uid = identity->euid;
+    node->gid = identity->egid;
+
+    *out = node;
+    return 0;
+}

fs/cache.c

@@ -0,0 +1,411 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/vfs.h>
+#include <string.h>
+#include "internal.h"
+
+constexpr size_t MAX_SYMLINK_DEPTH = 40;
+
+// If `path` is a mount point, crosses into the root of the topmost mounted file system.
+// Returns a new reference.
+static struct vfs_path cross_mounts(struct vfs_path path) {
+    struct vfs_mount* mount = path.mount;
+    struct vfs_entry* entry = path.entry;
+
+again:
+    mutex_lock(&entry->lock);
+    vec_foreach(&entry->mounts, i) {
+        struct vfs_mount* child = vec_elem_get(&entry->mounts, i);
+        if (child != mount) {
+            mutex_unlock(&entry->lock);
+            mount = child;
+            entry = child->root;
+            goto again;
+        }
+    }
+    mutex_unlock(&entry->lock);
+
+    return vfs_path_clone((struct vfs_path){.mount = mount, .entry = entry});
+}
+
+errno_t vfs_path_lookup_parent(struct vfs_path path, struct vfs_path* out) {
+    struct vfs_mount* mount = path.mount;
+    struct vfs_entry* entry = path.entry;
+
+    // Get the top mount point.
+    while (mount->has_mount_point && entry == mount->root) {
+        entry = mount->mount_point.entry;
+        mount = mount->mount_point.mount;
+    }
+
+    if (!entry->parent)
+        return ENOENT;
+
+    *out = vfs_path_clone((struct vfs_path){.mount = mount, .entry = entry->parent});
+    return 0;
+}
+
+errno_t vfs_path_lookup_child(
+    struct vfs_path path,
+    struct vfs_path* root,
+    const char* name,
+    size_t name_len,
+    struct vfs_path* out
+) {
+    errno_t status = 0;
+
+    // Traverse the mounts on the current entry.
+    struct vfs_path crossed = cross_mounts(path);
+
+    // A lookup only makes sense on directories.
+    struct vfs_node* parent = vfs_entry_get_inode(crossed.entry);
+    if (!parent) {
+        status = ENOENT;
+        goto fail;
+    }
+    if (parent->type != VNODE_DIRECTORY) {
+        status = ENOTDIR;
+        goto fail;
+    }
+
+    if (name_len == 1 && name[0] == '.') {
+        vfs_node_unref(parent);
+        *out = crossed;
+        return 0;
+    }
+
+    if (name_len == 2 && name[0] == '.' && name[1] == '.') {
+        vfs_node_unref(parent);
+
+        struct vfs_path crossed_root = cross_mounts(*root);
+        bool at_root = vfs_path_equal(crossed, crossed_root);
+        vfs_path_drop(crossed_root);
+        if (at_root) {
+            *out = crossed;
+            return 0;
+        }
+
+        status = vfs_path_lookup_parent(crossed, out);
+        if (status == ENOENT) {
+            *out = crossed;
+            return 0;
+        }
+        vfs_path_drop(crossed);
+        return status;
+    }
+
+    mutex_lock(&crossed.entry->lock);
+
+    // If this entry has already been looked up before, return that.
+    struct vfs_entry* child = nullptr;
+    for (struct vfs_entry* it = tailq_first(&crossed.entry->children); it; it = tailq_next(it, child_link)) {
+        if (it->name_len == name_len && !memcmp(it->name, name, name_len)) {
+            child = it;
+            vfs_entry_ref(child);
+            break;
+        }
+    }
+
+    if (!child) {
+        status = vfs_entry_new(name, name_len, crossed.entry, &child);
+        if (status) {
+            mutex_unlock(&crossed.entry->lock);
+            goto fail;
+        }
+
+        struct vfs_path child_path = {.mount = crossed.mount, .entry = child};
+        status = parent->ops->dir.lookup ? parent->ops->dir.lookup(parent, &child_path) : ENOENT;
+        if (status && status != ENOENT) {
+            mutex_unlock(&crossed.entry->lock);
+            vfs_entry_unref(child);
+            goto fail;
+        }
+
+        // Allow lookup failures so we can cache it as a negative entry.
+        if (status == ENOENT)
+            vfs_entry_set_negative(child);
+        status = 0;
+
+        // The children list takes over the initial reference.
+        tailq_insert_tail(&crossed.entry->children, child, child_link);
+        vfs_entry_ref(child);
+    }
+
+    mutex_unlock(&crossed.entry->lock);
+
+    *out = cross_mounts((struct vfs_path){.mount = crossed.mount, .entry = child});
+    vfs_entry_unref(child);
+    vfs_node_unref(parent);
+    vfs_path_drop(crossed);
+    return 0;
+
+fail:
+    if (parent)
+        vfs_node_unref(parent);
+    vfs_path_drop(crossed);
+    return status;
+}
+
+static errno_t lookup_inner(
+    struct vfs_path root,
+    struct vfs_path start,
+    const char* path,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    size_t* symlink_depth,
+    struct vfs_path* out
+);
+
+// Resolves `current` in place until it no longer points to a symbolic link.
+static errno_t resolve_symlink(
+    struct vfs_path root,
+    struct vfs_path* current,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    size_t* symlink_depth
+) {
+    errno_t status = 0;
+    char* link_buf = nullptr;
+
+    while (true) {
+        struct vfs_node* node = vfs_entry_get_inode(current->entry);
+        if (!node)
+            break;
+        if (node->type != VNODE_SYMLINK) {
+            vfs_node_unref(node);
+            break;
+        }
+
+        if (*symlink_depth >= MAX_SYMLINK_DEPTH || !node->ops->symlink.read_link) {
+            vfs_node_unref(node);
+            status = ELOOP;
+            break;
+        }
+        (*symlink_depth)++;
+
+        if (!link_buf) {
+            link_buf = mem_alloc(__PATH_MAX + 1, ALLOC_NOZERO);
+            if (!link_buf) {
+                vfs_node_unref(node);
+                status = ENOMEM;
+                break;
+            }
+        }
+
+        iovec_t vec = {.base = (uintptr_t)link_buf, .len = __PATH_MAX};
+        iovec_iter_t iter = iovec_iter_kernel(&vec, 1);
+        size_t link_len = 0;
+        status = node->ops->symlink.read_link(node, &iter, &link_len);
+        vfs_node_unref(node);
+        if (status)
+            break;
+        link_buf[link_len] = '\0';
+
+        struct vfs_path parent;
+        status = vfs_path_lookup_parent(*current, &parent);
+        if (status)
+            break;
+
+        struct vfs_path result;
+        status = lookup_inner(
+            root,
+            parent,
+            link_buf,
+            identity,
+            flags & ~(VFS_LOOKUP_MUST_EXIST | VFS_LOOKUP_MUST_NOT_EXIST),
+            symlink_depth,
+            &result
+        );
+        vfs_path_drop(parent);
+        if (status)
+            break;
+
+        if (vfs_path_equal(result, *current)) {
+            vfs_path_drop(result);
+            status = ELOOP;
+            break;
+        }
+
+        vfs_path_drop(*current);
+        *current = result;
+    }
+
+    mem_free(link_buf);
+    return status;
+}
+
+static errno_t lookup_inner(
+    struct vfs_path root,
+    struct vfs_path start,
+    const char* path,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    size_t* symlink_depth,
+    struct vfs_path* out
+) {
+    errno_t status = 0;
+
+    if (path[0] == '\0')
+        return ENOENT;
+
+    // If a path starts with '/', it's an absolute path.
+    // In that case, skip the first character and use the current root as a starting point.
+    struct vfs_path current;
+    if (path[0] == '/') {
+        current = vfs_path_clone(root);
+        path++;
+    } else {
+        current = vfs_path_clone(start);
+    }
+
+    // Parse each component.
+    while (true) {
+        while (*path == '/')
+            path++;
+        if (*path == '\0')
+            break;
+
+        const char* component = path;
+        size_t component_len = 0;
+        while (path[component_len] != '/' && path[component_len] != '\0')
+            component_len++;
+        path += component_len;
+
+        if (component_len > __NAME_MAX) {
+            status = ENAMETOOLONG;
+            goto fail;
+        }
+
+        status = resolve_symlink(root, &current, identity, flags, symlink_depth);
+        if (status)
+            goto fail;
+
+        struct vfs_node* node = vfs_entry_get_inode(current.entry);
+        if (!node) {
+            status = ENOENT;
+            goto fail;
+        }
+        if (node->type != VNODE_DIRECTORY) {
+            vfs_node_unref(node);
+            status = ENOTDIR;
+            goto fail;
+        }
+
+        status = vfs_node_try_access(node, identity, X_OK, flags & VFS_LOOKUP_REAL_ID);
+        vfs_node_unref(node);
+        if (status)
+            goto fail;
+
+        struct vfs_path next;
+        status = vfs_path_lookup_child(current, &root, component, component_len, &next);
+        if (status)
+            goto fail;
+        vfs_path_drop(current);
+        current = next;
+    }
+
+    if (flags & VFS_LOOKUP_FOLLOW_SYMLINKS) {
+        status = resolve_symlink(root, &current, identity, flags, symlink_depth);
+        if (status)
+            goto fail;
+    }
+
+    struct vfs_node* node = vfs_entry_get_inode(current.entry);
+    if (node)
+        vfs_node_unref(node);
+
+    if ((flags & VFS_LOOKUP_MUST_EXIST) && !node) {
+        status = ENOENT;
+        goto fail;
+    }
+    if ((flags & VFS_LOOKUP_MUST_NOT_EXIST) && node) {
+        status = EEXIST;
+        goto fail;
+    }
+
+    *out = current;
+    return 0;
+
+fail:
+    vfs_path_drop(current);
+    return status;
+}
+
+errno_t vfs_path_lookup(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    const struct ident* identity,
+    enum vfs_lookup_flags flags,
+    struct vfs_path* out
+) {
+    size_t symlink_depth = 0;
+    return lookup_inner(root, cwd, path, identity, flags, &symlink_depth, out);
+}
+
+errno_t vfs_path_absolute(struct vfs_path path, struct vfs_path* root, char* buf, size_t len, size_t* out_len) {
+    if (len == 0)
+        return ENAMETOOLONG;
+
+    struct vfs_path global_root;
+    if (!root) {
+        global_root = vfs_get_root();
+        root = &global_root;
+    }
+
+    errno_t status = 0;
+    struct vfs_mount* mount = path.mount;
+    struct vfs_entry* entry = path.entry;
+
+    // Build the path back to front, leaving room for the NUL terminator.
+    size_t cursor = len - 1;
+    buf[cursor] = '\0';
+
+    while (entry != root->entry || mount != root->mount) {
+        // At the root of a mounted file system we must step into the parent file system.
+        if (entry == mount->root) {
+            if (!mount->has_mount_point) {
+                // Global root with no parent.
+                status = ENOENT;
+                goto out;
+            }
+            entry = mount->mount_point.entry;
+            mount = mount->mount_point.mount;
+            continue;
+        }
+
+        if (entry->name_len + 1 > cursor) {
+            status = ENAMETOOLONG;
+            goto out;
+        }
+        cursor -= entry->name_len;
+        memcpy(buf + cursor, entry->name, entry->name_len);
+        buf[--cursor] = '/';
+
+        if (!entry->parent) {
+            status = ENOENT;
+            goto out;
+        }
+        entry = entry->parent;
+    }
+
+    // The root itself resolves to "/".
+    if (cursor == len - 1) {
+        if (cursor == 0) {
+            status = ENAMETOOLONG;
+            goto out;
+        }
+        buf[--cursor] = '/';
+    }
+
+    size_t result_len = len - 1 - cursor;
+    memmove(buf, buf + cursor, result_len + 1);
+    if (out_len)
+        *out_len = result_len;
+
+out:
+    if (root == &global_root)
+        vfs_path_drop(global_root);
+    return status;
+}

fs/devtmpfs.c

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/chardev.h>
+#include <zinnia/ident.h>
+#include <zinnia/vfs.h>
+
+static struct vfs_mount* dev_mount;
+
+struct vfs_path devtmpfs_root() {
+    return vfs_path_clone((struct vfs_path){.mount = dev_mount, .entry = dev_mount->root});
+}
+
+static errno_t devtmpfs_mount(
+    const struct vfs_filesystem* fs,
+    enum mount_flags flags,
+    void* arg,
+    struct vfs_mount** out
+) {
+    return vfs_mount_new(flags, dev_mount->root, out);
+}
+
+static const struct vfs_filesystem devtmpfs = {
+    .name = "devtmpfs",
+    .mount = devtmpfs_mount,
+};
+
+static void devtmpfs_stage_init(struct boot_info* info) {
+    errno_t status = vfs_mount("tmpfs", 0, nullptr, &dev_mount);
+    assert(status == 0, "Unable to create the devtmpfs (%i)", status);
+
+    status = vfs_register_filesystem(&devtmpfs);
+    assert(status == 0, "Unable to register the devtmpfs (%i)", status);
+}
+
+INIT_TASK(devtmpfs_stage, "fs.devtmpfs", devtmpfs_stage_init);
+INIT_DEPENDS(devtmpfs_stage, vfs_stage);
+
+static void devtmpfs_mount_stage_init(struct boot_info* info) {
+    // Create /dev.
+    struct vfs_path root = vfs_get_root();
+    errno_t status = vfs_mkdir(root, root, "/dev", 0755, ident_kernel());
+    assert(status == 0 || status == EEXIST, "Unable to create /dev (%i)", status);
+
+    // Get /dev.
+    struct vfs_path target;
+    status = vfs_path_lookup(root, root, "/dev", ident_kernel(), VFS_LOOKUP_MUST_EXIST, &target);
+    vfs_path_drop(root);
+    assert(status == 0, "Unable to look up /dev (%i)", status);
+
+    // Prepare a devtmpfs for mounting.
+    struct vfs_mount* mount;
+    status = vfs_mount("devtmpfs", 0, nullptr, &mount);
+    assert(status == 0, "Unable to mount the devtmpfs (%i)", status);
+
+    // Mount it on /dev.
+    status = vfs_path_mount(target, mount);
+    assert(status == 0, "Unable to mount the devtmpfs on /dev (%i)", status);
+    vfs_mount_unref(mount);
+    vfs_path_drop(target);
+}
+
+INIT_TASK(devtmpfs_mount_stage, "fs.devtmpfs.mount", devtmpfs_mount_stage_init);
+INIT_DEPENDS(devtmpfs_mount_stage, devtmpfs_stage);

fs/exec/elf.c

@@ -1,7 +1,11 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/assert.h>
 #include <zinnia/elf.h>
 #include <zinnia/errno.h>
+#include <zinnia/exec.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/mman.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
@@ -13,13 +17,59 @@
 #include <zinnia/vmo.h>
 #include <bits/elf.h>
 #include <stdckdint.h>
+#include <uapi/limits.h>
 
-errno_t elf_load(struct exec_info* info, struct task** out) {
+constexpr uintptr_t ELF_DYN_BASE = 0x10000;
+
+struct elf_image {
+    uintptr_t base;
+    uintptr_t entry;
+    uintptr_t phdr;
+    size_t phnum;
+    size_t phent;
+};
+
+static bool elf_identify(struct vmo* obj) {
+    elf_ehdr_t ehdr = {};
+    size_t read;
+    if (vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read) || read != sizeof(ehdr))
+        return false;
+
+    return memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) == 0;
+}
+
+static errno_t elf_read_interp(struct vmo* obj, const elf_phdr_t* phdr, char** out) {
+    if (phdr->p_filesz < 2 || phdr->p_filesz > __PATH_MAX)
+        return ENOEXEC;
+
+    char* path = mem_alloc(phdr->p_filesz, ALLOC_NOZERO);
+    if (!path)
+        return ENOMEM;
+
+    size_t read;
+    errno_t err = vmo_read(obj, phdr->p_offset, path, phdr->p_filesz, &read);
+    if (!err && (read != phdr->p_filesz || path[phdr->p_filesz - 1] != '\0'))
+        err = ENOEXEC;
+    if (err) {
+        mem_free(path);
+        return err;
+    }
+    *out = path;
+    return 0;
+}
+
+static errno_t elf_map(
+    struct vma* space,
+    struct vmo* obj,
+    uintptr_t dyn_base,
+    struct elf_image* out,
+    char** out_interp
+) {
     errno_t err;
 
     size_t read;
     elf_ehdr_t ehdr = {};
-    err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
+    err = vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read);
     if (err)
         return err;
     if (read != sizeof(ehdr))
@@ -36,9 +86,12 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
         return EINVAL;
     if (ehdr.e_machine != ELF_ARCH_MACHINE)
         return EINVAL;
-    if (ehdr.e_type != ET_EXEC)
+    if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN)
         return EINVAL;
-    if (!arch_mem_addr_is_user(ehdr.e_entry))
+
+    const uintptr_t base = ehdr.e_type == ET_DYN ? dyn_base : 0;
+    uintptr_t entry;
+    if (ckd_add(&entry, ehdr.e_entry, base) || !arch_mem_addr_is_user(entry))
         return ENOEXEC;
 
     const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
@@ -53,14 +106,22 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
 
     for (size_t i = 0; i < ehdr.e_phnum; i++) {
         elf_phdr_t phdr = {};
-        err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
+        err = vmo_read(obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
         if (err)
             return err;
         if (read != sizeof(phdr))
             return ENOEXEC;
 
         if (phdr.p_type == PT_PHDR)
-            phdr_addr = phdr.p_vaddr;
+            phdr_addr = phdr.p_vaddr + base;
+
+        if (phdr.p_type == PT_INTERP && out_interp) {
+            if (*out_interp)
+                return ENOEXEC;
+            err = elf_read_interp(obj, &phdr, out_interp);
+            if (err)
+                return err;
+        }
 
         if (phdr.p_type == PT_LOAD) {
             if (phdr.p_memsz == 0)
@@ -68,17 +129,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
             if (phdr.p_filesz > phdr.p_memsz)
                 return ENOEXEC;
 
-            uintptr_t seg_end;
-            if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz))
+            uintptr_t vaddr, seg_end;
+            if (ckd_add(&vaddr, phdr.p_vaddr, base) || ckd_add(&seg_end, vaddr, phdr.p_memsz))
                 return ENOEXEC;
-            if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1))
+            if (!arch_mem_addr_is_user(vaddr) || !arch_mem_addr_is_user(seg_end - 1))
                 return ENOEXEC;
 
             if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
                 return ENOEXEC;
 
             if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
-                phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset);
+                phdr_addr = vaddr + (ehdr.e_phoff - phdr.p_offset);
 
             enum prot_flags prot = 0;
             if (phdr.p_flags & PF_R)
@@ -88,44 +149,102 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
             if (phdr.p_flags & PF_X)
                 prot |= PROT_EXEC;
 
-            const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
-            const uintptr_t map_address = phdr.p_vaddr - misalign;
+            const uintptr_t misalign = vaddr & (page_size - 1);
+            const uintptr_t map_address = vaddr - misalign;
             const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
             const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
 
-            // Copy the file data into its own mapping.
-            struct paged_vmo* phdr_obj;
-            err = vmo_new_phys(&phdr_obj);
-            if (err)
-                return err;
-
-            err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
-            if (err) {
-                vmo_unref(&phdr_obj->object);
-                return err;
+            if (backed_map_size) {
+                err = vma_map(space, obj, map_address, backed_map_size, prot, 0, phdr.p_offset - misalign);
+                if (err)
+                    return err;
+            }
+            if (total_map_size > backed_map_size) {
+                err = vma_map(
+                    space,
+                    nullptr,
+                    map_address + backed_map_size,
+                    total_map_size - backed_map_size,
+                    prot,
+                    0,
+                    0
+                );
+                if (err)
+                    return err;
             }
 
-            // We map more than we copied so the rest is filled with zeroed pages.
-            err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
-            vmo_unref(&phdr_obj->object);
-            if (err)
-                return err;
+            const uintptr_t file_end = vaddr + phdr.p_filesz;
+            if (phdr.p_memsz > phdr.p_filesz && file_end % page_size != 0) {
+                const uintptr_t zero_end = MIN(ALIGN_UP(file_end, page_size), seg_end);
+                err = vma_zero(space, file_end, zero_end - file_end);
+                if (err)
+                    return err;
+            }
         }
     }
 
+    *out = (struct elf_image){
+        .base = base,
+        .entry = entry,
+        .phdr = phdr_addr,
+        .phnum = ehdr.e_phnum,
+        .phent = ehdr.e_phentsize,
+    };
+    return 0;
+}
+
+static errno_t elf_load(struct process* proc, struct exec_info* info, uintptr_t* out_ip, uintptr_t* out_sp) {
+    struct elf_image exe = {};
+    char* interp_path = nullptr;
+    errno_t err = elf_map(info->space, info->file_obj, ELF_DYN_BASE, &exe, &interp_path);
+
+    uintptr_t entry = exe.entry;
+    uintptr_t interp_base = 0;
+    if (!err && interp_path) {
+        struct vmo* interp;
+        err = exec_open(proc, interp_path, &interp);
+        if (!err) {
+            struct elf_image ld;
+            if (!elf_identify(interp))
+                err = ELIBBAD;
+            else
+                err = elf_map(info->space, interp, (uintptr_t)1 << (mem_high_shift() - 2), &ld, nullptr);
+            if (!err) {
+                entry = ld.entry;
+                interp_base = ld.base;
+            }
+            vmo_unref(interp);
+        }
+    }
+    mem_free(interp_path);
+    if (err)
+        return err;
+
+    const size_t page_size = arch_mem_page_size();
+
     const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
     const size_t stack_size = 2 * 1024 * 1024; // 2MiB
     const uintptr_t stack_start = highest - stack_size;
 
-    // Allocate stack.
-    struct paged_vmo* stack;
-    err = vmo_new_phys(&stack);
+    err = vma_map(info->space, nullptr, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0);
     if (err)
         return err;
 
     // Fill in stack info.
     uintptr_t stack_off = stack_size;
 
+    uintptr_t execfn = 0;
+    if (info->exec_path) {
+        const char nul = 0;
+        stack_off -= 1;
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
+
+        const size_t len = strlen(info->exec_path);
+        stack_off -= len;
+        vma_write(info->space, stack_start + stack_off, info->exec_path, len);
+        execfn = stack_start + stack_off;
+    }
+
     // TODO: Instead of allocating, calculate it on the fly.
     size_t num_envp = 0;
     for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
@@ -147,18 +266,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
         mem_free(envp_offsets);
         mem_free(argv_offsets);
-        vmo_unref(&stack->object);
         return ENOMEM;
     }
 
     for (size_t env = 0; env < num_envp; env++) {
         const char nul = 0;
         stack_off -= 1;
-        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
 
         const size_t len = strlen(info->envp[env]);
         stack_off -= len;
-        vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr);
+        vma_write(info->space, stack_start + stack_off, info->envp[env], len);
 
         envp_offsets[env] = stack_start + stack_off;
     }
@@ -166,11 +284,11 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     for (size_t arg = 0; arg < num_argv; arg++) {
         const char nul = 0;
         stack_off -= 1;
-        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
+        vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul));
 
         const size_t len = strlen(info->argv[arg]);
         stack_off -= len;
-        vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr);
+        vma_write(info->space, stack_start + stack_off, info->argv[arg], len);
 
         argv_offsets[arg] = stack_start + stack_off;
     }
@@ -181,43 +299,48 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     if ((1 + num_argv + num_envp) & 1) {
         stack_off -= sizeof(uintptr_t);
         uintptr_t zero = 0;
-        vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
+        vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
     }
 
 #define WRITE_AUXV(auxv, value) \
     do { \
         stack_off -= sizeof(uintptr_t); \
         uintptr_t auxv_val = value; \
-        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
+        vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \
         stack_off -= sizeof(uintptr_t); \
         auxv_val = auxv; \
-        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
+        vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \
     } while (0)
 
     // Write auxiliary values.
     WRITE_AUXV(AT_NULL, 0);
     WRITE_AUXV(AT_SECURE, 0);
-    WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
-    WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
-    WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
-    if (phdr_addr)
-        WRITE_AUXV(AT_PHDR, phdr_addr);
+    WRITE_AUXV(AT_PAGESZ, page_size);
+    WRITE_AUXV(AT_PHNUM, exe.phnum);
+    WRITE_AUXV(AT_PHENT, exe.phent);
+    WRITE_AUXV(AT_ENTRY, exe.entry);
+    if (exe.phdr)
+        WRITE_AUXV(AT_PHDR, exe.phdr);
+    if (interp_base)
+        WRITE_AUXV(AT_BASE, interp_base);
+    if (execfn)
+        WRITE_AUXV(AT_EXECFN, execfn);
 
     // envp pointers.
     stack_off -= sizeof(uintptr_t);
     const uintptr_t zero = 0;
-    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
+    vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
     for (size_t env = num_envp; env > 0; env--) {
         stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr);
+        vma_write(info->space, stack_start + stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]));
     }
 
     // argv pointers.
     stack_off -= sizeof(uintptr_t);
-    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
+    vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero));
     for (size_t arg = num_argv; arg > 0; arg--) {
         stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr);
+        vma_write(info->space, stack_start + stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]));
     }
 
     mem_free(envp_offsets);
@@ -226,27 +349,22 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
     // argc
     stack_off -= sizeof(uintptr_t);
     uintptr_t argc = num_argv;
-    vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
+    vma_write(info->space, stack_start + stack_off, &argc, sizeof(argc));
 
-    err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
-    vmo_unref(&stack->object);
-    if (err)
-        return err;
-
-    struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
-    if (!start_info)
-        return ENOMEM;
-    start_info->ip = ehdr.e_entry;
-    start_info->sp = highest - stack_size + stack_off;
-    start_info->arg = 0;
-
-    struct task* result;
-    err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
-    if (err) {
-        mem_free(start_info);
-        return err;
-    }
-
-    *out = result;
+    *out_ip = entry;
+    *out_sp = highest - stack_size + stack_off;
     return 0;
 }
+
+static struct exec_format elf_format = {
+    .name = "elf",
+    .identify = elf_identify,
+    .load = elf_load,
+};
+
+static void elf_stage_init(struct boot_info*) {
+    exec_register(&elf_format);
+}
+INIT_TASK(elf_stage, "fs.exec.elf", elf_stage_init);
+INIT_DEPENDS(elf_stage, mem_stage);
+INIT_ENTAILS(elf_stage, vfs_stage);

fs/exec/exec.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/exec.h>
+#include <zinnia/mutex.h>
+#include <zinnia/print.h>
+#include <zinnia/process.h>
+#include <zinnia/tailq.h>
+#include <zinnia/vfs.h>
+
+static mutex_t formats_lock;
+static tailq_head(struct exec_format) formats = {nullptr, &formats.tqh_first};
+
+struct exec_format* exec_identify(struct vmo* obj) {
+    mutex_lock(&formats_lock);
+    struct exec_format* format = tailq_first(&formats);
+    while (format != nullptr && !format->identify(obj))
+        format = tailq_next(format, link);
+    mutex_unlock(&formats_lock);
+    return format;
+}
+
+void exec_register(struct exec_format* format) {
+    mutex_lock(&formats_lock);
+    tailq_insert_tail(&formats, format, link);
+    mutex_unlock(&formats_lock);
+
+    log("exec: Registered new exec format \"%s\"\n", format->name);
+}
+
+errno_t exec_open(struct process* proc, const char* path, struct vmo** out) {
+    mutex_lock(&proc->lock);
+    struct vfs_path root = vfs_path_clone(proc->root);
+    struct vfs_path cwd = vfs_path_clone(proc->cwd);
+    mutex_unlock(&proc->lock);
+
+    struct vfs_file* file;
+    errno_t status = vfs_file_open(root, cwd, path, O_RDONLY | O_EXEC, 0, &proc->identity, &file);
+    vfs_path_drop(root);
+    vfs_path_drop(cwd);
+    if (status)
+        return status;
+
+    status = exec_file_object(proc, file, out);
+    vfs_file_unref(file);
+    return status;
+}
+
+errno_t exec_file_object(struct process* proc, struct vfs_file* file, struct vmo** out) {
+    errno_t status = vfs_node_check_exec(file->vnode, &proc->identity);
+    if (!status)
+        status = vfs_file_get_object(file, out);
+    return status;
+}

fs/exec/meson.build

@@ -1,3 +1,4 @@
 kernel_sources += files(
     'elf.c',
+    'exec.c',
 )

fs/file.c

@@ -0,0 +1,628 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/print.h>
+#include <zinnia/vfs.h>
+#include <stdckdint.h>
+#include <string.h>
+#include "internal.h"
+
+errno_t vfs_file_new(struct vfs_node* node, const struct vfs_path* path, enum open_flags flags, struct vfs_file** out) {
+    assert(node != nullptr, "Every open file must be backed by an inode");
+
+    struct vfs_file* file = mem_alloc(sizeof(*file), 0);
+    if (!file)
+        return ENOMEM;
+
+    refcount_init(&file->rc);
+    file->ops = node->file_ops;
+    file->vnode = node;
+    vfs_node_ref(node);
+    atomic_init(&file->flags, flags);
+    mutex_init(&file->lock);
+
+    switch (node->type) {
+    case VNODE_REGULAR:
+    case VNODE_DIRECTORY:
+        file->position_kind = VFS_POSITION_ATOMIC;
+        break;
+    default:
+        file->position_kind = VFS_POSITION_POSITION;
+        break;
+    }
+
+    if (path) {
+        file->path = vfs_path_clone(*path);
+        file->has_path = true;
+    }
+
+    *out = file;
+    return 0;
+}
+
+void vfs_stream_open(struct vfs_file* file) {
+    file->position_kind = VFS_POSITION_STREAM;
+}
+
+// Resolves the absolute path of `path` as seen from `root` into a fitting allocation.
+static char* make_absolute_path(struct vfs_path path, struct vfs_path* root) {
+    char* buf = mem_alloc(__PATH_MAX, ALLOC_NOZERO);
+    if (!buf)
+        return nullptr;
+
+    size_t len;
+    char* result = nullptr;
+    if (vfs_path_absolute(path, root, buf, __PATH_MAX, &len) == 0) {
+        result = mem_alloc(len + 1, ALLOC_NOZERO);
+        if (result)
+            memcpy(result, buf, len + 1);
+    }
+
+    mem_free(buf);
+    return result;
+}
+
+static errno_t open_special(
+    struct vfs_path* root,
+    struct vfs_path* file_path,
+    struct vfs_node* node,
+    enum open_flags flags,
+    struct vfs_file** out
+) {
+    if ((node->type == VNODE_CHARDEV || node->type == VNODE_BLOCKDEV) && (!node->ops || !node->ops->device.open))
+        return ENXIO;
+
+    struct vfs_file* file;
+    errno_t status = vfs_file_new(node, file_path, flags, &file);
+    if (status)
+        return status;
+    file->absolute_path = make_absolute_path(*file_path, root);
+
+    status = node->type == VNODE_FIFO ? vfs_fifo_open(node, file) : node->ops->device.open(node, file);
+    if (status) {
+        file->ops = nullptr;
+        vfs_file_unref(file);
+        return status;
+    }
+
+    *out = file;
+    return 0;
+}
+
+static enum access_flags open_access(enum open_flags flags) {
+    if (flags & O_PATH)
+        return F_OK;
+
+    enum access_flags access = F_OK;
+    if (flags & O_RDONLY)
+        access |= R_OK;
+    if (flags & (O_WRONLY | O_TRUNC))
+        access |= W_OK;
+    return access;
+}
+
+static errno_t do_open_inode(
+    struct vfs_path* root,
+    struct vfs_path* file_path,
+    struct vfs_node* node,
+    enum open_flags flags,
+    bool created,
+    const struct ident* identity,
+    struct vfs_file** out
+) {
+    // If we want to open as a directory, make sure this is actually a directory.
+    if ((flags & O_DIRECTORY) && node->type != VNODE_DIRECTORY)
+        return ENOTDIR;
+
+    if (node->type == VNODE_DIRECTORY && (flags & (O_WRONLY | O_CREAT | O_TRUNC)))
+        return EISDIR;
+
+    errno_t status = created ? 0 : vfs_node_try_access(node, identity, open_access(flags), false);
+    if (status)
+        return status;
+
+    switch (node->type) {
+    case VNODE_REGULAR: {
+        if (flags & O_TRUNC) {
+            if (!(flags & O_WRONLY))
+                return EINVAL;
+            if (!node->ops->regular.truncate)
+                return EPERM;
+            status = node->ops->regular.truncate(node, 0);
+            if (status)
+                return status;
+        }
+
+        struct vfs_file* file;
+        status = vfs_file_new(node, file_path, flags, &file);
+        if (status)
+            return status;
+        file->absolute_path = make_absolute_path(*file_path, root);
+        *out = file;
+        return 0;
+    }
+    case VNODE_DIRECTORY:
+        if (node->ops->dir.open)
+            return node->ops->dir.open(node, file_path, flags, identity, out);
+        return vfs_file_new(node, file_path, flags, out);
+    case VNODE_SYMLINK:
+        return ELOOP;
+    case VNODE_CHARDEV:
+    case VNODE_BLOCKDEV:
+    case VNODE_FIFO:
+        return open_special(root, file_path, node, flags, out);
+    case VNODE_SOCKET:
+        return ENXIO;
+    default:
+        return ENOTSUP;
+    }
+}
+
+errno_t vfs_file_open(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    enum open_flags flags,
+    mode_t mode,
+    const struct ident* identity,
+    struct vfs_file** out
+) {
+    if ((flags & O_DIRECTORY) && (flags & (O_CREAT | O_TMPFILE)))
+        return EINVAL;
+
+    const bool exclusive = (flags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL);
+
+    enum vfs_lookup_flags lookup_flags = 0;
+    // If we don't want to create a file when opening, the lookup must succeed.
+    // If we want to create a temporary unnamed file, `path` is the directory of the owning FS.
+    // In that case the path also has to exist.
+    if (!(flags & O_CREAT) || (flags & O_TMPFILE))
+        lookup_flags |= VFS_LOOKUP_MUST_EXIST;
+    if (exclusive)
+        lookup_flags |= VFS_LOOKUP_MUST_NOT_EXIST;
+    if (!(flags & O_NOFOLLOW) && !exclusive)
+        lookup_flags |= VFS_LOOKUP_FOLLOW_SYMLINKS;
+
+    struct vfs_path file_path;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, lookup_flags, &file_path);
+    if (status)
+        return status;
+
+    bool created = false;
+    struct vfs_node* node = vfs_entry_get_inode(file_path.entry);
+    if (!node) {
+        // If the lookup was successful, we expect that the entry is positive.
+        if (!(flags & O_CREAT)) {
+            log_warn("vfs: Tried opening a file without O_CREAT and backing inode, this is a bug!\n");
+            status = ENOENT;
+            goto out;
+        }
+
+        struct vfs_path parent_path;
+        status = vfs_path_lookup_parent(file_path, &parent_path);
+        if (status)
+            goto out;
+        struct vfs_node* parent = vfs_entry_get_inode(parent_path.entry);
+        vfs_path_drop(parent_path);
+        if (!parent) {
+            status = ENOENT;
+            goto out;
+        }
+
+        status = parent->type == VNODE_DIRECTORY ? 0 : ENOTDIR;
+        if (!status)
+            status = vfs_node_try_access(parent, identity, W_OK | X_OK, false);
+        if (!status)
+            status = parent->ops->dir.create ? parent->ops->dir.create(parent, file_path.entry, mode, identity) : EPERM;
+        vfs_node_unref(parent);
+        if (status)
+            goto out;
+
+        node = vfs_entry_get_inode(file_path.entry);
+        if (!node) {
+            status = EIO;
+            goto out;
+        }
+        created = true;
+    }
+
+    status = do_open_inode(&root, &file_path, node, flags, created, identity, out);
+    vfs_node_unref(node);
+
+out:
+    vfs_path_drop(file_path);
+    return status;
+}
+
+static errno_t advance_offset(off_t* offset, ssize_t amount) {
+    if (amount > 0 && ckd_add(offset, *offset, amount))
+        return EOVERFLOW;
+    return 0;
+}
+
+errno_t vfs_file_read(struct vfs_file* file, iovec_iter_t* buffer, ssize_t* out_read) {
+    *out_read = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (!file->ops || !file->ops->read)
+        return EINVAL;
+
+    errno_t status;
+    ssize_t read = 0;
+    switch (file->position_kind) {
+    case VFS_POSITION_STREAM:
+        status = file->ops->read(file, buffer, 0, &read);
+        break;
+    case VFS_POSITION_ATOMIC:
+        mutex_lock(&file->lock);
+        status = file->ops->read(file, buffer, file->position, &read);
+        if (!status)
+            status = advance_offset(&file->position, read);
+        mutex_unlock(&file->lock);
+        break;
+    case VFS_POSITION_POSITION: {
+        mutex_lock(&file->lock);
+        off_t pos = file->position;
+        mutex_unlock(&file->lock);
+
+        status = file->ops->read(file, buffer, pos, &read);
+        if (!status)
+            status = advance_offset(&pos, read);
+        if (!status) {
+            mutex_lock(&file->lock);
+            file->position = pos;
+            mutex_unlock(&file->lock);
+        }
+        break;
+    }
+    default:
+        return EINVAL;
+    }
+
+    if (status)
+        return status;
+    *out_read = read;
+    return 0;
+}
+
+errno_t vfs_file_pread(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    *out_read = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return ESPIPE;
+    if (!file->ops || !file->ops->read)
+        return EINVAL;
+
+    return file->ops->read(file, buffer, offset, out_read);
+}
+
+static errno_t check_write_progress(ssize_t written) {
+    if (written == 0) {
+        log_warn("vfs: write reported zero progress for a non-empty buffer\n");
+        return EIO;
+    }
+    return 0;
+}
+
+static errno_t sync_after_write(struct vfs_file* file, ssize_t written) {
+    if (written <= 0)
+        return 0;
+
+    if ((atomic_load(&file->flags, ATOMIC_RELAXED) & O_SYNC) == O_SYNC)
+        return vfs_file_sync(file, false);
+    if ((atomic_load(&file->flags, ATOMIC_RELAXED) & O_DSYNC) == O_DSYNC)
+        return vfs_file_sync(file, true);
+    return 0;
+}
+
+errno_t vfs_file_write(struct vfs_file* file, iovec_iter_t* buffer, ssize_t* out_written) {
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (!file->ops || !file->ops->write)
+        return EINVAL;
+
+    // TODO: Appending writes through different files of the same inode aren't serialized.
+    const bool append = atomic_load(&file->flags, ATOMIC_RELAXED) & O_APPEND;
+
+    errno_t status;
+    ssize_t written = 0;
+    switch (file->position_kind) {
+    case VFS_POSITION_STREAM:
+        status = file->ops->write(file, buffer, 0, &written);
+        break;
+    case VFS_POSITION_ATOMIC:
+        mutex_lock(&file->lock);
+        if (append)
+            file->position = file->vnode->size;
+        status = file->ops->write(file, buffer, file->position, &written);
+        if (!status)
+            status = advance_offset(&file->position, written);
+        mutex_unlock(&file->lock);
+        break;
+    case VFS_POSITION_POSITION: {
+        mutex_lock(&file->lock);
+        off_t pos = append ? (off_t)file->vnode->size : file->position;
+        mutex_unlock(&file->lock);
+
+        status = file->ops->write(file, buffer, pos, &written);
+        if (!status)
+            status = advance_offset(&pos, written);
+        if (!status) {
+            mutex_lock(&file->lock);
+            file->position = pos;
+            mutex_unlock(&file->lock);
+        }
+        break;
+    }
+    default:
+        return EINVAL;
+    }
+
+    if (!status)
+        status = check_write_progress(written);
+    if (!status)
+        status = sync_after_write(file, written);
+    if (status)
+        return status;
+
+    *out_written = written;
+    return 0;
+}
+
+errno_t vfs_file_pwrite(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return ESPIPE;
+    if (!file->ops || !file->ops->write)
+        return EINVAL;
+
+    ssize_t written = 0;
+    errno_t status = file->ops->write(file, buffer, offset, &written);
+    if (!status)
+        status = check_write_progress(written);
+    if (!status)
+        status = sync_after_write(file, written);
+    if (status)
+        return status;
+
+    *out_written = written;
+    return 0;
+}
+
+errno_t vfs_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    if (!file->ops || !file->ops->ioctl)
+        return ENOTTY;
+    return file->ops->ioctl(file, request, arg, out_result);
+}
+
+errno_t vfs_file_sync(struct vfs_file* file, bool data_only) {
+    if (!file->ops || !file->ops->sync)
+        return 0;
+    return file->ops->sync(file, data_only);
+}
+
+errno_t vfs_file_seek(struct vfs_file* file, enum vfs_seek_anchor anchor, off_t offset, off_t* out_position) {
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return ESPIPE;
+
+    errno_t status = 0;
+    mutex_lock(&file->lock);
+
+    off_t base;
+    switch (anchor) {
+    case VFS_SEEK_START:
+        base = 0;
+        break;
+    case VFS_SEEK_CURRENT:
+        base = file->position;
+        break;
+    case VFS_SEEK_END:
+        base = file->vnode->size;
+        break;
+    default:
+        status = EINVAL;
+        goto out;
+    }
+
+    off_t new_position;
+    if (ckd_add(&new_position, base, offset)) {
+        status = EOVERFLOW;
+        goto out;
+    }
+    if (new_position < 0) {
+        status = EINVAL;
+        goto out;
+    }
+
+    file->position = new_position;
+    if (out_position)
+        *out_position = new_position;
+
+out:
+    mutex_unlock(&file->lock);
+    return status;
+}
+
+off_t vfs_file_tell(struct vfs_file* file) {
+    mutex_lock(&file->lock);
+    off_t position = file->position;
+    mutex_unlock(&file->lock);
+    return position;
+}
+
+errno_t vfs_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    if (!file->ops || !file->ops->mmap)
+        return ENODEV;
+    return file->ops->mmap(file, vma, addr, len, prot, flags, offset, out_addr);
+}
+
+enum poll_flags vfs_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    const enum poll_flags always = POLLERR | POLLHUP | POLLNVAL;
+    if (!file->ops || !file->ops->poll)
+        return events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM);
+    return file->ops->poll(file, events | POLLERR | POLLHUP, table) & (events | always);
+}
+
+errno_t vfs_file_get_object(struct vfs_file* file, struct vmo** out) {
+    struct vfs_node* node = file->vnode;
+    if (node->type != VNODE_REGULAR || !node->ops || !node->ops->regular.get_vmo)
+        return ENODEV;
+    return node->ops->regular.get_vmo(node, out);
+}
+
+static uint8_t dirent_type(enum vfs_node_type type) {
+    switch (type) {
+    case VNODE_REGULAR:
+        return __DT_REG;
+    case VNODE_DIRECTORY:
+        return __DT_DIR;
+    case VNODE_SYMLINK:
+        return __DT_LNK;
+    case VNODE_FIFO:
+        return __DT_FIFO;
+    case VNODE_BLOCKDEV:
+        return __DT_BLK;
+    case VNODE_CHARDEV:
+        return __DT_CHR;
+    case VNODE_SOCKET:
+        return __DT_SOCK;
+    default:
+        // Should be unreachable, but just as a failsafe in case someone messes up.
+        return __DT_UNKNOWN;
+    }
+}
+
+errno_t vfs_dir_get_cached_entries(
+    struct vfs_node* node,
+    struct vfs_entry* entry,
+    off_t offset,
+    struct __dirent* buffer,
+    size_t buffer_len,
+    const struct ident* identity,
+    size_t* out_count
+) {
+    size_t read = 0;
+    off_t current = 0;
+
+    mutex_lock(&entry->lock);
+    for (struct vfs_entry* child = tailq_first(&entry->children); child && read < buffer_len;
+         child = tailq_next(child, child_link)) {
+        struct vfs_node* child_node = vfs_entry_get_inode(child);
+        if (!child_node)
+            continue;
+
+        if (current < offset) {
+            current++;
+            vfs_node_unref(child_node);
+            continue;
+        }
+
+        struct __dirent* d = &buffer[read];
+        d->d_ino = child_node->id;
+        d->d_off = current;
+        d->d_reclen = sizeof(*d);
+        d->d_type = dirent_type(child_node->type);
+        memcpy(d->d_name, child->name, child->name_len);
+        d->d_name[child->name_len] = '\0';
+        vfs_node_unref(child_node);
+
+        read++;
+        current++;
+    }
+    mutex_unlock(&entry->lock);
+
+    *out_count = read;
+    return 0;
+}
+
+static void fill_dot_entry(struct __dirent* d, ino_t ino, off_t off, const char* name) {
+    d->d_ino = ino;
+    d->d_off = off;
+    d->d_reclen = sizeof(*d);
+    d->d_type = __DT_DIR;
+    memset(d->d_name, 0, sizeof(d->d_name));
+    memcpy(d->d_name, name, strlen(name));
+}
+
+errno_t vfs_get_dir_entries(
+    struct vfs_file* file,
+    struct __dirent* buffer,
+    size_t buffer_len,
+    const struct ident* identity,
+    size_t* out_count
+) {
+    *out_count = 0;
+    if (buffer_len == 0)
+        return 0;
+
+    struct vfs_node* node = file->vnode;
+    if (!file->has_path || node->type != VNODE_DIRECTORY)
+        return ENOTDIR;
+    if (file->position_kind == VFS_POSITION_STREAM)
+        return EBADF;
+
+    ino_t parent_ino = node->id;
+    struct vfs_path parent;
+    if (vfs_path_lookup_parent(file->path, &parent) == 0) {
+        struct vfs_node* parent_node = vfs_entry_get_inode(parent.entry);
+        if (parent_node) {
+            parent_ino = parent_node->id;
+            vfs_node_unref(parent_node);
+        }
+        vfs_path_drop(parent);
+    }
+
+    errno_t status = 0;
+    mutex_lock(&file->lock);
+
+    const off_t start_offset = file->position;
+    size_t num_read = 0;
+
+    // Read `.` and `..` entries.
+    if (start_offset == 0)
+        fill_dot_entry(&buffer[num_read++], node->id, 0, ".");
+    if ((start_offset == 1 || num_read == 1) && num_read < buffer_len)
+        fill_dot_entry(&buffer[num_read++], parent_ino, 1, "..");
+
+    // Read normal dir entries.
+    if (num_read < buffer_len && node->ops->dir.get_dir_entries) {
+        size_t count = 0;
+        status = node->ops->dir.get_dir_entries(
+            node,
+            file->path.entry,
+            start_offset > 2 ? start_offset - 2 : 0,
+            buffer + num_read,
+            buffer_len - num_read,
+            identity,
+            &count
+        );
+        if (status)
+            goto out;
+        num_read += count;
+    }
+
+    file->position += num_read;
+    for (size_t i = 0; i < num_read; i++)
+        buffer[i].d_off = start_offset + i + 1;
+
+    *out_count = num_read;
+
+out:
+    mutex_unlock(&file->lock);
+    return status;
+}

fs/ident.c

@@ -15,3 +15,7 @@ bool ident_in_group(const struct ident* identity, gid_t gid, bool use_real) {
     }
     return false;
 }
+
+bool ident_is_privileged(const struct ident* identity, bool use_real) {
+    return (use_real ? identity->uid : identity->euid) == 0;
+}

fs/init.c

@@ -1,3 +1,21 @@
-#include <zinnia/vfs.h>
+// SPDX-License-Identifier: ISC
 
-void vfs_init() {}
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/vfs.h>
+#include "internal.h"
+
+static void vfs_stage_init(struct boot_info* info) {
+    // Mount a tmpfs as root.
+    struct vfs_mount* tmpfs;
+    errno_t status = vfs_mount("tmpfs", 0, nullptr, &tmpfs);
+    assert(status == 0, "Unable to mount the root tmpfs (%i)", status);
+
+    struct vfs_path root = {.mount = tmpfs, .entry = tmpfs->root};
+    vfs_set_root(root);
+    vfs_mount_unref(tmpfs);
+}
+
+INIT_TASK(vfs_stage, "fs.vfs", vfs_stage_init);
+INIT_DEPENDS(vfs_stage, mem_stage);

fs/initramfs.c

@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/errno.h>
+#include <zinnia/init.h>
+#include <zinnia/initramfs.h>
+#include <zinnia/print.h>
+#include <zinnia/vfs.h>
+#include <uapi/limits.h>
+#include <uapi/stat.h>
+
+enum ustar_type : uint8_t {
+    USTAR_REGULAR = 0,
+    USTAR_NORMAL = '0',
+    USTAR_SYM_LINK = '2',
+    USTAR_DIRECTORY = '5',
+    USTAR_CONTIGUOUS = '7',
+    USTAR_LONG_LINK = 'L',
+};
+
+struct ustar_header {
+    char name[100];
+    char mode[8];
+    char uid[8];
+    char gid[8];
+    char size[12];
+    char mtime[12];
+    char checksum[8];
+    enum ustar_type type;
+    char linkname[100];
+    char signature[6];
+    char version[2];
+    char owner[32];
+    char group[32];
+    char devmajor[8];
+    char devminor[8];
+    char prefix[155];
+    char pad[12];
+};
+
+static_assert(sizeof(struct ustar_header) == 512, "USTAR header must be 512 bytes");
+
+static size_t oct2bin(const char* str, size_t len) {
+    size_t value = 0;
+    for (size_t i = 0; i < len; i++) {
+        if (str[i] == '\0' || str[i] == ' ')
+            break;
+        value *= 8;
+        value += (size_t)(str[i] - '0');
+    }
+    return value;
+}
+
+static void copy_field(char* out, const char* field, size_t field_len, size_t out_len) {
+    size_t i = 0;
+    while (i < field_len && field[i] != '\0' && i + 1 < out_len) {
+        out[i] = field[i];
+        i++;
+    }
+    out[i] = '\0';
+}
+
+static errno_t create_dirs(
+    struct vfs_path root,
+    struct vfs_path at,
+    const char* path,
+    mode_t mode,
+    struct vfs_path* out_dir,
+    char* name_out,
+    size_t name_out_len
+) {
+    size_t path_len = strlen(path);
+
+    size_t dir_len = 0;
+    const char* name = path;
+    for (size_t i = path_len; i > 0; i--) {
+        if (path[i - 1] == '/') {
+            dir_len = i - 1;
+            name = path + i;
+            break;
+        }
+    }
+
+    strncpy(name_out, name, name_out_len - 1);
+    name_out[name_out_len - 1] = '\0';
+
+    struct vfs_path current = vfs_path_clone(at);
+
+    size_t index = 0;
+    while (index < dir_len) {
+        while (index < dir_len && path[index] == '/')
+            index++;
+        if (index >= dir_len)
+            break;
+
+        size_t start = index;
+        while (index < dir_len && path[index] != '/')
+            index++;
+
+        char component[__NAME_MAX + 1];
+        size_t length = MIN(index - start, __NAME_MAX);
+        memcpy(component, path + start, length);
+        component[length] = '\0';
+
+        errno_t status = vfs_mkdir(root, current, component, mode, ident_kernel());
+        if (status && status != EEXIST) {
+            vfs_path_drop(current);
+            return status;
+        }
+
+        struct vfs_file* dir;
+        status = vfs_file_open(root, current, component, O_DIRECTORY, 0, ident_kernel(), &dir);
+        if (status) {
+            vfs_path_drop(current);
+            return status;
+        }
+
+        struct vfs_path next = vfs_path_clone(dir->path);
+        vfs_file_unref(dir);
+        vfs_path_drop(current);
+        current = next;
+    }
+
+    *out_dir = current;
+    return 0;
+}
+
+bool initramfs_is_valid(const void* data, size_t len) {
+    const struct ustar_header* header = data;
+    return len >= sizeof(*header) && !memcmp(header->signature, "ustar", 5);
+}
+
+// Sets all timestamps of the node behind `file` to `mtime`.
+static void set_times(struct vfs_file* file, size_t mtime) {
+    struct vfs_node* node = file->vnode;
+    timespec_t time = time_from_sec(mtime);
+
+    mutex_lock(&node->lock);
+    node->atime = time;
+    node->mtime = time;
+    node->ctime = time;
+    mutex_unlock(&node->lock);
+}
+
+errno_t initramfs_load(struct vfs_path root, struct vfs_path target, const void* data, size_t len) {
+    const uint8_t* base = data;
+    size_t offset = 0;
+    size_t files_loaded = 0;
+    errno_t status = 0;
+
+    // Paths can be much longer than what fits on the stack.
+    char* override = mem_alloc(__PATH_MAX + 1, ALLOC_NOZERO);
+    char* name = mem_alloc(__PATH_MAX + 1, ALLOC_NOZERO);
+    if (!override || !name) {
+        status = ENOMEM;
+        goto out;
+    }
+    bool has_override = false;
+
+    while (offset + sizeof(struct ustar_header) <= len) {
+        const struct ustar_header* header = (const struct ustar_header*)(base + offset);
+        if (memcmp(header->signature, "ustar", 5) != 0)
+            break;
+
+        mode_t file_mode = (mode_t)oct2bin(header->mode, sizeof(header->mode));
+        size_t file_size = oct2bin(header->size, sizeof(header->size));
+        size_t mtime = oct2bin(header->mtime, sizeof(header->mtime));
+
+        size_t data_offset = offset + sizeof(struct ustar_header);
+        if (file_size > len - data_offset) {
+            log_warn("vfs: Truncated initramfs entry at offset %zu\n", offset);
+            status = EINVAL;
+            goto out;
+        }
+
+        if (has_override) {
+            memcpy(name, override, __PATH_MAX + 1);
+            has_override = false;
+        } else {
+            copy_field(name, header->name, sizeof(header->name), __PATH_MAX + 1);
+        }
+
+        // Directory entries usually end with a slash, which would leave us with an empty file name.
+        size_t name_len = strlen(name);
+        while (name_len > 1 && name[name_len - 1] == '/')
+            name[--name_len] = '\0';
+
+        struct vfs_path dir;
+        char file_name[__NAME_MAX + 1];
+
+        switch (header->type) {
+        case USTAR_REGULAR:
+        case USTAR_NORMAL:
+        case USTAR_CONTIGUOUS: {
+            status = create_dirs(root, target, name, file_mode, &dir, file_name, sizeof(file_name));
+            if (status)
+                goto out;
+
+            struct vfs_file* file;
+            status = vfs_file_open(root, dir, file_name, O_CREAT, file_mode, ident_kernel(), &file);
+            vfs_path_drop(dir);
+            if (status)
+                goto out;
+
+            if (file_size > 0) {
+                iovec_t vec = {
+                    .base = (uintptr_t)(base + data_offset),
+                    .len = file_size,
+                };
+                iovec_iter_t iter = iovec_iter_kernel(&vec, 1);
+                ssize_t written = 0;
+                status = vfs_file_pwrite(file, &iter, 0, &written);
+                if (status) {
+                    vfs_file_unref(file);
+                    goto out;
+                }
+            }
+
+            set_times(file, mtime);
+            vfs_file_unref(file);
+            files_loaded++;
+            break;
+        }
+        case USTAR_SYM_LINK: {
+            status = create_dirs(root, target, name, file_mode, &dir, file_name, sizeof(file_name));
+            if (status)
+                goto out;
+
+            char link_target[101];
+            copy_field(link_target, header->linkname, sizeof(header->linkname), sizeof(link_target));
+
+            status = vfs_symlink(root, dir, file_name, link_target, ident_kernel());
+            vfs_path_drop(dir);
+            if (status)
+                goto out;
+
+            files_loaded++;
+            break;
+        }
+        case USTAR_DIRECTORY: {
+            status = create_dirs(root, target, name, file_mode, &dir, file_name, sizeof(file_name));
+            if (status)
+                goto out;
+
+            status = vfs_mkdir(root, dir, file_name, file_mode, ident_kernel());
+            vfs_path_drop(dir);
+            if (status && status != EEXIST)
+                goto out;
+            status = 0;
+
+            files_loaded++;
+            break;
+        }
+        case USTAR_LONG_LINK: {
+            size_t copy_len = MIN(file_size, __PATH_MAX);
+            memcpy(override, base + data_offset, copy_len);
+            override[copy_len] = '\0';
+            has_override = true;
+            break;
+        }
+        default:
+            break;
+        }
+
+        offset += sizeof(struct ustar_header) + ALIGN_UP(file_size, 512);
+    }
+
+    log("vfs: Loaded %zu files from initramfs\n", files_loaded);
+
+out:
+    mem_free(override);
+    mem_free(name);
+    return status;
+}
+
+void initramfs_load_boot_modules(struct boot_info* info) {
+    struct vfs_path root = vfs_get_root();
+    size_t ramdisk_index = 0;
+
+    for (size_t i = 0; i < info->num_files; i++) {
+        struct boot_file* file = &info->files[i];
+        const void* data = HHDM_PTR(file->data);
+
+        if (!initramfs_is_valid(data, file->length)) {
+            char name[16];
+            ksnprintf(name, sizeof(name), "ram%zu", ramdisk_index);
+            struct blockdev* disk;
+            errno_t status = ramdisk_new(file->data, file->length, &disk);
+            if (!status) {
+                status = blockdev_register(name, disk);
+                blockdev_unref(disk);
+            }
+            if (status) {
+                log_warn("initramfs: Unable to expose module \"%s\" as a RAM disk (%i)\n", file->path, status);
+                continue;
+            }
+            log("initramfs: Exposed module \"%s\" as RAM disk /dev/block/%s\n", file->path, name);
+            ramdisk_index++;
+            continue;
+        }
+
+        errno_t status = initramfs_load(root, root, data, file->length);
+        assert(status == 0, "Failed to load initramfs \"%s\" (%i)", file->path, status);
+    }
+
+    vfs_path_drop(root);
+}

fs/internal.h

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/vfs.h>
+
+// Sets the global root of the VFS. Takes a new reference to `root`.
+void vfs_set_root(struct vfs_path root);
+
+// Returns true if both paths reference the same entry on the same mount.
+static inline bool vfs_path_equal(struct vfs_path a, struct vfs_path b) {
+    return a.mount == b.mount && a.entry == b.entry;
+}
+
+void pipe_destroy(struct pipe* pipe);
+errno_t vfs_fifo_open(struct vfs_node* node, struct vfs_file* file);

fs/meson.build

@@ -1,6 +1,17 @@
-subdir('exec')
-
 kernel_sources += files(
+    'anon.c',
+    'cache.c',
+    'devtmpfs.c',
+    'file.c',
     'ident.c',
     'init.c',
+    'initramfs.c',
+    'mount.c',
+    'ops.c',
+    'pipe.c',
+    'poll.c',
+    'refs.c',
+    'tmpfs.c',
 )
+
+subdir('exec')

fs/mount.c

@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/vfs.h>
+#include <string.h>
+#include "internal.h"
+
+// All known and registered file systems.
+static struct spinlock fs_table_lock;
+static vec(const struct vfs_filesystem*) fs_table;
+
+// Super blocks of all mounted file systems. Each holds a reference.
+static struct spinlock supers_lock;
+static vec(struct vfs_super_block*) mounted_supers;
+
+// The global root.
+static struct vfs_path vfs_root;
+static bool vfs_root_set;
+
+void vfs_set_root(struct vfs_path root) {
+    assert(!vfs_root_set, "The VFS root can only be set once");
+    vfs_root = vfs_path_clone(root);
+    vfs_root_set = true;
+}
+
+struct vfs_path vfs_get_root() {
+    assert(vfs_root_set, "The VFS root hasn't been set yet");
+    return vfs_path_clone(vfs_root);
+}
+
+errno_t vfs_register_filesystem(const struct vfs_filesystem* fs) {
+    errno_t status = 0;
+
+    spin_lock(&fs_table_lock);
+    vec_foreach(&fs_table, i) {
+        if (!strcmp(vec_elem_get(&fs_table, i)->name, fs->name)) {
+            status = EEXIST;
+            goto out;
+        }
+    }
+    status = vec_push(&fs_table, fs);
+
+out:
+    spin_unlock(&fs_table_lock);
+    if (!status)
+        log("vfs: Registered new file system \"%s\"\n", fs->name);
+    return status;
+}
+
+// Returns a new reference to the super block of `mount`, if it has one.
+static struct vfs_super_block* mount_get_super(struct vfs_mount* mount) {
+    struct vfs_node* root = vfs_entry_get_inode(mount->root);
+    if (!root)
+        return nullptr;
+
+    struct vfs_super_block* sb = root->sb;
+    if (sb)
+        vfs_super_block_ref(sb);
+    vfs_node_unref(root);
+    return sb;
+}
+
+errno_t vfs_mount(const char* fs_name, enum mount_flags flags, void* arg, struct vfs_mount** out) {
+    const struct vfs_filesystem* fs = nullptr;
+
+    spin_lock(&fs_table_lock);
+    vec_foreach(&fs_table, i) {
+        if (!strcmp(vec_elem_get(&fs_table, i)->name, fs_name)) {
+            fs = vec_elem_get(&fs_table, i);
+            break;
+        }
+    }
+    spin_unlock(&fs_table_lock);
+
+    if (!fs) {
+        log_warn("Attempted to mount unknown file system type \"%s\"\n", fs_name);
+        return ENODEV;
+    }
+
+    struct vfs_mount* mount;
+    errno_t status = fs->mount(fs, flags, arg, &mount);
+    if (status)
+        return status;
+
+    struct vfs_super_block* sb = mount_get_super(mount);
+    if (sb) {
+        bool known = false;
+        spin_lock(&supers_lock);
+        vec_foreach(&mounted_supers, i) {
+            if (vec_elem_get(&mounted_supers, i) == sb) {
+                known = true;
+                break;
+            }
+        }
+        // The list takes over our reference.
+        if (!known && vec_push(&mounted_supers, sb) == 0)
+            sb = nullptr;
+        spin_unlock(&supers_lock);
+
+        if (sb)
+            vfs_super_block_unref(sb);
+    }
+
+    *out = mount;
+    return 0;
+}
+
+errno_t vfs_unmount(struct vfs_mount* mount) {
+    struct vfs_super_block* sb = mount_get_super(mount);
+    if (!sb)
+        return 0;
+
+    spin_lock(&supers_lock);
+    vec_foreach(&mounted_supers, i) {
+        if (vec_elem_get(&mounted_supers, i) == sb) {
+            vec_swap_remove(&mounted_supers, i);
+            vfs_super_block_unref(sb);
+            break;
+        }
+    }
+    spin_unlock(&supers_lock);
+
+    errno_t status = sb->ops && sb->ops->sync ? sb->ops->sync(sb) : 0;
+    vfs_super_block_unref(sb);
+    return status;
+}
+
+errno_t vfs_sync_all() {
+    // Take a snapshot so we don't call into file systems while holding a spinlock.
+    spin_lock(&supers_lock);
+    size_t count = vec_length(&mounted_supers);
+    struct vfs_super_block** supers = count ? mem_alloc(count * sizeof(*supers), ALLOC_NOZERO) : nullptr;
+    if (count && !supers) {
+        spin_unlock(&supers_lock);
+        return ENOMEM;
+    }
+    for (size_t i = 0; i < count; i++) {
+        supers[i] = vec_elem_get(&mounted_supers, i);
+        vfs_super_block_ref(supers[i]);
+    }
+    spin_unlock(&supers_lock);
+
+    errno_t result = 0;
+    for (size_t i = 0; i < count; i++) {
+        struct vfs_super_block* sb = supers[i];
+        if (sb->ops && sb->ops->sync) {
+            errno_t status = sb->ops->sync(sb);
+            if (status)
+                result = status;
+        }
+        vfs_super_block_unref(sb);
+    }
+
+    mem_free(supers);
+    return result;
+}
+
+errno_t vfs_mount_new(enum mount_flags flags, struct vfs_entry* root, struct vfs_mount** out) {
+    struct vfs_mount* mount = mem_alloc(sizeof(*mount), 0);
+    if (!mount)
+        return ENOMEM;
+
+    refcount_init(&mount->rc);
+    mount->flags = flags;
+    mount->root = root;
+    vfs_entry_ref(root);
+    mount->has_mount_point = false;
+
+    *out = mount;
+    return 0;
+}
+
+errno_t vfs_path_mount(struct vfs_path at, struct vfs_mount* mount) {
+    if (mount->has_mount_point)
+        return EBUSY;
+
+    mutex_lock(&at.entry->lock);
+    errno_t status = vec_push(&at.entry->mounts, mount);
+    mutex_unlock(&at.entry->lock);
+    if (status)
+        return status;
+
+    vfs_mount_ref(mount);
+    mount->mount_point = vfs_path_clone(at);
+    mount->has_mount_point = true;
+    return 0;
+}
+
+errno_t vfs_path_unmount(struct vfs_path path) {
+    struct vfs_mount* mount = path.mount;
+    if (path.entry != mount->root || !mount->has_mount_point)
+        return EINVAL;
+
+    vfs_entry_prune(mount->root);
+    mutex_lock(&mount->root->lock);
+    const bool busy = !tailq_empty(&mount->root->children);
+    mutex_unlock(&mount->root->lock);
+    if (busy)
+        return EBUSY;
+
+    struct vfs_entry* point = mount->mount_point.entry;
+    bool found = false;
+    mutex_lock(&point->lock);
+    vec_foreach(&point->mounts, i) {
+        if (vec_elem_get(&point->mounts, i) == mount) {
+            vec_remove(&point->mounts, i);
+            found = true;
+            break;
+        }
+    }
+    mutex_unlock(&point->lock);
+    if (!found)
+        return EINVAL;
+
+    struct vfs_path mount_point = mount->mount_point;
+    mount->mount_point = (struct vfs_path){};
+    mount->has_mount_point = false;
+    vfs_path_drop(mount_point);
+
+    errno_t status = vfs_unmount(mount);
+    vfs_mount_unref(mount);
+    return status;
+}

fs/ops.c

@@ -0,0 +1,656 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/clock.h>
+#include <zinnia/vfs.h>
+#include <string.h>
+
+static bool is_read_only(struct vfs_node* node) {
+    return node->sb && (node->sb->flags & MNT_RDONLY);
+}
+
+errno_t vfs_node_try_access(
+    struct vfs_node* node,
+    const struct ident* identity,
+    enum access_flags access,
+    bool use_real
+) {
+    mutex_lock(&node->lock);
+    const mode_t mode = node->mode;
+    const uid_t owner = node->uid;
+    const gid_t group = node->gid;
+    mutex_unlock(&node->lock);
+
+    if ((access & W_OK) && is_read_only(node) &&
+        (node->type == VNODE_REGULAR || node->type == VNODE_DIRECTORY || node->type == VNODE_SYMLINK))
+        return EROFS;
+    if ((access & X_OK) && node->type == VNODE_REGULAR && node->sb && (node->sb->flags & MNT_NOEXEC))
+        return EACCES;
+
+    if (ident_is_privileged(identity, use_real)) {
+        if ((access & X_OK) && node->type != VNODE_DIRECTORY && !(mode & (__S_IXUSR | __S_IXGRP | __S_IXOTH)))
+            return EACCES;
+        return 0;
+    }
+
+    mode_t granted;
+    if ((use_real ? identity->uid : identity->euid) == owner)
+        granted = (mode >> 6) & 7;
+    else if (ident_in_group(identity, group, use_real))
+        granted = (mode >> 3) & 7;
+    else
+        granted = mode & 7;
+
+    return (access & (R_OK | W_OK | X_OK) & ~granted) ? EACCES : 0;
+}
+
+static errno_t sync_node(struct vfs_node* node) {
+    return node->ops && node->ops->sync ? node->ops->sync(node) : 0;
+}
+
+static timespec_t now() {
+    return time_from_nano(clock_get_realtime_ns());
+}
+
+// Looks up the parent directory of `path` and checks that `identity` may write to it.
+// Returns a new reference to the parent inode in `out`.
+static errno_t get_writable_parent(struct vfs_path path, const struct ident* identity, struct vfs_node** out) {
+    struct vfs_path parent;
+    errno_t status = vfs_path_lookup_parent(path, &parent);
+    if (status)
+        return status;
+
+    struct vfs_node* node = vfs_entry_get_inode(parent.entry);
+    vfs_path_drop(parent);
+    if (!node)
+        return ENOENT;
+
+    if (node->type != VNODE_DIRECTORY) {
+        vfs_node_unref(node);
+        return ENOTDIR;
+    }
+
+    status = vfs_node_try_access(node, identity, W_OK | X_OK, false);
+    if (status) {
+        vfs_node_unref(node);
+        return status;
+    }
+
+    *out = node;
+    return 0;
+}
+
+errno_t vfs_mkdir(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    mode_t mode,
+    const struct ident* identity
+) {
+    struct vfs_path target;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, VFS_LOOKUP_MUST_NOT_EXIST, &target);
+    if (status)
+        return status;
+
+    struct vfs_node* parent;
+    status = get_writable_parent(target, identity, &parent);
+    if (status)
+        goto out;
+
+    status = parent->ops->dir.mkdir ? parent->ops->dir.mkdir(parent, &target, mode, identity) : EPERM;
+    vfs_node_unref(parent);
+
+out:
+    vfs_path_drop(target);
+    return status;
+}
+
+errno_t vfs_symlink(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    const char* target_path,
+    const struct ident* identity
+) {
+    struct vfs_path target;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, VFS_LOOKUP_MUST_NOT_EXIST, &target);
+    if (status)
+        return status;
+
+    struct vfs_node* parent;
+    status = get_writable_parent(target, identity, &parent);
+    if (status)
+        goto out;
+
+    status = parent->ops->dir.symlink
+        ? parent->ops->dir.symlink(parent, &target, target_path, strlen(target_path), identity)
+        : EPERM;
+    vfs_node_unref(parent);
+
+out:
+    vfs_path_drop(target);
+    return status;
+}
+
+static bool last_component_is(const char* path, const char* name) {
+    size_t len = strlen(path);
+    while (len > 1 && path[len - 1] == '/')
+        len--;
+    size_t begin = len;
+    while (begin > 0 && path[begin - 1] != '/')
+        begin--;
+    return len - begin == strlen(name) && !memcmp(path + begin, name, len - begin);
+}
+
+static errno_t check_sticky(struct vfs_node* dir, struct vfs_node* victim, const struct ident* identity) {
+    if (identity->euid == 0)
+        return 0;
+
+    mutex_lock(&dir->lock);
+    const bool sticky = dir->mode & __S_ISVTX;
+    const uid_t dir_owner = dir->uid;
+    mutex_unlock(&dir->lock);
+    if (!sticky)
+        return 0;
+
+    mutex_lock(&victim->lock);
+    const uid_t owner = victim->uid;
+    mutex_unlock(&victim->lock);
+    return identity->euid == owner || identity->euid == dir_owner ? 0 : EPERM;
+}
+
+errno_t vfs_unlink(struct vfs_path root, struct vfs_path cwd, const char* path, const struct ident* identity) {
+    struct vfs_path target;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, VFS_LOOKUP_MUST_EXIST, &target);
+    if (status)
+        return status;
+
+    struct vfs_node* node = vfs_entry_get_inode(target.entry);
+    if (!node) {
+        status = ENOENT;
+        goto out;
+    }
+    if (node->type == VNODE_DIRECTORY) {
+        status = EISDIR;
+        goto out_node;
+    }
+
+    struct vfs_node* parent;
+    status = get_writable_parent(target, identity, &parent);
+    if (status)
+        goto out_node;
+
+    status = check_sticky(parent, node, identity);
+    if (!status)
+        status = parent->ops->dir.unlink ? parent->ops->dir.unlink(parent, &target, identity) : EPERM;
+    vfs_node_unref(parent);
+
+out_node:
+    vfs_node_unref(node);
+out:
+    vfs_path_drop(target);
+    return status;
+}
+
+// Probe for a single entry to check if the directory is empty.
+static errno_t check_dir_empty(struct vfs_node* node, struct vfs_entry* entry, const struct ident* identity) {
+    struct __dirent probe;
+    size_t count = 0;
+    if (node->ops->dir.get_dir_entries) {
+        errno_t status = node->ops->dir.get_dir_entries(node, entry, 0, &probe, 1, identity, &count);
+        if (status)
+            return status;
+    }
+    return count > 0 ? ENOTEMPTY : 0;
+}
+
+errno_t vfs_rmdir(struct vfs_path root, struct vfs_path cwd, const char* path, const struct ident* identity) {
+    if (last_component_is(path, "."))
+        return EINVAL;
+    if (last_component_is(path, ".."))
+        return ENOTEMPTY;
+
+    struct vfs_path target;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, VFS_LOOKUP_MUST_EXIST, &target);
+    if (status)
+        return status;
+
+    struct vfs_node* node = vfs_entry_get_inode(target.entry);
+    if (!node) {
+        status = ENOENT;
+        goto out;
+    }
+    if (node->type != VNODE_DIRECTORY) {
+        status = ENOTDIR;
+        goto out_node;
+    }
+    if (target.entry == target.mount->root) {
+        status = EBUSY;
+        goto out_node;
+    }
+
+    status = check_dir_empty(node, target.entry, identity);
+    if (status)
+        goto out_node;
+
+    struct vfs_node* parent;
+    status = get_writable_parent(target, identity, &parent);
+    if (status)
+        goto out_node;
+
+    status = check_sticky(parent, node, identity);
+    if (!status) {
+        if (parent->ops->dir.rmdir)
+            status = parent->ops->dir.rmdir(parent, &target, identity);
+        else if (parent->ops->dir.unlink)
+            status = parent->ops->dir.unlink(parent, &target, identity);
+        else
+            status = EPERM;
+    }
+    vfs_node_unref(parent);
+
+out_node:
+    vfs_node_unref(node);
+out:
+    vfs_path_drop(target);
+    return status;
+}
+
+errno_t vfs_mknod(
+    struct vfs_path root,
+    struct vfs_path cwd,
+    const char* path,
+    mode_t mode,
+    struct vfs_mknod_target* target,
+    const struct ident* identity,
+    struct vfs_node** out
+) {
+    struct vfs_path new_path;
+    errno_t status = vfs_path_lookup(root, cwd, path, identity, VFS_LOOKUP_MUST_NOT_EXIST, &new_path);
+    if (status)
+        return status;
+
+    struct vfs_node* parent;
+    status = get_writable_parent(new_path, identity, &parent);
+    if (status)
+        goto out;
+
+    status = parent->ops->dir.mknod ? parent->ops->dir.mknod(parent, &new_path, mode, target, identity) : ENODEV;
+    vfs_node_unref(parent);
+    if (!status && out) {
+        *out = vfs_entry_get_inode(new_path.entry);
+        if (!*out)
+            status = ENOENT;
+    }
+
+out:
+    vfs_path_drop(new_path);
+    return status;
+}
+
+static bool entry_is_ancestor(struct vfs_entry* ancestor, struct vfs_entry* entry) {
+    for (struct vfs_entry* it = entry; it; it = it->parent) {
+        if (it == ancestor)
+            return true;
+    }
+    return false;
+}
+
+static bool entry_points_to(struct vfs_entry* entry, struct vfs_node* node) {
+    struct vfs_node* current = vfs_entry_get_inode(entry);
+    if (current)
+        vfs_node_unref(current);
+    return current == node;
+}
+
+static errno_t check_rename_target(struct vfs_node* moved, struct vfs_path target, const struct ident* identity) {
+    struct vfs_node* existing = vfs_entry_get_inode(target.entry);
+    if (!existing)
+        return 0;
+
+    errno_t status = 0;
+    if (moved->type == VNODE_DIRECTORY && existing->type != VNODE_DIRECTORY)
+        status = ENOTDIR;
+    else if (moved->type != VNODE_DIRECTORY && existing->type == VNODE_DIRECTORY)
+        status = EISDIR;
+    else if (existing->type == VNODE_DIRECTORY)
+        status = check_dir_empty(existing, target.entry, identity);
+
+    vfs_node_unref(existing);
+    return status;
+}
+
+static errno_t check_rename_perms(
+    struct vfs_node* moved,
+    struct vfs_path new,
+    struct vfs_node* old_parent,
+    struct vfs_node* new_parent,
+    const struct ident* identity
+) {
+    errno_t status = check_sticky(old_parent, moved, identity);
+    if (status)
+        return status;
+
+    struct vfs_node* existing = vfs_entry_get_inode(new.entry);
+    if (existing) {
+        status = check_sticky(new_parent, existing, identity);
+        vfs_node_unref(existing);
+        if (status)
+            return status;
+    }
+
+    if (moved->type == VNODE_DIRECTORY && old_parent != new_parent)
+        return vfs_node_try_access(moved, identity, W_OK, false);
+    return 0;
+}
+
+static errno_t do_rename(
+    struct vfs_path old,
+    struct vfs_path new,
+    struct vfs_node* moved,
+    const struct ident* identity
+) {
+    struct vfs_node* old_parent;
+    errno_t status = get_writable_parent(old, identity, &old_parent);
+    if (status)
+        return status;
+
+    struct vfs_node* new_parent;
+    status = get_writable_parent(new, identity, &new_parent);
+    if (status) {
+        vfs_node_unref(old_parent);
+        return status;
+    }
+
+    status = check_rename_perms(moved, new, old_parent, new_parent, identity);
+    if (!status) {
+        if (old_parent->ops->dir.rename)
+            status = old_parent->ops->dir.rename(old_parent, &old, new_parent, &new, identity);
+        else
+            status = EPERM;
+    }
+    vfs_node_unref(new_parent);
+    vfs_node_unref(old_parent);
+    return status;
+}
+
+errno_t vfs_rename(
+    struct vfs_path root,
+    struct vfs_path old_cwd,
+    const char* old_path,
+    struct vfs_path new_cwd,
+    const char* new_path,
+    const struct ident* identity
+) {
+    if (last_component_is(old_path, ".") || last_component_is(old_path, "..") || last_component_is(new_path, ".") ||
+        last_component_is(new_path, ".."))
+        return EINVAL;
+
+    struct vfs_path old;
+    errno_t status = vfs_path_lookup(root, old_cwd, old_path, identity, VFS_LOOKUP_MUST_EXIST, &old);
+    if (status)
+        return status;
+
+    struct vfs_path new;
+    status = vfs_path_lookup(root, new_cwd, new_path, identity, 0, &new);
+    if (status)
+        goto out_old;
+
+    if (old.mount != new.mount) {
+        status = EXDEV;
+        goto out_new;
+    }
+    if (old.entry == old.mount->root || new.entry == new.mount->root) {
+        status = EBUSY;
+        goto out_new;
+    }
+    if (old.entry == new.entry)
+        goto out_new;
+
+    struct vfs_node* moved = vfs_entry_get_inode(old.entry);
+    if (!moved) {
+        status = ENOENT;
+        goto out_new;
+    }
+    if (moved->type == VNODE_DIRECTORY && entry_is_ancestor(old.entry, new.entry))
+        status = EINVAL;
+    if (!status && !entry_points_to(new.entry, moved)) {
+        status = check_rename_target(moved, new, identity);
+        if (!status)
+            status = do_rename(old, new, moved, identity);
+    }
+
+    vfs_node_unref(moved);
+out_new:
+    vfs_path_drop(new);
+out_old:
+    vfs_path_drop(old);
+    return status;
+}
+
+errno_t vfs_link(
+    struct vfs_path root,
+    struct vfs_path old_cwd,
+    const char* old_path,
+    struct vfs_path new_cwd,
+    const char* new_path,
+    bool follow_symlinks,
+    const struct ident* identity
+) {
+    enum vfs_lookup_flags flags = VFS_LOOKUP_MUST_EXIST;
+    if (follow_symlinks)
+        flags |= VFS_LOOKUP_FOLLOW_SYMLINKS;
+
+    struct vfs_path old;
+    errno_t status = vfs_path_lookup(root, old_cwd, old_path, identity, flags, &old);
+    if (status)
+        return status;
+
+    struct vfs_node* target = vfs_entry_get_inode(old.entry);
+    if (!target) {
+        status = ENOENT;
+        goto out_old;
+    }
+    if (target->type == VNODE_DIRECTORY) {
+        status = EPERM;
+        goto out_target;
+    }
+
+    struct vfs_path new;
+    status = vfs_path_lookup(root, new_cwd, new_path, identity, VFS_LOOKUP_MUST_NOT_EXIST, &new);
+    if (status)
+        goto out_target;
+
+    if (new.mount != old.mount) {
+        status = EXDEV;
+        goto out_new;
+    }
+
+    struct vfs_node* parent;
+    status = get_writable_parent(new, identity, &parent);
+    if (status)
+        goto out_new;
+
+    status = parent->ops->dir.link ? parent->ops->dir.link(parent, &new, target, identity) : EPERM;
+    vfs_node_unref(parent);
+
+out_new:
+    vfs_path_drop(new);
+out_target:
+    vfs_node_unref(target);
+out_old:
+    vfs_path_drop(old);
+    return status;
+}
+
+static mode_t node_type_mode(enum vfs_node_type type) {
+    switch (type) {
+    case VNODE_DIRECTORY:
+        return __S_IFDIR;
+    case VNODE_SYMLINK:
+        return __S_IFLNK;
+    case VNODE_FIFO:
+        return __S_IFIFO;
+    case VNODE_BLOCKDEV:
+        return __S_IFBLK;
+    case VNODE_CHARDEV:
+        return __S_IFCHR;
+    case VNODE_SOCKET:
+        return __S_IFSOCK;
+    default:
+        return __S_IFREG;
+    }
+}
+
+void vfs_stat(struct vfs_node* node, struct __stat* out) {
+    mutex_lock(&node->lock);
+    *out = (struct __stat){
+        .st_ino = node->id,
+        .st_mode = node->mode | node_type_mode(node->type),
+        .st_nlink = node->nlink,
+        .st_uid = node->uid,
+        .st_gid = node->gid,
+        .st_rdev = node->rdev,
+        .st_size = node->size,
+        .st_atim = node->atime,
+        .st_mtim = node->mtime,
+        .st_ctim = node->ctime,
+        .st_blksize = 4096, // TODO: Get from file system
+        .st_blocks = node->blocks,
+    };
+    mutex_unlock(&node->lock);
+}
+
+static bool is_owner(struct vfs_node* node, const struct ident* identity) {
+    return identity->euid == 0 || identity->euid == node->uid;
+}
+
+errno_t vfs_chmod(struct vfs_node* node, mode_t mode, const struct ident* identity) {
+    if (is_read_only(node))
+        return EROFS;
+
+    mutex_lock(&node->lock);
+    if (!is_owner(node, identity)) {
+        mutex_unlock(&node->lock);
+        return EPERM;
+    }
+    mode &= 07777;
+    if (identity->euid != 0 && node->type == VNODE_REGULAR && !ident_in_group(identity, node->gid, false))
+        mode &= ~__S_ISGID;
+    node->mode = mode;
+    node->ctime = now();
+    mutex_unlock(&node->lock);
+
+    return sync_node(node);
+}
+
+errno_t vfs_chown(struct vfs_node* node, uid_t uid, gid_t gid, const struct ident* identity) {
+    if (is_read_only(node))
+        return EROFS;
+
+    mutex_lock(&node->lock);
+    if (identity->euid != 0) {
+        const bool uid_ok = uid == (uid_t)-1 || uid == node->uid;
+        const bool gid_ok = gid == (gid_t)-1 || gid == node->gid || ident_in_group(identity, gid, false);
+        if (identity->euid != node->uid || !uid_ok || !gid_ok) {
+            mutex_unlock(&node->lock);
+            return EPERM;
+        }
+    }
+
+    if (uid == (uid_t)-1 && gid == (gid_t)-1) {
+        mutex_unlock(&node->lock);
+        return 0;
+    }
+
+    if (uid != (uid_t)-1)
+        node->uid = uid;
+    if (gid != (gid_t)-1)
+        node->gid = gid;
+    if (identity->euid != 0 && node->type == VNODE_REGULAR && (node->mode & (__S_IXUSR | __S_IXGRP | __S_IXOTH)))
+        node->mode &= ~(__S_ISUID | __S_ISGID);
+    node->ctime = now();
+    mutex_unlock(&node->lock);
+
+    return sync_node(node);
+}
+
+static bool valid_utime(const timespec_t* ts) {
+    return ts->tv_nsec == __UTIME_NOW || ts->tv_nsec == __UTIME_OMIT ||
+        (ts->tv_nsec >= 0 && ts->tv_nsec < 1'000'000'000);
+}
+
+errno_t vfs_utimens(struct vfs_node* node, const timespec_t* times, const struct ident* identity) {
+    const timespec_t current = now();
+    timespec_t ts[2] = {
+        {.tv_nsec = __UTIME_NOW},
+        {.tv_nsec = __UTIME_NOW},
+    };
+    if (times) {
+        ts[0] = times[0];
+        ts[1] = times[1];
+        if (!valid_utime(&ts[0]) || !valid_utime(&ts[1]))
+            return EINVAL;
+    }
+
+    if (ts[0].tv_nsec == __UTIME_OMIT && ts[1].tv_nsec == __UTIME_OMIT)
+        return 0;
+    if (is_read_only(node))
+        return EROFS;
+
+    const bool only_now = (ts[0].tv_nsec == __UTIME_NOW || ts[0].tv_nsec == __UTIME_OMIT) &&
+        (ts[1].tv_nsec == __UTIME_NOW || ts[1].tv_nsec == __UTIME_OMIT);
+
+    mutex_lock(&node->lock);
+    const bool owner = is_owner(node, identity);
+    mutex_unlock(&node->lock);
+
+    if (!owner) {
+        if (!only_now)
+            return EPERM;
+        errno_t status = vfs_node_try_access(node, identity, W_OK, false);
+        if (status)
+            return status;
+    }
+
+    mutex_lock(&node->lock);
+    for (size_t i = 0; i < 2; i++) {
+        if (ts[i].tv_nsec == __UTIME_OMIT)
+            continue;
+        timespec_t* target = i == 0 ? &node->atime : &node->mtime;
+        *target = ts[i].tv_nsec == __UTIME_NOW ? current : ts[i];
+    }
+    node->ctime = current;
+    mutex_unlock(&node->lock);
+
+    return sync_node(node);
+}
+
+errno_t vfs_node_check_exec(struct vfs_node* node, const struct ident* identity) {
+    if (node->type != VNODE_REGULAR)
+        return EACCES;
+    return vfs_node_try_access(node, identity, X_OK, false);
+}
+
+errno_t vfs_truncate(struct vfs_node* node, uint64_t length) {
+    if (node->type == VNODE_DIRECTORY)
+        return EISDIR;
+    if (node->type != VNODE_REGULAR)
+        return EINVAL;
+    if (!node->ops->regular.truncate)
+        return EPERM;
+    return node->ops->regular.truncate(node, length);
+}
+
+errno_t vfs_read_link(struct vfs_node* node, iovec_iter_t* iter, size_t* out_read) {
+    if (node->type != VNODE_SYMLINK)
+        return EINVAL;
+    if (!node->ops->symlink.read_link)
+        return EINVAL;
+    return node->ops->symlink.read_link(node, iter, out_read);
+}
+
+errno_t vfs_statvfs(struct vfs_node* node, struct __statvfs* out) {
+    if (!node->sb || !node->sb->ops || !node->sb->ops->statvfs)
+        return ENOSYS;
+    return node->sb->ops->statvfs(node->sb, out);
+}

fs/pipe.c

@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pipe.h>
+#include <zinnia/poll.h>
+#include <zinnia/ringbuf.h>
+#include <zinnia/sched.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/vfs.h>
+#include <uapi/ioctls.h>
+#include "internal.h"
+
+constexpr size_t PIPE_CAPACITY = 0x10000;
+
+struct pipe {
+    mutex_t lock;
+    struct ringbuf buffer;
+    size_t readers;
+    size_t writers;
+    uint64_t read_opens;
+    uint64_t write_opens;
+    struct poll_event read_event;
+    struct poll_event write_event;
+    struct poll_event open_event;
+};
+
+static const struct vfs_file_ops pipe_file_ops;
+
+static errno_t pipe_new(struct pipe** out) {
+    struct pipe* pipe = mem_alloc(sizeof(*pipe), 0);
+    if (!pipe)
+        return ENOMEM;
+
+    errno_t status = ringbuf_init(&pipe->buffer, PIPE_CAPACITY);
+    if (status) {
+        mem_free(pipe);
+        return status;
+    }
+
+    mutex_init(&pipe->lock);
+    poll_event_init(&pipe->read_event);
+    poll_event_init(&pipe->write_event);
+    poll_event_init(&pipe->open_event);
+    *out = pipe;
+    return 0;
+}
+
+void pipe_destroy(struct pipe* pipe) {
+    ringbuf_fini(&pipe->buffer);
+    mem_free(pipe);
+}
+
+static bool file_readable(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_RDONLY;
+}
+
+static bool file_writable(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_WRONLY;
+}
+
+static bool file_nonblocking(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_NONBLOCK;
+}
+
+static void notify_all(struct pipe* pipe) {
+    poll_event_notify(&pipe->read_event);
+    poll_event_notify(&pipe->write_event);
+    poll_event_notify(&pipe->open_event);
+}
+
+static uint64_t attach_locked(struct pipe* pipe, struct vfs_file* file) {
+    file->ops = &pipe_file_ops;
+    file->priv = pipe;
+    vfs_stream_open(file);
+
+    if (file_readable(file)) {
+        pipe->readers++;
+        pipe->read_opens++;
+    }
+    if (file_writable(file)) {
+        pipe->writers++;
+        pipe->write_opens++;
+    }
+    return file_readable(file) ? pipe->write_opens : pipe->read_opens;
+}
+
+static void pipe_release(struct vfs_file* file) {
+    struct pipe* pipe = file->priv;
+
+    mutex_lock(&pipe->lock);
+    if (file_readable(file))
+        pipe->readers--;
+    if (file_writable(file))
+        pipe->writers--;
+    mutex_unlock(&pipe->lock);
+
+    notify_all(pipe);
+}
+
+static errno_t pipe_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct pipe* pipe = file->priv;
+    const size_t want = iovec_iter_len(buffer);
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &pipe->read_event);
+
+    errno_t status = 0;
+    size_t read = 0;
+    mutex_lock(&pipe->lock);
+    while (true) {
+        if (ringbuf_len(&pipe->buffer)) {
+            status = ringbuf_read_iter(&pipe->buffer, buffer, want, &read);
+            if (read)
+                poll_event_notify(&pipe->write_event);
+            break;
+        }
+        if (pipe->writers == 0)
+            break;
+        if (file_nonblocking(file)) {
+            status = EAGAIN;
+            break;
+        }
+        if (poll_table_sleep(&table, &pipe->lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&pipe->lock);
+    poll_table_fini(&table);
+
+    *out_read = read;
+    return status;
+}
+
+static errno_t pipe_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct pipe* pipe = file->priv;
+    const size_t total = iovec_iter_len(buffer);
+    const bool atomic = total <= __PIPE_BUF;
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &pipe->write_event);
+
+    errno_t status = 0;
+    size_t done = 0;
+    mutex_lock(&pipe->lock);
+    while (done < total) {
+        if (pipe->readers == 0) {
+            status = EPIPE;
+            break;
+        }
+
+        const size_t remaining = total - done;
+        const size_t space = ringbuf_space(&pipe->buffer);
+        if (space && (!atomic || space >= remaining)) {
+            size_t written;
+            status = ringbuf_write_iter(&pipe->buffer, buffer, remaining, &written);
+            done += written;
+            if (written)
+                poll_event_notify(&pipe->read_event);
+            if (status)
+                break;
+            continue;
+        }
+
+        if (file_nonblocking(file)) {
+            status = EAGAIN;
+            break;
+        }
+        if (poll_table_sleep(&table, &pipe->lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&pipe->lock);
+    poll_table_fini(&table);
+
+    *out_written = done;
+    return done ? 0 : status;
+}
+
+static errno_t pipe_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    struct pipe* pipe = file->priv;
+
+    switch ((uint32_t)request) {
+    case __FIONREAD: {
+        mutex_lock(&pipe->lock);
+        int len = ringbuf_len(&pipe->buffer);
+        mutex_unlock(&pipe->lock);
+        return usercopy_write((__user void*)arg, &len, sizeof(len)) ? 0 : EFAULT;
+    }
+    default:
+        return ENOTTY;
+    }
+}
+
+static enum poll_flags pipe_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    struct pipe* pipe = file->priv;
+
+    if (file_readable(file))
+        poll_wait(table, &pipe->read_event);
+    if (file_writable(file))
+        poll_wait(table, &pipe->write_event);
+
+    enum poll_flags revents = 0;
+    mutex_lock(&pipe->lock);
+    if (file_readable(file)) {
+        if (ringbuf_len(&pipe->buffer))
+            revents |= POLLIN | POLLRDNORM;
+        if (pipe->writers == 0)
+            revents |= POLLHUP;
+    }
+    if (file_writable(file)) {
+        if (ringbuf_space(&pipe->buffer) >= __PIPE_BUF)
+            revents |= POLLOUT | POLLWRNORM;
+        if (pipe->readers == 0)
+            revents |= POLLERR;
+    }
+    mutex_unlock(&pipe->lock);
+    return revents;
+}
+
+static const struct vfs_file_ops pipe_file_ops = {
+    .read = pipe_read,
+    .write = pipe_write,
+    .ioctl = pipe_ioctl,
+    .poll = pipe_poll,
+    .release = pipe_release,
+};
+
+errno_t pipe_create(
+    enum open_flags flags,
+    const struct ident* identity,
+    struct vfs_file** out_read,
+    struct vfs_file** out_write
+) {
+    struct vfs_node* node;
+    errno_t status = vfs_create_anon_inode(&pipe_file_ops, VNODE_FIFO, 0600, identity, &node);
+    if (status)
+        return status;
+
+    status = pipe_new(&node->pipe);
+    if (status) {
+        vfs_node_unref(node);
+        return status;
+    }
+
+    struct pipe* pipe = node->pipe;
+    const enum open_flags extra = flags & O_NONBLOCK;
+    struct vfs_file* read_end;
+    struct vfs_file* write_end;
+    status = vfs_file_new(node, nullptr, O_RDONLY | extra, &read_end);
+    if (!status) {
+        mutex_lock(&pipe->lock);
+        attach_locked(pipe, read_end);
+        mutex_unlock(&pipe->lock);
+
+        status = vfs_file_new(node, nullptr, O_WRONLY | extra, &write_end);
+        if (status)
+            vfs_file_unref(read_end);
+    }
+    if (!status) {
+        mutex_lock(&pipe->lock);
+        attach_locked(pipe, write_end);
+        mutex_unlock(&pipe->lock);
+    }
+    vfs_node_unref(node);
+    if (status)
+        return status;
+
+    *out_read = read_end;
+    *out_write = write_end;
+    return 0;
+}
+
+errno_t vfs_fifo_open(struct vfs_node* node, struct vfs_file* file) {
+    const bool readable = file_readable(file);
+    const bool writable = file_writable(file);
+    const bool nonblock = atomic_load(&file->flags, ATOMIC_RELAXED) & O_NONBLOCK;
+    if (!readable && !writable)
+        return EINVAL;
+
+    errno_t status = 0;
+    mutex_lock(&node->lock);
+    if (!node->pipe)
+        status = pipe_new(&node->pipe);
+    struct pipe* pipe = node->pipe;
+    mutex_unlock(&node->lock);
+    if (status)
+        return status;
+
+    mutex_lock(&pipe->lock);
+    if (writable && !readable && nonblock && pipe->readers == 0) {
+        mutex_unlock(&pipe->lock);
+        return ENXIO;
+    }
+    const uint64_t peer_opens = attach_locked(pipe, file);
+    mutex_unlock(&pipe->lock);
+    notify_all(pipe);
+
+    if (nonblock || (readable && writable))
+        return 0;
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &pipe->open_event);
+
+    mutex_lock(&pipe->lock);
+    while (true) {
+        const size_t peers = readable ? pipe->writers : pipe->readers;
+        const uint64_t opens = readable ? pipe->write_opens : pipe->read_opens;
+        if (peers || opens != peer_opens)
+            break;
+        if (poll_table_sleep(&table, &pipe->lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&pipe->lock);
+    poll_table_fini(&table);
+    return status;
+}

fs/poll.c

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/poll.h>
+#include <zinnia/sched.h>
+#include <zinnia/signal.h>
+#include <zinnia/timer.h>
+
+struct poll_timeout {
+    struct timer timer;
+    struct task* task;
+    atomic_bool expired;
+};
+
+void poll_event_init(struct poll_event* event) {
+    spin_init(&event->lock);
+    tailq_init(&event->waiters);
+}
+
+void poll_event_notify(struct poll_event* event) {
+    spin_lock(&event->lock);
+    for (struct poll_waiter* w = tailq_first(&event->waiters); w; w = tailq_next(w, link)) {
+        if (w->table->notify) {
+            w->table->notify(w->table);
+            continue;
+        }
+        atomic_store(&w->table->triggered, true, ATOMIC_RELEASE);
+        sched_wake(w->table->task);
+    }
+    spin_unlock(&event->lock);
+}
+
+void poll_table_init(struct poll_table* table) {
+    table->task = percpu_get()->sched.current;
+    table->notify = nullptr;
+    atomic_init(&table->triggered, false);
+    table->waiters = nullptr;
+    table->inline_used = false;
+    table->interruptible = true;
+    table->error = 0;
+}
+
+void poll_table_init_callback(struct poll_table* table, void (*notify)(struct poll_table* table)) {
+    poll_table_init(table);
+    table->task = nullptr;
+    table->notify = notify;
+}
+
+void poll_table_fini(struct poll_table* table) {
+    struct poll_waiter* w = table->waiters;
+    while (w) {
+        struct poll_waiter* next = w->next;
+        spin_lock(&w->event->lock);
+        tailq_remove(&w->event->waiters, w, link);
+        spin_unlock(&w->event->lock);
+        if (w != &table->inline_waiter)
+            mem_free(w);
+        w = next;
+    }
+    table->waiters = nullptr;
+    table->inline_used = false;
+}
+
+void poll_wait(struct poll_table* table, struct poll_event* event) {
+    if (!table)
+        return;
+
+    struct poll_waiter* w;
+    if (!table->inline_used) {
+        w = &table->inline_waiter;
+        table->inline_used = true;
+    } else if (!(w = mem_alloc(sizeof(*w), 0))) {
+        table->error = ENOMEM;
+        return;
+    }
+    w->event = event;
+    w->table = table;
+    w->next = table->waiters;
+    table->waiters = w;
+
+    spin_lock(&event->lock);
+    tailq_insert_tail(&event->waiters, w, link);
+    spin_unlock(&event->lock);
+}
+
+static void timeout_fn(struct timer* timer) {
+    struct poll_timeout* timeout = containerof(timer, struct poll_timeout, timer);
+    atomic_store(&timeout->expired, true, ATOMIC_RELEASE);
+    sched_wake(timeout->task);
+}
+
+errno_t poll_table_wait(struct poll_table* table, const uint64_t* deadline_ns) {
+    struct poll_timeout timeout = {.task = table->task};
+    atomic_init(&timeout.expired, false);
+    timer_init(&timeout.timer, timeout_fn);
+    if (deadline_ns)
+        timer_arm(&timeout.timer, *deadline_ns);
+
+    errno_t status = 0;
+    while (!atomic_load(&table->triggered, ATOMIC_ACQUIRE)) {
+        if (atomic_load(&timeout.expired, ATOMIC_ACQUIRE)) {
+            status = ETIMEDOUT;
+            break;
+        }
+        if (table->interruptible && signal_pending(table->task)) {
+            status = EINTR;
+            break;
+        }
+        sched_block(&percpu_get()->sched);
+    }
+
+    if (deadline_ns)
+        timer_cancel(&timeout.timer);
+    return status;
+}
+
+errno_t poll_table_sleep(struct poll_table* table, mutex_t* lock, const uint64_t* deadline_ns) {
+    mutex_unlock(lock);
+    errno_t status = poll_table_wait(table, deadline_ns);
+    mutex_lock(lock);
+    atomic_store(&table->triggered, false, ATOMIC_RELAXED);
+    return status;
+}

fs/refs.c

@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/vfs.h>
+#include <string.h>
+#include "internal.h"
+
+errno_t vfs_node_new(
+    struct vfs_super_block* sb,
+    enum vfs_node_type type,
+    const struct vfs_node_ops* ops,
+    mode_t mode,
+    struct vfs_node** out
+) {
+    struct vfs_node* node = mem_alloc(sizeof(*node), 0);
+    if (!node)
+        return ENOMEM;
+
+    refcount_init(&node->rc);
+    node->type = type;
+    node->ops = ops;
+    node->file_ops = ops ? ops->file_ops : nullptr;
+    node->sb = sb;
+    if (sb)
+        vfs_super_block_ref(sb);
+    mutex_init(&node->lock);
+    node->mode = mode & 07777;
+    node->nlink = 1;
+    const timespec_t now = time_from_nano(clock_get_realtime_ns());
+    node->atime = now;
+    node->mtime = now;
+    node->ctime = now;
+
+    *out = node;
+    return 0;
+}
+
+void vfs_node_ref(struct vfs_node* node) {
+    refcount_inc(&node->rc);
+}
+
+void vfs_node_unref(struct vfs_node* node) {
+    if (!refcount_dec(&node->rc))
+        return;
+
+    if (node->ops && node->ops->destroy)
+        node->ops->destroy(node);
+    if (node->type == VNODE_FIFO && node->pipe)
+        pipe_destroy(node->pipe);
+    if (node->sb)
+        vfs_super_block_unref(node->sb);
+    mem_free(node);
+}
+
+errno_t vfs_entry_new(const char* name, size_t name_len, struct vfs_entry* parent, struct vfs_entry** out) {
+    struct vfs_entry* entry = mem_alloc(sizeof(*entry), 0);
+    if (!entry)
+        return ENOMEM;
+
+    entry->name = mem_alloc(name_len + 1, 0);
+    if (!entry->name) {
+        mem_free(entry);
+        return ENOMEM;
+    }
+    memcpy(entry->name, name, name_len);
+    entry->name[name_len] = '\0';
+    entry->name_len = name_len;
+
+    refcount_init(&entry->rc);
+    mutex_init(&entry->lock);
+    entry->state = VFS_ENTRY_NOT_CACHED;
+    entry->inode = nullptr;
+    entry->parent = parent;
+    if (parent)
+        vfs_entry_ref(parent);
+    tailq_init(&entry->children);
+    vec_init(&entry->mounts);
+
+    *out = entry;
+    return 0;
+}
+
+void vfs_entry_ref(struct vfs_entry* entry) {
+    refcount_inc(&entry->rc);
+}
+
+void vfs_entry_unref(struct vfs_entry* entry) {
+    if (!refcount_dec(&entry->rc))
+        return;
+
+    // Children and mounts hold references to this entry, so both must be gone by now.
+    assert(tailq_empty(&entry->children), "Destroying an entry that still has children");
+    assert(vec_is_empty(&entry->mounts), "Destroying an entry that is still a mount point");
+
+    if (entry->inode)
+        vfs_node_unref(entry->inode);
+    if (entry->parent)
+        vfs_entry_unref(entry->parent);
+    vec_destroy(&entry->mounts);
+    mem_free(entry->name);
+    mem_free(entry);
+}
+
+void vfs_entry_set_inode(struct vfs_entry* entry, struct vfs_node* node) {
+    vfs_node_ref(node);
+
+    mutex_lock(&entry->lock);
+    struct vfs_node* old = entry->inode;
+    entry->inode = node;
+    entry->state = VFS_ENTRY_PRESENT;
+    mutex_unlock(&entry->lock);
+
+    if (old)
+        vfs_node_unref(old);
+}
+
+void vfs_entry_set_negative(struct vfs_entry* entry) {
+    mutex_lock(&entry->lock);
+    struct vfs_node* old = entry->inode;
+    entry->inode = nullptr;
+    entry->state = VFS_ENTRY_NOT_PRESENT;
+    mutex_unlock(&entry->lock);
+
+    if (old)
+        vfs_node_unref(old);
+}
+
+void vfs_entry_replace_children(struct vfs_entry* dst, struct vfs_entry* src) {
+    tailq_head(struct vfs_entry) stale;
+    tailq_init(&stale);
+
+    mutex_lock(&src->lock);
+    mutex_lock(&dst->lock);
+    struct vfs_entry* child;
+    while ((child = tailq_first(&dst->children))) {
+        tailq_remove(&dst->children, child, child_link);
+        tailq_insert_tail(&stale, child, child_link);
+    }
+    while ((child = tailq_first(&src->children))) {
+        tailq_remove(&src->children, child, child_link);
+        child->parent = dst;
+        vfs_entry_ref(dst);
+        vfs_entry_unref(src);
+        tailq_insert_tail(&dst->children, child, child_link);
+    }
+    mutex_unlock(&dst->lock);
+    mutex_unlock(&src->lock);
+
+    while ((child = tailq_first(&stale))) {
+        tailq_remove(&stale, child, child_link);
+        vfs_entry_unref(child);
+    }
+}
+
+void vfs_entry_prune(struct vfs_entry* entry) {
+    tailq_head(struct vfs_entry) stale;
+    tailq_init(&stale);
+
+    mutex_lock(&entry->lock);
+    struct vfs_entry* child = tailq_first(&entry->children);
+    while (child) {
+        struct vfs_entry* next = tailq_next(child, child_link);
+        vfs_entry_prune(child);
+
+        mutex_lock(&child->lock);
+        const bool idle = tailq_empty(&child->children) && vec_is_empty(&child->mounts);
+        mutex_unlock(&child->lock);
+        if (idle && atomic_load(&child->rc.count, ATOMIC_RELAXED) == 1) {
+            tailq_remove(&entry->children, child, child_link);
+            tailq_insert_tail(&stale, child, child_link);
+        }
+        child = next;
+    }
+    mutex_unlock(&entry->lock);
+
+    while ((child = tailq_first(&stale))) {
+        tailq_remove(&stale, child, child_link);
+        vfs_entry_unref(child);
+    }
+}
+
+struct vfs_node* vfs_entry_get_inode(struct vfs_entry* entry) {
+    struct vfs_node* node = nullptr;
+
+    mutex_lock(&entry->lock);
+    if (entry->state == VFS_ENTRY_PRESENT) {
+        node = entry->inode;
+        vfs_node_ref(node);
+    }
+    mutex_unlock(&entry->lock);
+
+    return node;
+}
+
+void vfs_mount_ref(struct vfs_mount* mount) {
+    refcount_inc(&mount->rc);
+}
+
+void vfs_mount_unref(struct vfs_mount* mount) {
+    if (!refcount_dec(&mount->rc))
+        return;
+
+    if (mount->has_mount_point)
+        vfs_path_drop(mount->mount_point);
+    vfs_entry_unref(mount->root);
+    mem_free(mount);
+}
+
+void vfs_super_block_ref(struct vfs_super_block* sb) {
+    refcount_inc(&sb->rc);
+}
+
+void vfs_super_block_unref(struct vfs_super_block* sb) {
+    if (!refcount_dec(&sb->rc))
+        return;
+
+    if (sb->ops && sb->ops->destroy)
+        sb->ops->destroy(sb);
+}
+
+void vfs_file_ref(struct vfs_file* file) {
+    refcount_inc(&file->rc);
+}
+
+void vfs_file_unref(struct vfs_file* file) {
+    if (!refcount_dec(&file->rc))
+        return;
+
+    if (file->ops && file->ops->release)
+        file->ops->release(file);
+    if (file->has_path)
+        vfs_path_drop(file->path);
+    vfs_node_unref(file->vnode);
+    mem_free(file->absolute_path);
+    mem_free(file);
+}
+
+struct vfs_path vfs_path_clone(struct vfs_path path) {
+    vfs_mount_ref(path.mount);
+    vfs_entry_ref(path.entry);
+    return path;
+}
+
+void vfs_path_drop(struct vfs_path path) {
+    vfs_entry_unref(path.entry);
+    vfs_mount_unref(path.mount);
+}

fs/tmpfs.c

@@ -0,0 +1,509 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/vfs.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+#include <arch/mem.h>
+#include <string.h>
+
+struct tmpfs_super {
+    struct vfs_super_block base;
+    _Atomic(ino_t) inode_counter;
+};
+
+struct tmpfs_symlink {
+    char* target;
+    size_t len;
+};
+
+struct tmpfs_file {
+    struct vmo vmo;
+    _Atomic(uint64_t) size;
+};
+
+static errno_t tmpfs_file_get_pages(
+    struct vmo* vmo,
+    uint64_t index,
+    struct page** pages,
+    size_t count,
+    size_t* out_valid
+) {
+    struct tmpfs_file* file = containerof(vmo, struct tmpfs_file, vmo);
+    const size_t page_size = arch_mem_page_size();
+    const uint64_t size = atomic_load(&file->size, ATOMIC_ACQUIRE);
+    const uint64_t limit = size / page_size + (size % page_size != 0);
+
+    const size_t valid = index >= limit ? 0 : MIN(count, limit - index);
+    for (size_t i = 0; i < valid; i++)
+        memset(HHDM_PTR(vm_page_addr(pages[i])), 0, page_size);
+    *out_valid = valid;
+    return 0;
+}
+
+static void tmpfs_file_destroy(struct vmo* vmo) {
+    mem_free(containerof(vmo, struct tmpfs_file, vmo));
+}
+
+static const struct vmo_pager_ops tmpfs_file_pager = {
+    .get_pages = tmpfs_file_get_pages,
+    .destroy = tmpfs_file_destroy,
+};
+
+static const struct vfs_node_ops tmpfs_dir_ops;
+static const struct vfs_node_ops tmpfs_regular_ops;
+static const struct vfs_node_ops tmpfs_symlink_ops;
+
+static errno_t tmpfs_create_inode(
+    struct vfs_super_block* sb,
+    enum vfs_node_type type,
+    const struct vfs_node_ops* ops,
+    mode_t mode,
+    const struct ident* identity,
+    struct vfs_node** out
+) {
+    struct tmpfs_super* super = containerof(sb, struct tmpfs_super, base);
+
+    struct vfs_node* node;
+    errno_t status = vfs_node_new(sb, type, ops, mode, &node);
+    if (status)
+        return status;
+
+    node->id = atomic_fetch_add(&super->inode_counter, 1, ATOMIC_RELAXED);
+    node->uid = identity->euid;
+    node->gid = identity->egid;
+
+    *out = node;
+    return 0;
+}
+
+// Super block
+
+static errno_t tmpfs_super_sync(struct vfs_super_block* sb) {
+    // This is a no-op.
+    return 0;
+}
+
+static errno_t tmpfs_super_statvfs(struct vfs_super_block* sb, struct __statvfs* out) {
+    memset(out, 0, sizeof(*out));
+    out->f_namemax = __NAME_MAX;
+    return 0;
+}
+
+static void tmpfs_super_destroy(struct vfs_super_block* sb) {
+    mem_free(containerof(sb, struct tmpfs_super, base));
+}
+
+static const struct vfs_super_block_ops tmpfs_super_ops = {
+    .sync = tmpfs_super_sync,
+    .statvfs = tmpfs_super_statvfs,
+    .destroy = tmpfs_super_destroy,
+};
+
+// Directories
+
+static errno_t tmpfs_dir_lookup(struct vfs_node* node, struct vfs_path* child) {
+    // tmpfs directories only live in memory, so we cannot look up entries that do not exist.
+    return ENOENT;
+}
+
+static errno_t tmpfs_dir_open(
+    struct vfs_node* node,
+    struct vfs_path* path,
+    enum open_flags flags,
+    const struct ident* identity,
+    struct vfs_file** out_file
+) {
+    return vfs_file_new(node, path, flags, out_file);
+}
+
+static errno_t tmpfs_dir_create(
+    struct vfs_node* node,
+    struct vfs_entry* entry,
+    mode_t mode,
+    const struct ident* identity
+) {
+    struct tmpfs_file* data = mem_alloc(sizeof(struct tmpfs_file), 0);
+    if (!data)
+        return ENOMEM;
+    vmo_init(&data->vmo, &tmpfs_file_pager);
+    struct vmo* cache = &data->vmo;
+    errno_t status;
+
+    struct vfs_node* file;
+    status = tmpfs_create_inode(node->sb, VNODE_REGULAR, &tmpfs_regular_ops, mode, identity, &file);
+    if (status) {
+        vmo_unref(cache);
+        return status;
+    }
+
+    // Attach the cache before the node becomes visible through the entry.
+    file->priv = cache;
+    vfs_entry_set_inode(entry, file);
+    vfs_node_unref(file);
+    return 0;
+}
+
+static errno_t tmpfs_dir_mkdir(
+    struct vfs_node* node,
+    struct vfs_path* path,
+    mode_t mode,
+    const struct ident* identity
+) {
+    struct vfs_node* dir;
+    errno_t status = tmpfs_create_inode(node->sb, VNODE_DIRECTORY, &tmpfs_dir_ops, mode, identity, &dir);
+    if (status)
+        return status;
+
+    vfs_entry_set_inode(path->entry, dir);
+    vfs_node_unref(dir);
+    return 0;
+}
+
+static errno_t tmpfs_dir_symlink(
+    struct vfs_node* node,
+    struct vfs_path* path,
+    const char* target,
+    size_t target_len,
+    const struct ident* identity
+) {
+    struct tmpfs_symlink* link = mem_alloc(sizeof(*link), 0);
+    if (!link)
+        return ENOMEM;
+    link->target = mem_alloc(target_len + 1, ALLOC_NOZERO);
+    if (!link->target) {
+        mem_free(link);
+        return ENOMEM;
+    }
+    memcpy(link->target, target, target_len);
+    link->target[target_len] = '\0';
+    link->len = target_len;
+
+    struct vfs_node* sym;
+    errno_t status = tmpfs_create_inode(node->sb, VNODE_SYMLINK, &tmpfs_symlink_ops, 0777, identity, &sym);
+    if (status) {
+        mem_free(link->target);
+        mem_free(link);
+        return status;
+    }
+
+    sym->priv = link;
+    sym->size = target_len;
+    vfs_entry_set_inode(path->entry, sym);
+    vfs_node_unref(sym);
+    return 0;
+}
+
+static void adjust_links(struct vfs_node* node, int delta) {
+    mutex_lock(&node->lock);
+    node->nlink += delta;
+    node->ctime = time_from_nano(clock_get_realtime_ns());
+    mutex_unlock(&node->lock);
+}
+
+static errno_t tmpfs_dir_link(
+    struct vfs_node* node,
+    struct vfs_path* path,
+    struct vfs_node* target,
+    const struct ident* identity
+) {
+    adjust_links(target, 1);
+    vfs_entry_set_inode(path->entry, target);
+    return 0;
+}
+
+// Removes `child` from the cached children of `parent` and drops the list's reference.
+static void detach_child(struct vfs_entry* parent, struct vfs_entry* child) {
+    bool found = false;
+
+    mutex_lock(&parent->lock);
+    for (struct vfs_entry* it = tailq_first(&parent->children); it; it = tailq_next(it, child_link)) {
+        if (it == child) {
+            tailq_remove(&parent->children, child, child_link);
+            found = true;
+            break;
+        }
+    }
+    mutex_unlock(&parent->lock);
+
+    if (found)
+        vfs_entry_unref(child);
+}
+
+static errno_t tmpfs_dir_unlink(struct vfs_node* node, struct vfs_path* path, const struct ident* identity) {
+    struct vfs_path parent;
+    errno_t status = vfs_path_lookup_parent(*path, &parent);
+    if (status)
+        return status;
+
+    struct vfs_node* victim = vfs_entry_get_inode(path->entry);
+    if (victim) {
+        adjust_links(victim, -1);
+        vfs_node_unref(victim);
+    }
+    vfs_entry_set_negative(path->entry);
+    detach_child(parent.entry, path->entry);
+    vfs_path_drop(parent);
+    return 0;
+}
+
+static errno_t tmpfs_dir_rename(
+    struct vfs_node* node,
+    struct vfs_path* path,
+    struct vfs_node* target,
+    struct vfs_path* target_path,
+    const struct ident* identity
+) {
+    struct vfs_node* moved = vfs_entry_get_inode(path->entry);
+    if (!moved)
+        return ENOENT;
+
+    struct vfs_path parent;
+    errno_t status = vfs_path_lookup_parent(*path, &parent);
+    if (status) {
+        vfs_node_unref(moved);
+        return status;
+    }
+
+    struct vfs_node* replaced = vfs_entry_get_inode(target_path->entry);
+    if (replaced) {
+        adjust_links(replaced, -1);
+        vfs_node_unref(replaced);
+    }
+    vfs_entry_set_inode(target_path->entry, moved);
+    vfs_node_unref(moved);
+
+    // Transfer children for directory renames.
+    struct vfs_entry* src = path->entry;
+    vfs_entry_replace_children(target_path->entry, src);
+
+    vfs_entry_set_negative(src);
+    detach_child(parent.entry, src);
+    vfs_path_drop(parent);
+    return 0;
+}
+
+static errno_t tmpfs_dir_mknod(
+    struct vfs_node* node,
+    struct vfs_path* path,
+    mode_t mode,
+    struct vfs_mknod_target* target,
+    const struct ident* identity
+) {
+    if (!target)
+        return ENODEV;
+
+    struct vfs_node* new_node;
+    errno_t status = tmpfs_create_inode(node->sb, target->type, target->ops, mode, identity, &new_node);
+    if (status)
+        return status;
+
+    new_node->priv = target->priv;
+    new_node->rdev = target->rdev;
+    vfs_entry_set_inode(path->entry, new_node);
+    vfs_node_unref(new_node);
+    return 0;
+}
+
+static const struct vfs_file_ops tmpfs_dir_file_ops = {};
+
+static const struct vfs_node_ops tmpfs_dir_ops = {
+    .file_ops = &tmpfs_dir_file_ops,
+    .dir = {
+        .lookup = tmpfs_dir_lookup,
+        .open = tmpfs_dir_open,
+        .create = tmpfs_dir_create,
+        .mkdir = tmpfs_dir_mkdir,
+        .symlink = tmpfs_dir_symlink,
+        .link = tmpfs_dir_link,
+        .unlink = tmpfs_dir_unlink,
+        .rmdir = tmpfs_dir_unlink,
+        .rename = tmpfs_dir_rename,
+        .mknod = tmpfs_dir_mknod,
+        .get_dir_entries = vfs_dir_get_cached_entries,
+    },
+};
+
+// Regular files
+
+static struct vmo* regular_cache(struct vfs_node* node) {
+    return node->priv;
+}
+
+static void set_size(struct vfs_node* node, uint64_t size) {
+    node->size = size;
+    atomic_store(&containerof(regular_cache(node), struct tmpfs_file, vmo)->size, size, ATOMIC_RELEASE);
+}
+
+static errno_t tmpfs_regular_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct vfs_node* node = file->vnode;
+
+    mutex_lock(&node->lock);
+    size_t size = node->size;
+    mutex_unlock(&node->lock);
+
+    *out_read = 0;
+    if (offset >= size)
+        return 0;
+
+    size_t progress;
+    errno_t status =
+        vmo_read_iter(regular_cache(node), offset, buffer, MIN(iovec_iter_len(buffer), size - offset), &progress);
+    *out_read = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t tmpfs_regular_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct vfs_node* node = file->vnode;
+    const size_t total = iovec_iter_len(buffer);
+
+    mutex_lock(&node->lock);
+    const uint64_t old_size = node->size;
+    if (offset + total > old_size)
+        set_size(node, offset + total);
+    size_t progress;
+    errno_t status = vmo_write_iter(regular_cache(node), offset, buffer, total, &progress);
+    if (progress == 0 && total > 0 && !status)
+        status = ENOMEM;
+    if (offset + total > old_size)
+        set_size(node, MAX(old_size, offset + progress));
+    mutex_unlock(&node->lock);
+
+    *out_written = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t tmpfs_regular_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    enum vma_flags vflags = (flags & MAP_SHARED) ? VMA_SHARED : 0;
+    errno_t status = vma_map(vma, regular_cache(file->vnode), addr, len, prot, vflags, offset);
+    if (status)
+        return status;
+
+    *out_addr = addr;
+    return 0;
+}
+
+static errno_t tmpfs_regular_truncate(struct vfs_node* node, uint64_t new_length) {
+    mutex_lock(&node->lock);
+    const uint64_t old_size = node->size;
+    set_size(node, new_length);
+    if (new_length < old_size)
+        vmo_truncate(regular_cache(node), new_length);
+    mutex_unlock(&node->lock);
+    return 0;
+}
+
+static errno_t tmpfs_regular_get_object(struct vfs_node* node, struct vmo** out) {
+    struct vmo* obj = regular_cache(node);
+    vmo_ref(obj);
+    *out = obj;
+    return 0;
+}
+
+static void tmpfs_regular_destroy(struct vfs_node* node) {
+    if (node->priv)
+        vmo_unref(regular_cache(node));
+}
+
+static const struct vfs_file_ops tmpfs_regular_file_ops = {
+    .read = tmpfs_regular_read,
+    .write = tmpfs_regular_write,
+    .mmap = tmpfs_regular_mmap,
+};
+
+static const struct vfs_node_ops tmpfs_regular_ops = {
+    .destroy = tmpfs_regular_destroy,
+    .file_ops = &tmpfs_regular_file_ops,
+    .regular = {
+        .truncate = tmpfs_regular_truncate,
+        .get_vmo = tmpfs_regular_get_object,
+    },
+};
+
+// Symbolic links
+
+static errno_t tmpfs_symlink_read_link(struct vfs_node* node, iovec_iter_t* iter, size_t* out_read) {
+    struct tmpfs_symlink* link = node->priv;
+    return iovec_iter_copy_to(iter, link->target, link->len, out_read);
+}
+
+static void tmpfs_symlink_destroy(struct vfs_node* node) {
+    struct tmpfs_symlink* link = node->priv;
+    if (link) {
+        mem_free(link->target);
+        mem_free(link);
+    }
+}
+
+static const struct vfs_file_ops tmpfs_symlink_file_ops = {};
+
+static const struct vfs_node_ops tmpfs_symlink_ops = {
+    .destroy = tmpfs_symlink_destroy,
+    .file_ops = &tmpfs_symlink_file_ops,
+    .symlink = {
+        .read_link = tmpfs_symlink_read_link,
+    },
+};
+
+// File system
+
+static errno_t tmpfs_mount(const struct vfs_filesystem* fs, enum mount_flags flags, void* arg, struct vfs_mount** out) {
+    struct tmpfs_super* super = mem_alloc(sizeof(*super), 0);
+    if (!super)
+        return ENOMEM;
+    refcount_init(&super->base.rc);
+    super->base.ops = &tmpfs_super_ops;
+    super->base.flags = flags;
+    atomic_init(&super->inode_counter, 1);
+
+    struct vfs_node* root_node = nullptr;
+    struct vfs_entry* root_entry = nullptr;
+
+    errno_t status =
+        tmpfs_create_inode(&super->base, VNODE_DIRECTORY, &tmpfs_dir_ops, 0755, ident_kernel(), &root_node);
+    if (status)
+        goto out;
+
+    status = vfs_entry_new("", 0, nullptr, &root_entry);
+    if (status)
+        goto out;
+    vfs_entry_set_inode(root_entry, root_node);
+
+    status = vfs_mount_new(flags, root_entry, out);
+
+out:
+    if (root_entry)
+        vfs_entry_unref(root_entry);
+    if (root_node)
+        vfs_node_unref(root_node);
+    // The root inode keeps the super block alive now.
+    vfs_super_block_unref(&super->base);
+    return status;
+}
+
+static const struct vfs_filesystem tmpfs = {
+    .name = "tmpfs",
+    .mount = tmpfs_mount,
+};
+
+static void tmpfs_stage_init(struct boot_info* info) {
+    errno_t status = vfs_register_filesystem(&tmpfs);
+    assert(status == 0, "Unable to register tmpfs (%i)", status);
+}
+
+INIT_TASK(tmpfs_stage, "fs.tmpfs", tmpfs_stage_init);
+INIT_DEPENDS(tmpfs_stage, mem_stage);
+INIT_ENTAILS(tmpfs_stage, vfs_stage);

include/sys/console.h

@@ -2,12 +2,41 @@
 #ifndef __ZINNIA_SYS_CONSOLE_H
 #define __ZINNIA_SYS_CONSOLE_H
 
+#include <uapi/console.h>
 #include <uapi/ioctls.h>
 
-#define VT_ACTIVATE     __IOW('v', 1, int)
-#define VT_WAITACTIVATE __IOW('v', 2, int)
-#define VT_GETACTIVE    __IOR('v', 3, int)
-#define VT_GETINDEX     __IOR('v', 4, int)
-#define VT_LOCKSWITCH   __IOW('v', 5, int)
+#define vt_mode __vt_mode
+#define vt_stat __vt_stat
+typedef struct __vt_mode vtmode_t;
+
+#define VT_AUTO    __VT_AUTO
+#define VT_PROCESS __VT_PROCESS
+
+#define VT_FALSE  __VT_FALSE
+#define VT_TRUE   __VT_TRUE
+#define VT_ACKACQ __VT_ACKACQ
+
+#define KD_TEXT     __KD_TEXT
+#define KD_GRAPHICS __KD_GRAPHICS
+
+#define K_RAW   __K_RAW
+#define K_XLATE __K_XLATE
+#define K_CODE  __K_CODE
+#define K_OFF   __K_OFF
+
+#define VT_OPENQRY    __VT_OPENQRY
+#define VT_SETMODE    __VT_SETMODE
+#define VT_GETMODE    __VT_GETMODE
+#define VT_RELDISP    __VT_RELDISP
+#define VT_ACTIVATE   __VT_ACTIVATE
+#define VT_WAITACTIVE __VT_WAITACTIVE
+#define VT_GETACTIVE  __VT_GETACTIVE
+#define VT_GETINDEX   __VT_GETINDEX
+#define VT_GETSTATE   __VT_GETSTATE
+
+#define KDGKBMODE __KDGKBMODE
+#define KDSKBMODE __KDSKBMODE
+#define KDGETMODE __KDGETMODE
+#define KDSETMODE __KDSETMODE
 
 #endif

include/uapi/block.h

@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: Unlicense */
+#ifndef __ZINNIA_UAPI_BLOCK_H
+#define __ZINNIA_UAPI_BLOCK_H
+
+#include <uapi/ioctls.h>
+
+#define __BLKGETSIZE64 __IOR('B', 0x00, unsigned long long)
+#define __BLKSSZGET    __IOR('B', 0x01, unsigned int)
+#define __BLKRRPART    __IO('B', 0x02)
+
+#endif

include/uapi/console.h

@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: Unlicense */
+#ifndef __ZINNIA_UAPI_CONSOLE_H
+#define __ZINNIA_UAPI_CONSOLE_H
+
+#include <uapi/ioctls.h>
+
+#define __VT_AUTO    0
+#define __VT_PROCESS 1
+
+#define __VT_FALSE  0
+#define __VT_TRUE   1
+#define __VT_ACKACQ 2
+
+#define __KD_TEXT     0
+#define __KD_GRAPHICS 1
+
+#define __K_RAW   0
+#define __K_XLATE 1
+#define __K_CODE  2
+#define __K_OFF   4
+
+struct __vt_mode {
+    char mode;
+    char waitv;
+    short relsig;
+    short acqsig;
+    short frsig;
+};
+
+struct __vt_stat {
+    unsigned short v_active;
+    unsigned short v_signal;
+    unsigned short v_state;
+};
+
+#define __VT_OPENQRY    __IOR('v', 1, int)
+#define __VT_SETMODE    __IOW('v', 2, struct __vt_mode)
+#define __VT_GETMODE    __IOR('v', 3, struct __vt_mode)
+#define __VT_RELDISP    __IOWINT('v', 4)
+#define __VT_ACTIVATE   __IOWINT('v', 5)
+#define __VT_WAITACTIVE __IOWINT('v', 6)
+#define __VT_GETACTIVE  __IOR('v', 7, int)
+#define __VT_GETINDEX   __IOR('v', 8, int)
+#define __VT_GETSTATE   __IOR('v', 0x80, struct __vt_stat)
+
+#define __KDGKBMODE __IOR('K', 6, int)
+#define __KDSKBMODE __IOWINT('K', 7)
+#define __KDGETMODE __IOR('K', 9, int)
+#define __KDSETMODE __IOWINT('K', 10)
+
+#endif

include/uapi/drm.h

@@ -0,0 +1,796 @@
+/* SPDX-License-Identifier: Unlicense */
+#ifndef __ZINNIA_UAPI_DRM_H
+#define __ZINNIA_UAPI_DRM_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <uapi/fcntl.h>
+#include <uapi/ioctls.h>
+
+#define __DRM_BASE 'd'
+
+#define __DRM_COMMAND_BASE 0x40
+#define __DRM_COMMAND_END  0xA0
+
+#define __DRM_IO(num)                __IO(__DRM_BASE, (num))
+#define __DRM_IOR(num, type)         __IOR(__DRM_BASE, (num), type)
+#define __DRM_IOW(num, type)         __IOW(__DRM_BASE, (num), type)
+#define __DRM_IOWR(num, type)        __IOWR(__DRM_BASE, (num), type)
+#define __DRM_DRIVER_IOWR(num, type) __IOWR(__DRM_BASE, __DRM_COMMAND_BASE + (num), type)
+
+typedef uint32_t __drm_context_t;
+typedef uint32_t __drm_drawable_t;
+typedef uint32_t __drm_magic_t;
+
+struct __drm_clip_rect {
+    uint16_t x1;
+    uint16_t y1;
+    uint16_t x2;
+    uint16_t y2;
+};
+
+struct __drm_drawable_info {
+    uint32_t num_rects;
+    uintptr_t rects;
+};
+
+struct __drm_version {
+    int32_t version_major;
+    int32_t version_minor;
+    int32_t version_patchlevel;
+    size_t name_len;
+    uintptr_t name;
+    size_t date_len;
+    uintptr_t date;
+    size_t desc_len;
+    uintptr_t desc;
+};
+
+struct __drm_unique {
+    size_t unique_len;
+    uintptr_t unique;
+};
+
+struct __drm_list {
+    int32_t count;
+    uintptr_t version;
+};
+
+struct __drm_block {
+    uint32_t unused;
+};
+
+struct __drm_auth {
+    __drm_magic_t magic;
+};
+
+#define __DRM_CAP_DUMB_BUFFER            0x1
+#define __DRM_CAP_VBLANK_HIGH_CRTC       0x2
+#define __DRM_CAP_DUMB_PREFERRED_DEPTH   0x3
+#define __DRM_CAP_DUMB_PREFER_SHADOW     0x4
+#define __DRM_CAP_PRIME                  0x5
+#define __DRM_PRIME_CAP_IMPORT           0x1
+#define __DRM_PRIME_CAP_EXPORT           0x2
+#define __DRM_CAP_TIMESTAMP_MONOTONIC    0x6
+#define __DRM_CAP_ASYNC_PAGE_FLIP        0x7
+#define __DRM_CAP_CURSOR_WIDTH           0x8
+#define __DRM_CAP_CURSOR_HEIGHT          0x9
+#define __DRM_CAP_ADDFB2_MODIFIERS       0x10
+#define __DRM_CAP_PAGE_FLIP_TARGET       0x11
+#define __DRM_CAP_CRTC_IN_VBLANK_EVENT   0x12
+#define __DRM_CAP_SYNCOBJ                0x13
+#define __DRM_CAP_SYNCOBJ_TIMELINE       0x14
+#define __DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP 0x15
+
+struct __drm_get_cap {
+    uint64_t capability;
+    uint64_t value;
+};
+
+#define __DRM_CLIENT_CAP_STEREO_3D            1
+#define __DRM_CLIENT_CAP_UNIVERSAL_PLANES     2
+#define __DRM_CLIENT_CAP_ATOMIC               3
+#define __DRM_CLIENT_CAP_ASPECT_RATIO         4
+#define __DRM_CLIENT_CAP_WRITEBACK_CONNECTORS 5
+#define __DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT 6
+
+struct __drm_set_client_cap {
+    uint64_t capability;
+    uint64_t value;
+};
+
+struct __drm_tex_region {
+    uint8_t next;
+    uint8_t prev;
+    uint8_t in_use;
+    uint8_t padding;
+    uint32_t age;
+};
+
+#define __DRM_ADD_COMMAND    0
+#define __DRM_RM_COMMAND     1
+#define __DRM_INST_HANDLER   2
+#define __DRM_UNINST_HANDLER 3
+
+struct __drm_control {
+    uint32_t func;
+    int32_t irq;
+};
+
+typedef uint32_t __drm_map_type;
+#define __DRM_FRAME_BUFFER   0
+#define __DRM_REGISTERS      1
+#define __DRM_SHM            2
+#define __DRM_AGP            3
+#define __DRM_SCATTER_GATHER 4
+#define __DRM_CONSISTENT     5
+
+typedef uint32_t __drm_map_flags;
+#define __DRM_RESTRICTED      0x01
+#define __DRM_READ_ONLY       0x02
+#define __DRM_LOCKED          0x04
+#define __DRM_KERNEL          0x08
+#define __DRM_WRITE_COMBINING 0x10
+#define __DRM_CONTAINS_LOCK   0x20
+#define __DRM_REMOVABLE       0x40
+#define __DRM_DRIVER          0x80
+
+struct __drm_ctx_priv_map {
+    uint32_t ctx_id;
+    uintptr_t handle;
+};
+
+struct __drm_map {
+    size_t offset;
+    size_t size;
+    __drm_map_type type;
+    __drm_map_flags flags;
+    uintptr_t handle;
+    int32_t mtrr;
+};
+
+struct __drm_client {
+    int32_t idx;
+    int32_t auth;
+    size_t pid;
+    size_t uid;
+    size_t magic;
+    size_t iocs;
+};
+
+typedef uint32_t __drm_stat_type;
+#define __DRM_STAT_LOCK      0
+#define __DRM_STAT_OPENS     1
+#define __DRM_STAT_CLOSES    2
+#define __DRM_STAT_IOCTLS    3
+#define __DRM_STAT_LOCKS     4
+#define __DRM_STAT_UNLOCKS   5
+#define __DRM_STAT_VALUE     6
+#define __DRM_STAT_BYTE      7
+#define __DRM_STAT_COUNT     8
+#define __DRM_STAT_IRQ       9
+#define __DRM_STAT_PRIMARY   10
+#define __DRM_STAT_SECONDARY 11
+#define __DRM_STAT_DMA       12
+#define __DRM_STAT_SPECIAL   13
+#define __DRM_STAT_MISSED    14
+
+struct __drm_stats_data {
+    size_t value;
+    __drm_stat_type type;
+};
+
+struct __drm_stats {
+    size_t count;
+    struct __drm_stats_data data[15];
+};
+
+struct __drm_draw {
+    __drm_drawable_t handle;
+};
+
+struct __drm_set_version {
+    int32_t drm_di_major;
+    int32_t drm_di_minor;
+    int32_t drm_dd_major;
+    int32_t drm_dd_minor;
+};
+
+typedef uint32_t __drm_ctx_flags;
+#define __DRM_CONTEXT_PRESERVED 0x01
+#define __DRM_CONTEXT_2DONLY    0x02
+
+struct __drm_ctx {
+    __drm_context_t handle;
+    __drm_ctx_flags flags;
+};
+
+struct __drm_ctx_res {
+    int32_t count;
+    uintptr_t contexts;
+};
+
+#define __DRM_DISPLAY_MODE_LEN 32
+
+struct __drm_mode_modeinfo {
+    uint32_t clock;
+    uint16_t hdisplay;
+    uint16_t hsync_start;
+    uint16_t hsync_end;
+    uint16_t htotal;
+    uint16_t hskew;
+    uint16_t vdisplay;
+    uint16_t vsync_start;
+    uint16_t vsync_end;
+    uint16_t vtotal;
+    uint16_t vscan;
+    uint32_t vrefresh;
+    uint32_t flags;
+    uint32_t type;
+    char name[__DRM_DISPLAY_MODE_LEN];
+};
+
+struct __drm_mode_crtc {
+    uint64_t set_connectors_ptr;
+    uint32_t count_connectors;
+    uint32_t crtc_id;
+    uint32_t fb_id;
+    uint32_t x;
+    uint32_t y;
+    uint32_t gamma_size;
+    uint32_t mode_valid;
+    struct __drm_mode_modeinfo mode;
+};
+
+struct __drm_mode_get_encoder {
+    uint32_t encoder_id;
+    uint32_t encoder_type;
+    uint32_t crtc_id;
+    uint32_t possible_crtcs;
+    uint32_t possible_clones;
+};
+
+#define __DRM_MODE_CONNECTOR_Unknown     0
+#define __DRM_MODE_CONNECTOR_VGA         1
+#define __DRM_MODE_CONNECTOR_DVII        2
+#define __DRM_MODE_CONNECTOR_DVID        3
+#define __DRM_MODE_CONNECTOR_DVIA        4
+#define __DRM_MODE_CONNECTOR_Composite   5
+#define __DRM_MODE_CONNECTOR_SVIDEO      6
+#define __DRM_MODE_CONNECTOR_LVDS        7
+#define __DRM_MODE_CONNECTOR_Component   8
+#define __DRM_MODE_CONNECTOR_9PinDIN     9
+#define __DRM_MODE_CONNECTOR_DisplayPort 10
+#define __DRM_MODE_CONNECTOR_HDMIA       11
+#define __DRM_MODE_CONNECTOR_HDMIB       12
+#define __DRM_MODE_CONNECTOR_TV          13
+#define __DRM_MODE_CONNECTOR_eDP         14
+#define __DRM_MODE_CONNECTOR_VIRTUAL     15
+#define __DRM_MODE_CONNECTOR_DSI         16
+#define __DRM_MODE_CONNECTOR_DPI         17
+#define __DRM_MODE_CONNECTOR_WRITEBACK   18
+#define __DRM_MODE_CONNECTOR_SPI         19
+#define __DRM_MODE_CONNECTOR_USB         20
+
+#define __DRM_MODE_CONNECTED         1
+#define __DRM_MODE_DISCONNECTED      2
+#define __DRM_MODE_UNKNOWNCONNECTION 3
+
+struct __drm_mode_get_connector {
+    uint64_t encoders_ptr;
+    uint64_t modes_ptr;
+    uint64_t props_ptr;
+    uint64_t prop_values_ptr;
+    uint32_t count_modes;
+    uint32_t count_props;
+    uint32_t count_encoders;
+    uint32_t encoder_id;
+    uint32_t connector_id;
+    uint32_t connector_type;
+    uint32_t connector_type_id;
+    uint32_t connection;
+    uint32_t mm_width;
+    uint32_t mm_height;
+    uint32_t subpixel;
+    uint32_t pad;
+};
+
+struct __drm_mode_fb_cmd {
+    uint32_t fb_id;
+    uint32_t width;
+    uint32_t height;
+    uint32_t pitch;
+    uint32_t bpp;
+    uint32_t depth;
+    uint32_t handle;
+};
+
+struct __drm_mode_card_res {
+    uint64_t fb_id_ptr;
+    uint64_t crtc_id_ptr;
+    uint64_t connector_id_ptr;
+    uint64_t encoder_id_ptr;
+    uint32_t count_fbs;
+    uint32_t count_crtcs;
+    uint32_t count_connectors;
+    uint32_t count_encoders;
+    uint32_t min_width;
+    uint32_t max_width;
+    uint32_t min_height;
+    uint32_t max_height;
+};
+
+#define __DRM_CLOEXEC __O_CLOEXEC
+#define __DRM_RDWR    __O_RDWR
+
+struct __drm_prime_handle {
+    uint32_t handle;
+    uint32_t flags;
+    int32_t fd;
+};
+
+struct __drm_gem_close {
+    uint32_t handle;
+    uint32_t pad;
+};
+
+struct __drm_gem_flink {
+    uint32_t handle;
+    uint32_t name;
+};
+
+struct __drm_gem_open {
+    uint32_t name;
+    uint32_t handle;
+    uint64_t size;
+};
+
+struct __drm_mode_closefb {
+    uint32_t fb_id;
+    uint32_t pad;
+};
+
+struct __drm_mode_crtc_lut {
+    uint32_t crtc_id;
+    uint32_t gamma_size;
+    uint64_t red;
+    uint64_t green;
+    uint64_t blue;
+};
+
+struct __drm_mode_list_lessees {
+    uint32_t count_lessees;
+    uint32_t pad;
+    uint64_t lessees_ptr;
+};
+
+struct __drm_mode_cursor {
+    uint32_t flags;
+    uint32_t crtc_id;
+    int32_t x;
+    int32_t y;
+    uint32_t width;
+    uint32_t height;
+    uint32_t handle;
+};
+
+#define __DRM_MODE_CURSOR_BO   0x01
+#define __DRM_MODE_CURSOR_MOVE 0x02
+
+struct __drm_mode_cursor2 {
+    uint32_t flags;
+    uint32_t crtc_id;
+    int32_t x;
+    int32_t y;
+    uint32_t width;
+    uint32_t height;
+    uint32_t handle;
+    int32_t hot_x;
+    int32_t hot_y;
+};
+
+#define __DRM_PROP_NAME_LEN          32
+#define __DRM_MODE_PROP_PENDING      (1U << 0)
+#define __DRM_MODE_PROP_RANGE        (1U << 1)
+#define __DRM_MODE_PROP_IMMUTABLE    (1U << 2)
+#define __DRM_MODE_PROP_ENUM         (1U << 3)
+#define __DRM_MODE_PROP_BLOB         (1U << 4)
+#define __DRM_MODE_PROP_BITMASK      (1U << 5)
+#define __DRM_MODE_PROP_OBJECT       (1U << 6)
+#define __DRM_MODE_PROP_SIGNED_RANGE (1U << 7)
+#define __DRM_MODE_PROP_ATOMIC       (1U << 31)
+
+struct __drm_mode_get_property {
+    uint64_t values_ptr;
+    uint64_t enum_blob_ptr;
+    uint32_t prop_id;
+    uint32_t flags;
+    char name[__DRM_PROP_NAME_LEN];
+    uint32_t count_values;
+    uint32_t count_enum_blobs;
+};
+
+struct __drm_mode_connector_set_property {
+    uint64_t value;
+    uint32_t prop_id;
+    uint32_t connector_id;
+};
+
+struct __drm_mode_get_plane {
+    uint32_t plane_id;
+    uint32_t crtc_id;
+    uint32_t fb_id;
+    uint32_t possible_crtcs;
+    uint32_t gamma_size;
+    uint32_t count_format_types;
+    uint64_t format_type_ptr;
+};
+
+#define __DRM_PLANE_TYPE_OVERLAY 0
+#define __DRM_PLANE_TYPE_PRIMARY 1
+#define __DRM_PLANE_TYPE_CURSOR  2
+
+struct __drm_mode_get_plane_res {
+    uint64_t plane_id_ptr;
+    uint32_t count_planes;
+};
+
+struct __drm_mode_create_dumb {
+    uint32_t height;
+    uint32_t width;
+    uint32_t bpp;
+    uint32_t flags;
+    uint32_t handle;
+    uint32_t pitch;
+    uint64_t size;
+};
+
+struct __drm_mode_get_blob {
+    uint32_t blob_id;
+    uint32_t length;
+    uint64_t data;
+};
+
+struct __drm_mode_crtc_page_flip {
+    uint32_t crtc_id;
+    uint32_t fb_id;
+    uint32_t flags;
+    uint32_t reserved;
+    uint64_t user_data;
+};
+
+struct __drm_crtc_get_sequence {
+    uint32_t crtc_id;
+    uint32_t active;
+    uint64_t sequence;
+    int64_t sequence_ns;
+};
+
+struct __drm_crtc_queue_sequence {
+    uint32_t crtc_id;
+    uint32_t flags;
+    uint64_t sequence;
+    uint64_t user_data;
+};
+
+struct __drm_mode_fb_dirty_cmd {
+    uint32_t fb_id;
+    uint32_t flags;
+    uint32_t color;
+    uint32_t num_clips;
+    uint64_t clips_ptr;
+};
+
+struct __drm_mode_map_dumb {
+    uint32_t handle;
+    uint32_t pad;
+    uint64_t offset;
+};
+
+struct __drm_mode_destroy_dumb {
+    uint32_t handle;
+};
+
+struct __drm_mode_fb_cmd2 {
+    uint32_t fb_id;
+    uint32_t width;
+    uint32_t height;
+    uint32_t pixel_format;
+    uint32_t flags;
+    uint32_t handles[4];
+    uint32_t pitches[4];
+    uint32_t offsets[4];
+    uint64_t modifier[4];
+};
+
+struct __drm_mode_obj_get_properties {
+    uint64_t props_ptr;
+    uint64_t prop_values_ptr;
+    uint32_t count_props;
+    uint32_t obj_id;
+    uint32_t obj_type;
+};
+
+struct __drm_mode_atomic {
+    uint32_t flags;
+    uint32_t count_objs;
+    uint64_t objs_ptr;
+    uint64_t count_props_ptr;
+    uint64_t props_ptr;
+    uint64_t prop_values_ptr;
+    uint64_t reserved;
+    uint64_t user_data;
+};
+
+struct __drm_mode_create_blob {
+    uint64_t data;
+    uint32_t length;
+    uint32_t blob_id;
+};
+
+struct __drm_mode_destroy_blob {
+    uint32_t blob_id;
+};
+
+struct __drm_mode_property_enum {
+    uint64_t value;
+    char name[32];
+};
+
+#define __DRM_MODE_OBJECT_CRTC      0xcccccccc
+#define __DRM_MODE_OBJECT_CONNECTOR 0xc0c0c0c0
+#define __DRM_MODE_OBJECT_ENCODER   0xe0e0e0e0
+#define __DRM_MODE_OBJECT_MODE      0xdededede
+#define __DRM_MODE_OBJECT_PROPERTY  0xb0b0b0b0
+#define __DRM_MODE_OBJECT_FB        0xfbfbfbfb
+#define __DRM_MODE_OBJECT_BLOB      0xbbbbbbbb
+#define __DRM_MODE_OBJECT_PLANE     0xeeeeeeee
+#define __DRM_MODE_OBJECT_ANY       0
+
+#define __DRM_MODE_FLAG_PHSYNC    (1 << 0)
+#define __DRM_MODE_FLAG_NHSYNC    (1 << 1)
+#define __DRM_MODE_FLAG_PVSYNC    (1 << 2)
+#define __DRM_MODE_FLAG_NVSYNC    (1 << 3)
+#define __DRM_MODE_FLAG_INTERLACE (1 << 4)
+#define __DRM_MODE_TYPE_PREFERRED (1 << 3)
+#define __DRM_MODE_TYPE_DRIVER    (1 << 6)
+
+#define __DRM_IOCTL_VERSION                __DRM_IOWR(0x00, struct __drm_version)
+#define __DRM_IOCTL_GET_UNIQUE             __DRM_IOWR(0x01, struct __drm_unique)
+#define __DRM_IOCTL_GET_MAGIC              __DRM_IOR(0x02, struct __drm_auth)
+/*#define __DRM_IOCTL_IRQ_BUSID __DRM_IOWR(0x03, struct __drm_irq_busid)*/
+#define __DRM_IOCTL_GET_MAP                __DRM_IOWR(0x04, struct __drm_map)
+#define __DRM_IOCTL_GET_CLIENT             __DRM_IOWR(0x05, struct __drm_client)
+#define __DRM_IOCTL_GET_STATS              __DRM_IOR(0x06, struct __drm_stats)
+#define __DRM_IOCTL_SET_VERSION            __DRM_IOWR(0x07, struct __drm_set_version)
+/*#define __DRM_IOCTL_MODESET_CTL __DRM_IOW(0x08, struct __drm_modeset_ctl)*/
+#define __DRM_IOCTL_GEM_CLOSE              __DRM_IOW(0x09, struct __drm_gem_close)
+#define __DRM_IOCTL_GEM_FLINK              __DRM_IOWR(0x0a, struct __drm_gem_flink)
+#define __DRM_IOCTL_GEM_OPEN               __DRM_IOWR(0x0b, struct __drm_gem_open)
+#define __DRM_IOCTL_GET_CAP                __DRM_IOWR(0x0c, struct __drm_get_cap)
+#define __DRM_IOCTL_SET_CLIENT_CAP         __DRM_IOW(0x0d, struct __drm_set_client_cap)
+#define __DRM_IOCTL_SET_UNIQUE             __DRM_IOW(0x10, struct __drm_unique)
+#define __DRM_IOCTL_AUTH_MAGIC             __DRM_IOW(0x11, struct __drm_auth)
+/*#define __DRM_IOCTL_BLOCK __DRM_IOWR(0x12, struct __drm_block)*/
+/*#define __DRM_IOCTL_UNBLOCK __DRM_IOWR(0x13, struct __drm_block)*/
+#define __DRM_IOCTL_CONTROL                __DRM_IOW(0x14, struct __drm_control)
+#define __DRM_IOCTL_ADD_MAP                __DRM_IOWR(0x15, struct __drm_map)
+/*#define __DRM_IOCTL_ADD_BUFS __DRM_IOWR(0x16, struct __drm_buf_desc)*/
+/*#define __DRM_IOCTL_MARK_BUFS __DRM_IOW(0x17, struct __drm_buf_desc)*/
+/*#define __DRM_IOCTL_INFO_BUFS __DRM_IOWR(0x18, struct __drm_buf_info)*/
+/*#define __DRM_IOCTL_MAP_BUFS __DRM_IOWR(0x19, struct __drm_buf_map)*/
+/*#define __DRM_IOCTL_FREE_BUFS __DRM_IOW(0x1a, struct __drm_buf_free)*/
+#define __DRM_IOCTL_RM_MAP                 __DRM_IOW(0x1b, struct __drm_map)
+/*#define __DRM_IOCTL_SET_SAREA_CTX __DRM_IOW(0x1c, struct __drm_ctx_priv_map)*/
+/*#define __DRM_IOCTL_GET_SAREA_CTX __DRM_IOWR(0x1d, struct __drm_ctx_priv_map)*/
+#define __DRM_IOCTL_SET_MASTER             __DRM_IO(0x1e)
+#define __DRM_IOCTL_DROP_MASTER            __DRM_IO(0x1f)
+#define __DRM_IOCTL_ADD_CTX                __DRM_IOWR(0x20, struct __drm_ctx)
+#define __DRM_IOCTL_RM_CTX                 __DRM_IOWR(0x21, struct __drm_ctx)
+#define __DRM_IOCTL_MOD_CTX                __DRM_IOW(0x22, struct __drm_ctx)
+#define __DRM_IOCTL_GET_CTX                __DRM_IOWR(0x23, struct __drm_ctx)
+#define __DRM_IOCTL_SWITCH_CTX             __DRM_IOW(0x24, struct __drm_ctx)
+#define __DRM_IOCTL_NEW_CTX                __DRM_IOW(0x25, struct __drm_ctx)
+#define __DRM_IOCTL_RES_CTX                __DRM_IOWR(0x26, struct __drm_ctx_res)
+#define __DRM_IOCTL_ADD_DRAW               __DRM_IOWR(0x27, struct __drm_draw)
+#define __DRM_IOCTL_RM_DRAW                __DRM_IOWR(0x28, struct __drm_draw)
+/*#define __DRM_IOCTL_DMA __DRM_IOWR(0x29, struct __drm_dma)*/
+/*#define __DRM_IOCTL_LOCK __DRM_IOW(0x2a, struct __drm_lock)*/
+/*#define __DRM_IOCTL_UNLOCK __DRM_IOW(0x2b, struct __drm_lock)*/
+/*#define __DRM_IOCTL_FINISH __DRM_IOW(0x2c, struct __drm_lock)*/
+#define __DRM_IOCTL_PRIME_HANDLE_TO_FD     __DRM_IOWR(0x2d, struct __drm_prime_handle)
+#define __DRM_IOCTL_PRIME_FD_TO_HANDLE     __DRM_IOWR(0x2e, struct __drm_prime_handle)
+/*#define __DRM_IOCTL_AGP_ENABLE __DRM_IOW(0x32, struct __drm_agp_mode)*/
+/*#define __DRM_IOCTL_AGP_INFO __DRM_IOR(0x33, struct __drm_agp_info)*/
+/*#define __DRM_IOCTL_AGP_ALLOC __DRM_IOWR(0x34, struct __drm_agp_buffer)*/
+/*#define __DRM_IOCTL_AGP_FREE __DRM_IOW(0x35, struct __drm_agp_buffer)*/
+/*#define __DRM_IOCTL_AGP_BIND __DRM_IOW(0x36, struct __drm_agp_binding)*/
+/*#define __DRM_IOCTL_AGP_UNBIND __DRM_IOW(0x37, struct __drm_agp_binding)*/
+/*#define __DRM_IOCTL_SG_ALLOC __DRM_IOWR(0x38, struct __drm_scatter_gather)*/
+/*#define __DRM_IOCTL_SG_FREE __DRM_IOW(0x39, struct __drm_scatter_gather)*/
+#define __DRM_IOCTL_WAIT_VBLANK            __DRM_IOWR(0x3a, uint64_t[3])
+#define __DRM_IOCTL_CRTC_GET_SEQUENCE      __DRM_IOWR(0x3b, struct __drm_crtc_get_sequence)
+#define __DRM_IOCTL_CRTC_QUEUE_SEQUENCE    __DRM_IOWR(0x3c, struct __drm_crtc_queue_sequence)
+/*#define __DRM_IOCTL_UPDATE_DRAW __DRM_IOW(0x3f, struct __drm_update_draw)*/
+#define __DRM_IOCTL_MODE_GETRESOURCES      __DRM_IOWR(0xA0, struct __drm_mode_card_res)
+#define __DRM_IOCTL_MODE_GETCRTC           __DRM_IOWR(0xA1, struct __drm_mode_crtc)
+#define __DRM_IOCTL_MODE_SETCRTC           __DRM_IOWR(0xA2, struct __drm_mode_crtc)
+#define __DRM_IOCTL_MODE_CURSOR            __DRM_IOWR(0xA3, struct __drm_mode_cursor)
+/*#define __DRM_IOCTL_MODE_GETGAMMA __DRM_IOWR(0xA4, struct __drm_mode_crtc_lut)*/
+#define __DRM_IOCTL_MODE_SETGAMMA          __DRM_IOWR(0xA5, struct __drm_mode_crtc_lut)
+#define __DRM_IOCTL_MODE_GETENCODER        __DRM_IOWR(0xA6, struct __drm_mode_get_encoder)
+#define __DRM_IOCTL_MODE_GETCONNECTOR      __DRM_IOWR(0xA7, struct __drm_mode_get_connector)
+#define __DRM_IOCTL_MODE_GETPROPERTY       __DRM_IOWR(0xAA, struct __drm_mode_get_property)
+#define __DRM_IOCTL_MODE_SETPROPERTY       __DRM_IOWR(0xAB, struct __drm_mode_connector_set_property)
+#define __DRM_IOCTL_MODE_GETPROPBLOB       __DRM_IOWR(0xAC, struct __drm_mode_get_blob)
+#define __DRM_IOCTL_MODE_GETFB             __DRM_IOWR(0xAD, struct __drm_mode_fb_cmd)
+#define __DRM_IOCTL_MODE_ADDFB             __DRM_IOWR(0xAE, struct __drm_mode_fb_cmd)
+#define __DRM_IOCTL_MODE_RMFB              __DRM_IOWR(0xAF, uint32_t)
+#define __DRM_IOCTL_MODE_PAGE_FLIP         __DRM_IOWR(0xB0, struct __drm_mode_crtc_page_flip)
+#define __DRM_IOCTL_MODE_DIRTYFB           __DRM_IOWR(0xB1, struct __drm_mode_fb_dirty_cmd)
+#define __DRM_IOCTL_MODE_CREATE_DUMB       __DRM_IOWR(0xB2, struct __drm_mode_create_dumb)
+#define __DRM_IOCTL_MODE_MAP_DUMB          __DRM_IOWR(0xB3, struct __drm_mode_map_dumb)
+#define __DRM_IOCTL_MODE_DESTROY_DUMB      __DRM_IOWR(0xB4, struct __drm_mode_destroy_dumb)
+#define __DRM_IOCTL_MODE_GETPLANERESOURCES __DRM_IOWR(0xB5, struct __drm_mode_get_plane_res)
+#define __DRM_IOCTL_MODE_GETPLANE          __DRM_IOWR(0xB6, struct __drm_mode_get_plane)
+/*#define __DRM_IOCTL_MODE_SETPLANE __DRM_IOWR(0xB7, struct __drm_mode_set_plane)*/
+#define __DRM_IOCTL_MODE_ADDFB2            __DRM_IOWR(0xB8, struct __drm_mode_fb_cmd2)
+#define __DRM_IOCTL_MODE_OBJ_GETPROPERTIES __DRM_IOWR(0xB9, struct __drm_mode_obj_get_properties)
+/*#define __DRM_IOCTL_MODE_OBJ_SETPROPERTY __DRM_IOWR(0xBA, struct __drm_mode_obj_set_property)*/
+#define __DRM_IOCTL_MODE_CURSOR2           __DRM_IOWR(0xBB, struct __drm_mode_cursor2)
+#define __DRM_IOCTL_MODE_ATOMIC            __DRM_IOWR(0xBC, struct __drm_mode_atomic)
+#define __DRM_IOCTL_MODE_CREATEPROPBLOB    __DRM_IOWR(0xBD, struct __drm_mode_create_blob)
+#define __DRM_IOCTL_MODE_DESTROYPROPBLOB   __DRM_IOWR(0xBE, struct __drm_mode_destroy_blob)
+/*#define __DRM_IOCTL_SYNCOBJ_CREATE __DRM_IOWR(0xBF, struct __drm_syncobj_create)*/
+/*#define __DRM_IOCTL_SYNCOBJ_DESTROY __DRM_IOWR(0xC0, struct __drm_syncobj_destroy)*/
+/*#define __DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD __DRM_IOWR(0xC1, struct __drm_syncobj_handle)*/
+/*#define __DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE __DRM_IOWR(0xC2, struct __drm_syncobj_handle)*/
+/*#define __DRM_IOCTL_SYNCOBJ_WAIT __DRM_IOWR(0xC3, struct __drm_syncobj_wait)*/
+/*#define __DRM_IOCTL_SYNCOBJ_RESET __DRM_IOWR(0xC4, struct __drm_syncobj_array)*/
+/*#define __DRM_IOCTL_SYNCOBJ_SIGNAL __DRM_IOWR(0xC5, struct __drm_syncobj_array)*/
+/*#define __DRM_IOCTL_MODE_CREATE_LEASE __DRM_IOWR(0xC6, struct __drm_mode_create_lease)*/
+#define __DRM_IOCTL_MODE_LIST_LESSEES      __DRM_IOWR(0xC7, struct __drm_mode_list_lessees)
+/*#define __DRM_IOCTL_MODE_GET_LEASE __DRM_IOWR(0xC8, struct __drm_mode_get_lease)*/
+/*#define __DRM_IOCTL_MODE_REVOKE_LEASE __DRM_IOWR(0xC9, struct __drm_mode_revoke_lease)*/
+/*#define __DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT __DRM_IOWR(0xCA, struct __drm_syncobj_timeline_wait)*/
+/*#define __DRM_IOCTL_SYNCOBJ_QUERY __DRM_IOWR(0xCB, struct __drm_syncobj_timeline_array)*/
+/*#define __DRM_IOCTL_SYNCOBJ_TRANSFER __DRM_IOWR(0xCC, struct __drm_syncobj_transfer)*/
+/*#define __DRM_IOCTL_SYNCOBJ_TIMELINE_SIGNAL __DRM_IOWR(0xCD, struct __drm_syncobj_timeline_array)*/
+#define __DRM_IOCTL_MODE_GETFB2            __DRM_IOWR(0xCE, struct __drm_mode_fb_cmd2)
+/*#define __DRM_IOCTL_SYNCOBJ_EVENTFD __DRM_IOWR(0xCF, struct __drm_syncobj_eventfd)*/
+#define __DRM_IOCTL_MODE_CLOSEFB           __DRM_IOWR(0xD0, struct __drm_mode_closefb)
+/*#define __DRM_IOCTL_SET_CLIENT_NAME __DRM_IOWR(0xD1, struct __drm_set_client_name)*/
+/*#define __DRM_IOCTL_GEM_CHANGE_HANDLE __DRM_IOWR(0xD2, struct __drm_gem_change_handle)*/
+
+#define __drm_fourcc_code(a, b, c, d) \
+    ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
+
+#define __DRM_FORMAT_BIG_ENDIAN           (1U << 31)
+#define __DRM_FORMAT_INVALID              0
+#define __DRM_FORMAT_C1                   __drm_fourcc_code('C', '1', ' ', ' ')
+#define __DRM_FORMAT_C2                   __drm_fourcc_code('C', '2', ' ', ' ')
+#define __DRM_FORMAT_C4                   __drm_fourcc_code('C', '4', ' ', ' ')
+#define __DRM_FORMAT_C8                   __drm_fourcc_code('C', '8', ' ', ' ')
+#define __DRM_FORMAT_D1                   __drm_fourcc_code('D', '1', ' ', ' ')
+#define __DRM_FORMAT_D2                   __drm_fourcc_code('D', '2', ' ', ' ')
+#define __DRM_FORMAT_D4                   __drm_fourcc_code('D', '4', ' ', ' ')
+#define __DRM_FORMAT_D8                   __drm_fourcc_code('D', '8', ' ', ' ')
+#define __DRM_FORMAT_R1                   __drm_fourcc_code('R', '1', ' ', ' ')
+#define __DRM_FORMAT_R2                   __drm_fourcc_code('R', '2', ' ', ' ')
+#define __DRM_FORMAT_R4                   __drm_fourcc_code('R', '4', ' ', ' ')
+#define __DRM_FORMAT_R8                   __drm_fourcc_code('R', '8', ' ', ' ')
+#define __DRM_FORMAT_R10                  __drm_fourcc_code('R', '1', '0', ' ')
+#define __DRM_FORMAT_R12                  __drm_fourcc_code('R', '1', '2', ' ')
+#define __DRM_FORMAT_R16                  __drm_fourcc_code('R', '1', '6', ' ')
+#define __DRM_FORMAT_RG88                 __drm_fourcc_code('R', 'G', '8', '8')
+#define __DRM_FORMAT_GR88                 __drm_fourcc_code('G', 'R', '8', '8')
+#define __DRM_FORMAT_RG1616               __drm_fourcc_code('R', 'G', '3', '2')
+#define __DRM_FORMAT_GR1616               __drm_fourcc_code('G', 'R', '3', '2')
+#define __DRM_FORMAT_RGB332               __drm_fourcc_code('R', 'G', 'B', '8')
+#define __DRM_FORMAT_BGR233               __drm_fourcc_code('B', 'G', 'R', '8')
+#define __DRM_FORMAT_XRGB4444             __drm_fourcc_code('X', 'R', '1', '2')
+#define __DRM_FORMAT_XBGR4444             __drm_fourcc_code('X', 'B', '1', '2')
+#define __DRM_FORMAT_RGBX4444             __drm_fourcc_code('R', 'X', '1', '2')
+#define __DRM_FORMAT_BGRX4444             __drm_fourcc_code('B', 'X', '1', '2')
+#define __DRM_FORMAT_ARGB4444             __drm_fourcc_code('A', 'R', '1', '2')
+#define __DRM_FORMAT_ABGR4444             __drm_fourcc_code('A', 'B', '1', '2')
+#define __DRM_FORMAT_RGBA4444             __drm_fourcc_code('R', 'A', '1', '2')
+#define __DRM_FORMAT_BGRA4444             __drm_fourcc_code('B', 'A', '1', '2')
+#define __DRM_FORMAT_XRGB1555             __drm_fourcc_code('X', 'R', '1', '5')
+#define __DRM_FORMAT_XBGR1555             __drm_fourcc_code('X', 'B', '1', '5')
+#define __DRM_FORMAT_RGBX5551             __drm_fourcc_code('R', 'X', '1', '5')
+#define __DRM_FORMAT_BGRX5551             __drm_fourcc_code('B', 'X', '1', '5')
+#define __DRM_FORMAT_ARGB1555             __drm_fourcc_code('A', 'R', '1', '5')
+#define __DRM_FORMAT_ABGR1555             __drm_fourcc_code('A', 'B', '1', '5')
+#define __DRM_FORMAT_RGBA5551             __drm_fourcc_code('R', 'A', '1', '5')
+#define __DRM_FORMAT_BGRA5551             __drm_fourcc_code('B', 'A', '1', '5')
+#define __DRM_FORMAT_RGB565               __drm_fourcc_code('R', 'G', '1', '6')
+#define __DRM_FORMAT_BGR565               __drm_fourcc_code('B', 'G', '1', '6')
+#define __DRM_FORMAT_RGB888               __drm_fourcc_code('R', 'G', '2', '4')
+#define __DRM_FORMAT_BGR888               __drm_fourcc_code('B', 'G', '2', '4')
+#define __DRM_FORMAT_XRGB8888             __drm_fourcc_code('X', 'R', '2', '4')
+#define __DRM_FORMAT_XBGR8888             __drm_fourcc_code('X', 'B', '2', '4')
+#define __DRM_FORMAT_RGBX8888             __drm_fourcc_code('R', 'X', '2', '4')
+#define __DRM_FORMAT_BGRX8888             __drm_fourcc_code('B', 'X', '2', '4')
+#define __DRM_FORMAT_ARGB8888             __drm_fourcc_code('A', 'R', '2', '4')
+#define __DRM_FORMAT_ABGR8888             __drm_fourcc_code('A', 'B', '2', '4')
+#define __DRM_FORMAT_RGBA8888             __drm_fourcc_code('R', 'A', '2', '4')
+#define __DRM_FORMAT_BGRA8888             __drm_fourcc_code('B', 'A', '2', '4')
+#define __DRM_FORMAT_XRGB2101010          __drm_fourcc_code('X', 'R', '3', '0')
+#define __DRM_FORMAT_XBGR2101010          __drm_fourcc_code('X', 'B', '3', '0')
+#define __DRM_FORMAT_RGBX1010102          __drm_fourcc_code('R', 'X', '3', '0')
+#define __DRM_FORMAT_BGRX1010102          __drm_fourcc_code('B', 'X', '3', '0')
+#define __DRM_FORMAT_ARGB2101010          __drm_fourcc_code('A', 'R', '3', '0')
+#define __DRM_FORMAT_ABGR2101010          __drm_fourcc_code('A', 'B', '3', '0')
+#define __DRM_FORMAT_RGBA1010102          __drm_fourcc_code('R', 'A', '3', '0')
+#define __DRM_FORMAT_BGRA1010102          __drm_fourcc_code('B', 'A', '3', '0')
+#define __DRM_FORMAT_XRGB16161616         __drm_fourcc_code('X', 'R', '4', '8')
+#define __DRM_FORMAT_XBGR16161616         __drm_fourcc_code('X', 'B', '4', '8')
+#define __DRM_FORMAT_ARGB16161616         __drm_fourcc_code('A', 'R', '4', '8')
+#define __DRM_FORMAT_ABGR16161616         __drm_fourcc_code('A', 'B', '4', '8')
+#define __DRM_FORMAT_XRGB16161616F        __drm_fourcc_code('X', 'R', '4', 'H')
+#define __DRM_FORMAT_XBGR16161616F        __drm_fourcc_code('X', 'B', '4', 'H')
+#define __DRM_FORMAT_ARGB16161616F        __drm_fourcc_code('A', 'R', '4', 'H')
+#define __DRM_FORMAT_ABGR16161616F        __drm_fourcc_code('A', 'B', '4', 'H')
+#define __DRM_FORMAT_AXBXGXRX106106106106 __drm_fourcc_code('A', 'B', '1', '0')
+#define __DRM_FORMAT_YUYV                 __drm_fourcc_code('Y', 'U', 'Y', 'V')
+#define __DRM_FORMAT_YVYU                 __drm_fourcc_code('Y', 'V', 'Y', 'U')
+#define __DRM_FORMAT_UYVY                 __drm_fourcc_code('U', 'Y', 'V', 'Y')
+#define __DRM_FORMAT_VYUY                 __drm_fourcc_code('V', 'Y', 'U', 'Y')
+#define __DRM_FORMAT_AYUV                 __drm_fourcc_code('A', 'Y', 'U', 'V')
+#define __DRM_FORMAT_AVUY8888             __drm_fourcc_code('A', 'V', 'U', 'Y')
+#define __DRM_FORMAT_XYUV8888             __drm_fourcc_code('X', 'Y', 'U', 'V')
+#define __DRM_FORMAT_XVUY8888             __drm_fourcc_code('X', 'V', 'U', 'Y')
+#define __DRM_FORMAT_VUY888               __drm_fourcc_code('V', 'U', '2', '4')
+#define __DRM_FORMAT_VUY101010            __drm_fourcc_code('V', 'U', '3', '0')
+#define __DRM_FORMAT_Y210                 __drm_fourcc_code('Y', '2', '1', '0')
+#define __DRM_FORMAT_Y212                 __drm_fourcc_code('Y', '2', '1', '2')
+#define __DRM_FORMAT_Y216                 __drm_fourcc_code('Y', '2', '1', '6')
+#define __DRM_FORMAT_Y410                 __drm_fourcc_code('Y', '4', '1', '0')
+#define __DRM_FORMAT_Y412                 __drm_fourcc_code('Y', '4', '1', '2')
+#define __DRM_FORMAT_Y416                 __drm_fourcc_code('Y', '4', '1', '6')
+#define __DRM_FORMAT_XVYU2101010          __drm_fourcc_code('X', 'V', '3', '0')
+#define __DRM_FORMAT_XVYU12_16161616      __drm_fourcc_code('X', 'V', '3', '6')
+#define __DRM_FORMAT_XVYU16161616         __drm_fourcc_code('X', 'V', '4', '8')
+#define __DRM_FORMAT_Y0L0                 __drm_fourcc_code('Y', '0', 'L', '0')
+#define __DRM_FORMAT_X0L0                 __drm_fourcc_code('X', '0', 'L', '0')
+#define __DRM_FORMAT_Y0L2                 __drm_fourcc_code('Y', '0', 'L', '2')
+#define __DRM_FORMAT_X0L2                 __drm_fourcc_code('X', '0', 'L', '2')
+#define __DRM_FORMAT_YUV420_8BIT          __drm_fourcc_code('Y', 'U', '0', '8')
+#define __DRM_FORMAT_YUV420_10BIT         __drm_fourcc_code('Y', 'U', '1', '0')
+#define __DRM_FORMAT_XRGB8888_A8          __drm_fourcc_code('X', 'R', 'A', '8')
+#define __DRM_FORMAT_XBGR8888_A8          __drm_fourcc_code('X', 'B', 'A', '8')
+#define __DRM_FORMAT_RGBX8888_A8          __drm_fourcc_code('R', 'X', 'A', '8')
+#define __DRM_FORMAT_BGRX8888_A8          __drm_fourcc_code('B', 'X', 'A', '8')
+#define __DRM_FORMAT_RGB888_A8            __drm_fourcc_code('R', '8', 'A', '8')
+#define __DRM_FORMAT_BGR888_A8            __drm_fourcc_code('B', '8', 'A', '8')
+#define __DRM_FORMAT_RGB565_A8            __drm_fourcc_code('R', '5', 'A', '8')
+#define __DRM_FORMAT_BGR565_A8            __drm_fourcc_code('B', '5', 'A', '8')
+#define __DRM_FORMAT_NV12                 __drm_fourcc_code('N', 'V', '1', '2')
+#define __DRM_FORMAT_NV21                 __drm_fourcc_code('N', 'V', '2', '1')
+#define __DRM_FORMAT_NV16                 __drm_fourcc_code('N', 'V', '1', '6')
+#define __DRM_FORMAT_NV61                 __drm_fourcc_code('N', 'V', '6', '1')
+#define __DRM_FORMAT_NV24                 __drm_fourcc_code('N', 'V', '2', '4')
+#define __DRM_FORMAT_NV42                 __drm_fourcc_code('N', 'V', '4', '2')
+#define __DRM_FORMAT_NV15                 __drm_fourcc_code('N', 'V', '1', '5')
+#define __DRM_FORMAT_NV20                 __drm_fourcc_code('N', 'V', '2', '0')
+#define __DRM_FORMAT_NV30                 __drm_fourcc_code('N', 'V', '3', '0')
+#define __DRM_FORMAT_P210                 __drm_fourcc_code('P', '2', '1', '0')
+#define __DRM_FORMAT_P010                 __drm_fourcc_code('P', '0', '1', '0')
+#define __DRM_FORMAT_P012                 __drm_fourcc_code('P', '0', '1', '2')
+#define __DRM_FORMAT_P016                 __drm_fourcc_code('P', '0', '1', '6')
+#define __DRM_FORMAT_P030                 __drm_fourcc_code('P', '0', '3', '0')
+#define __DRM_FORMAT_Q410                 __drm_fourcc_code('Q', '4', '1', '0')
+#define __DRM_FORMAT_Q401                 __drm_fourcc_code('Q', '4', '0', '1')
+#define __DRM_FORMAT_YUV410               __drm_fourcc_code('Y', 'U', 'V', '9')
+#define __DRM_FORMAT_YVU410               __drm_fourcc_code('Y', 'V', 'U', '9')
+#define __DRM_FORMAT_YUV411               __drm_fourcc_code('Y', 'U', '1', '1')
+#define __DRM_FORMAT_YVU411               __drm_fourcc_code('Y', 'V', '1', '1')
+#define __DRM_FORMAT_YUV420               __drm_fourcc_code('Y', 'U', '1', '2')
+#define __DRM_FORMAT_YVU420               __drm_fourcc_code('Y', 'V', '1', '2')
+#define __DRM_FORMAT_YUV422               __drm_fourcc_code('Y', 'U', '1', '6')
+#define __DRM_FORMAT_YVU422               __drm_fourcc_code('Y', 'V', '1', '6')
+#define __DRM_FORMAT_YUV444               __drm_fourcc_code('Y', 'U', '2', '4')
+#define __DRM_FORMAT_YVU444               __drm_fourcc_code('Y', 'V', '2', '4')
+
+#endif

include/uapi/ioctls.h

@@ -11,41 +11,13 @@
 #define __IO(group, num)              __IOC(__IOC_VOID, (group), (num), 0)
 #define __IOR(group, num, type)       __IOC(__IOC_OUT, (group), (num), sizeof(type))
 #define __IOW(group, num, type)       __IOC(__IOC_IN, (group), (num), sizeof(type))
+#define __IOWR(group, num, type)      __IOC(__IOC_INOUT, (group), (num), sizeof(type))
+#define __IOWINT(group, num)          __IOC(__IOC_VOID, (group), (num), sizeof(int))
 #define __IOC_GROUP(req)              (((req) >> 8) & 0xFF)
 #define __IOC_NUM(req)                ((req) & 0xFF)
 #define __IOC_LEN(req)                (((req) >> 16) & 0x3FFF)
 
-#define __TCGETS     0x5401
-#define __TCSETS     0x5402
-#define __TCSETSW    0x5403
-#define __TCSETSF    0x5404
-#define __TCFLSH     0x540B
-#define __TIOCSCTTY  0x540E
-#define __TIOCGPGRP  0x540F
-#define __TIOCSPGRP  0x5410
-#define __TIOCGWINSZ 0x5413
-#define __TIOCSWINSZ 0x5414
-#define __FIONREAD   0x541B
-#define __TIOCCONS   0x541D
-#define __TIOCPKT    0x5420
-#define __FIONBIO    0x5421
-#define __TIOCNOTTY  0x5422
-#define __TIOCGSID   0x5429
-#define __TIOCSIG    0x5436
-#define __TIOCGPKT   0x5438
-#define __TIOCGNAME  0x5470
-#define __TIOCGPTN   __IOR('T', 0x30, int)
-#define __TIOCSPTLCK __IOW('T', 0x31, int)
-#define __TIOCGPTLCK 3
-
-#define __BLKGETSIZE64 __IOR('B', 0x00, unsigned long long)
-#define __BLKSSZGET    __IOR('B', 0x01, unsigned int)
-#define __BLKRRPART    __IO('B', 0x02)
-
-#define __TIOCPKT_DATA       0
-#define __TIOCPKT_FLUSHREAD  1
-#define __TIOCPKT_FLUSHWRITE 2
-#define __TIOCPKT_STOP       4
-#define __TIOCPKT_START      8
+#define __FIONREAD __IO('T', 0x1B)
+#define __FIONBIO  __IO('T', 0x21)
 
 #endif

include/uapi/sched.h

@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: Unlicense */
+#ifndef __ZINNIA_UAPI_SCHED_H
+#define __ZINNIA_UAPI_SCHED_H
+
+#define __SCHED_OTHER 0
+#define __SCHED_FIFO  1
+#define __SCHED_RR    2
+
+struct __sched_param {
+    int sched_priority;
+    int sched_priority_min;
+    int sched_priority_max;
+};
+
+#endif

include/uapi/signal.h

@@ -57,9 +57,13 @@
 #define __SA_NOCLDWAIT (1 << 5)
 #define __SA_NODEFER   (1 << 6)
 
+#if defined(__aarch64__)
+#define __MINSIGSTKSZ 5120
+#else
 #define __MINSIGSTKSZ 2048
-#define __SS_ONSTACK  1
-#define __SS_DISABLE  2
+#endif
+#define __SS_ONSTACK 1
+#define __SS_DISABLE 2
 
 #define __SIGEV_NONE   1
 #define __SIGEV_SIGNAL 2
@@ -105,7 +109,7 @@ struct __siginfo {
 
 struct __sigaction {
     uintptr_t sa_handler;
-    unsigned long sa_flags;
+    int sa_flags;
     uintptr_t sa_restorer;
     __sigset_t sa_mask;
 };

include/uapi/socket.h

@@ -152,6 +152,9 @@ struct __ucred {
 #define __IPPROTO_UDP  17
 #define __IPPROTO_RAW  255
 
+#define __TCP_NODELAY 1
+#define __TCP_MAXSEG  2
+
 #define __SHUT_RD   1
 #define __SHUT_WR   2
 #define __SHUT_RDWR (__SHUT_RD | __SHUT_WR)

include/uapi/termios.h

@@ -2,6 +2,7 @@
 #ifndef __ZINNIA_UAPI_TERMIOS_H
 #define __ZINNIA_UAPI_TERMIOS_H
 
+#include <uapi/ioctls.h>
 #include <uapi/types.h>
 
 #define __NCCS 32
@@ -93,4 +94,31 @@ struct __winsize {
     unsigned short ws_ypixel;
 };
 
+#define __TCGETS     __IO('T', 0x01)
+#define __TCSETS     __IO('T', 0x02)
+#define __TCSETSW    __IO('T', 0x03)
+#define __TCSETSF    __IO('T', 0x04)
+#define __TCFLSH     __IO('T', 0x0B)
+#define __TIOCSCTTY  __IO('T', 0x0E)
+#define __TIOCGPGRP  __IO('T', 0x0F)
+#define __TIOCSPGRP  __IO('T', 0x10)
+#define __TIOCGWINSZ __IO('T', 0x13)
+#define __TIOCSWINSZ __IO('T', 0x14)
+#define __TIOCCONS   __IO('T', 0x1D)
+#define __TIOCPKT    __IO('T', 0x20)
+#define __TIOCNOTTY  __IO('T', 0x22)
+#define __TIOCGSID   __IO('T', 0x29)
+#define __TIOCSIG    __IO('T', 0x36)
+#define __TIOCGPKT   __IO('T', 0x38)
+#define __TIOCGNAME  __IO('T', 0x70)
+#define __TIOCGPTN   __IOR('T', 0x30, int)
+#define __TIOCSPTLCK __IOW('T', 0x31, int)
+#define __TIOCGPTLCK 3
+
+#define __TIOCPKT_DATA       0
+#define __TIOCPKT_FLUSHREAD  1
+#define __TIOCPKT_FLUSHWRITE 2
+#define __TIOCPKT_STOP       4
+#define __TIOCPKT_START      8
+
 #endif

include/zinnia/amap.h

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/mutex.h>
+#include <zinnia/radix.h>
+#include <zinnia/refcount.h>
+#include <zinnia/virt.h>
+#include <stddef.h>
+
+struct vm_anon {
+    _Atomic(size_t) rc;
+    struct page* page;
+};
+
+struct amap {
+    struct refcount rc;
+    mutex_t lock;
+    struct radix_tree anons;
+};
+
+errno_t anon_new(struct page* src, struct vm_anon** out);
+void anon_ref(struct vm_anon* anon);
+void anon_unref(struct vm_anon* anon);
+
+errno_t amap_new(struct amap** out);
+void amap_ref(struct amap* amap);
+void amap_unref(struct amap* amap);
+bool amap_is_shared(struct amap* amap);
+errno_t amap_copy(struct amap* src, size_t offset, size_t num_slots, struct amap** out);

include/zinnia/bio.h

@@ -1,18 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-#include <zinnia/mman.h>
-
-// TODO
-
-enum bio_type : uint8_t {
-    BLOCK_READ,
-    BLOCK_WRITE,
-};
-
-struct bio {
-    enum bio_type type;
-    uint32_t num_lbas;
-    uint64_t lba;
-};

include/zinnia/blockdev.h

@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/event.h>
+#include <zinnia/guid.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/types.h>
+#include <zinnia/vfs.h>
+#include <zinnia/vmo.h>
+#include <stddef.h>
+#include <stdint.h>
+
+enum bio_op : uint8_t {
+    BIO_READ,
+    BIO_WRITE,
+    BIO_FLUSH,
+};
+
+struct bio_seg {
+    phys_t addr;
+    size_t len;
+};
+
+struct blockdev;
+
+// A block IO operation.
+struct bio {
+    enum bio_op op;               // What to do.
+    uint64_t lba;                 // Starting block.
+    size_t num_lbas;              // Amount of blocks to read.
+    struct blockdev* dev;         // Device to use for this bio.
+    errno_t status;               // Result of this operation.
+    size_t done_lbas;             // Current progress.
+    void (*end)(struct bio* bio); // Callback which runs when the bio completes.
+    void* priv;                   // Driver data.
+    struct bio* parent;           // For nested bios.
+    uint64_t parent_offset;
+    _Atomic(size_t) pending;
+    _Atomic(uint64_t) short_offset;
+    _Atomic(errno_t) child_status;
+    struct event event; // Event for poll.
+    size_t num_segs;    // Counter for `segs`.
+    struct bio_seg segs[];
+};
+
+struct blockdev_limits {
+    size_t max_lbas;
+    size_t max_segs;
+};
+
+struct blockdev_ops {
+    // Submit a bio for execution on this block device.
+    errno_t (*submit)(struct blockdev* dev, struct bio* bio);
+    // Handles an ioctl for the block device.
+    errno_t (*ioctl)(struct blockdev* dev, struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out);
+    void (*destroy)(struct blockdev* dev);
+};
+
+struct blockdev {
+    struct vmo cache;
+    const struct blockdev_ops* ops;
+    dev_t dev;
+    size_t lba_size;
+    uint64_t lba_count;
+    struct blockdev_limits limits;
+    _Atomic(bool) claimed;
+};
+
+struct bio* bio_alloc(enum bio_op op, size_t num_segs);
+
+void bio_free(struct bio* bio);
+
+// Submit a bio to run on `dev`.
+void bio_submit(struct blockdev* dev, struct bio* bio);
+
+void bio_complete(struct bio* bio, errno_t status, size_t done_lbas);
+
+errno_t bio_wait(struct bio* bio, size_t* out_lbas);
+
+void blockdev_init(
+    struct blockdev* dev,
+    const struct blockdev_ops* ops,
+    uint32_t major,
+    uint32_t minor,
+    size_t lba_size,
+    uint64_t lba_count
+);
+void blockdev_ref(struct blockdev* dev);
+void blockdev_unref(struct blockdev* dev);
+
+// Performs IO on a block device using `bio_seg` segments.
+errno_t blockdev_rw(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    const struct bio_seg* segs,
+    size_t num_segs,
+    size_t* out_lbas
+);
+
+// Performs IO on a block device using pages.
+errno_t blockdev_rw_pages(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    struct page** pages,
+    size_t count,
+    size_t* out_lbas
+);
+
+// Flushes out all outstanding bios on `dev`.
+errno_t blockdev_flush(struct blockdev* dev);
+
+errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out);
+errno_t blockdev_claim(struct blockdev* dev);
+void blockdev_release(struct blockdev* dev);
+
+// Adds a new block device to the list of known devices.
+// Will also add a node in `/dev/block`.
+errno_t blockdev_register(const char* name, struct blockdev* dev);
+
+// Rescans `dev` for partitions or file systems.
+errno_t blockdev_rescan(struct blockdev* dev);
+
+// Syncs all outstanding requests blockdevs
+errno_t blockdev_sync_all();
+
+errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode);
+
+errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out);
+
+struct gpt_partition {
+    struct guid type_guid;
+    struct guid unique_guid;
+    uint64_t start_lba;
+    uint64_t end_lba;
+};
+
+errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count);
+
+errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out);
+
+INIT_EXTERN(block_stage);

include/zinnia/chardev.h

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/refcount.h>
+#include <zinnia/types.h>
+#include <zinnia/vfs.h>
+#include <stdint.h>
+
+static inline dev_t dev_make(uint32_t major, uint32_t minor) {
+    return (minor & 0xFF) | ((major & 0xFFF) << 8) | ((dev_t)(minor & ~0xFFU) << 12) | ((dev_t)(major & ~0xFFFU) << 32);
+}
+
+struct chardev;
+
+struct chardev_ops {
+    errno_t (*open)(struct chardev* dev, struct vfs_file* file);
+    void (*destroy)(struct chardev* dev);
+};
+
+struct chardev {
+    struct refcount rc;
+    const struct chardev_ops* ops;
+    const struct vfs_file_ops* file_ops;
+    dev_t dev;
+};
+
+void chardev_init(
+    struct chardev* dev,
+    const struct chardev_ops* ops,
+    const struct vfs_file_ops* file_ops,
+    uint32_t major,
+    uint32_t minor
+);
+void chardev_ref(struct chardev* dev);
+void chardev_unref(struct chardev* dev);
+
+struct vfs_path devtmpfs_root();
+errno_t devtmpfs_add_chardev(const char* name, struct chardev* dev, mode_t mode);
+errno_t devtmpfs_mkdir(const char* name, mode_t mode);
+errno_t devtmpfs_remove(const char* name);
+errno_t devtmpfs_symlink(const char* name, const char* target);
+
+INIT_EXTERN(devtmpfs_stage);

include/zinnia/condvar.h

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/mutex.h>
+#include <stdint.h>
+
+struct condvar {
+    _Atomic(uint32_t) seq;
+    _Atomic(uint32_t) waiters;
+};
+
+void condvar_init(struct condvar* cv);
+void condvar_wait(struct condvar* cv, mutex_t* mutex);
+void condvar_signal(struct condvar* cv);
+void condvar_broadcast(struct condvar* cv);

include/zinnia/devctl.h

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+void devctl_notify_create(const char* name);
+void devctl_notify_destroy(const char* name);

include/zinnia/dpc.h

@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-#include <zinnia/atomic.h>
-#include <zinnia/list.h>
-#include <zinnia/percpu.h>
-
-struct dpc {
-    void (*callback)(struct dpc* dpc, void* data);
-    void* data;
-    atomic_bool inserted;
-
-    list_link(struct dpc*) link;
-};
-
-void dpc_dispatch();
-
-struct dpc_percpu {};
-extern struct dpc_percpu dpc_percpu_data;

include/zinnia/event.h

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+struct event {
+    _Atomic(uint32_t) state;
+};
+
+void event_init(struct event* event);
+
+// Sets the event and wakes all waiters.
+void event_set(struct event* event);
+
+// Clears the event, so that future waiters block again.
+void event_reset(struct event* event);
+
+bool event_is_set(struct event* event);
+
+// Blocks until the event is set.
+void event_wait(struct event* event);

include/zinnia/exec.h

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/tailq.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct process;
+struct vfs_file;
+struct vma;
+struct vmo;
+
+// Information passed to `exec_format.load`.
+struct exec_info {
+    // An address space for the new process.
+    struct vma* space;
+    // The excutable to load.
+    struct vmo* file_obj;
+    // Path passed to execve.
+    const char* exec_path;
+    // Arguments.
+    const char** argv;
+    // Environment variables.
+    const char** envp;
+
+    size_t depth;
+};
+
+// An executable format.
+struct exec_format {
+    const char* name;
+
+    // Identifies whether a file is a valid executable of this format.
+    bool (*identify)(struct vmo* obj);
+
+    // Loads the executable and returns a new initial thread.
+    // The implementation should not modify `proc` at all.
+    errno_t (*load)(struct process* proc, struct exec_info* info, uintptr_t* out_ip, uintptr_t* out_sp);
+
+    tailq_link(struct exec_format) link;
+};
+
+// Attempts to identify the format of this executable file.
+struct exec_format* exec_identify(struct vmo* obj);
+
+// Installs a new executable format.
+void exec_register(struct exec_format* format);
+
+errno_t exec_open(struct process* proc, const char* path, struct vmo** out);
+
+errno_t exec_file_object(struct process* proc, struct vfs_file* file, struct vmo** out);

include/zinnia/ident.h

@@ -15,3 +15,5 @@ struct ident {
 const struct ident* ident_kernel(void);
 
 bool ident_in_group(const struct ident* identity, gid_t gid, bool use_real);
+
+bool ident_is_privileged(const struct ident* identity, bool use_real);

include/zinnia/initramfs.h

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/vfs.h>
+#include <stddef.h>
+
+struct boot_info;
+
+bool initramfs_is_valid(const void* data, size_t len);
+
+errno_t initramfs_load(struct vfs_path root, struct vfs_path target, const void* data, size_t len);
+
+void initramfs_load_boot_modules(struct boot_info* info);

include/zinnia/pager.h

@@ -1,10 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-#include <zinnia/errno.h>
-#include <stdint.h>
-
-struct pager {
-    errno_t (*try_get_page)(struct pager* pager, uintptr_t offset);
-};

include/zinnia/park.h

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct task;
+
+enum park_status {
+    PARK_UNPARKED,
+    PARK_INVALID,
+    PARK_TIMED_OUT,
+    PARK_INTERRUPTED,
+};
+
+struct park_result {
+    enum park_status status;
+    uintptr_t token;
+};
+
+enum park_flags {
+    // Queue in front of waiters with the same priority.
+    PARK_FRONT = 1 << 0,
+    PARK_INTERRUPTIBLE = 1 << 1,
+};
+
+struct park_unpark_result {
+    struct task* task; // The task that is being woken up, or nullptr if there was no waiter.
+    bool have_more;    // Whether there are more tasks parked on the same key.
+};
+
+// Parks the current task on `key` if `validate(ctx)` returns true.
+// `validate` is called with the queue for `key` locked, so it must be short and must not sleep.
+// Returns the token passed by the waker, or `PARK_INVALID` if `validate` returned false.
+// Must not be called while holding a spinlock.
+struct park_result park(const void* key, bool (*validate)(void* ctx), void* ctx, enum park_flags flags);
+
+// Very similar to `park`, but with an optional `deadline_ns`.
+// If the deadline is missed, `cancelled` gets called (also optional).
+struct park_result park_timeout(
+    const void* key,
+    bool (*validate)(void* ctx),
+    void (*cancelled)(void* ctx, bool was_last),
+    void* ctx,
+    enum park_flags flags,
+    const uint64_t* deadline_ns
+);
+
+// Wakes the first task parked on `key`.
+// `callback`, if set, is called with the queue locked before the task is woken, even if there was no waiter.
+// Its return value is passed to the woken task as the token. Without a callback, the token is 0.
+// Returns true if a task was woken.
+bool unpark_one(const void* key, uintptr_t (*callback)(void* ctx, const struct park_unpark_result* result), void* ctx);
+
+// Wakes all tasks parked on `key` and passes `token` to them. Returns the amount of tasks woken.
+size_t unpark_all(const void* key, uintptr_t token);
+
+// Parks the current task as long as `*addr == expected`. Returns false if the value didn't match.
+bool park_wait(atomic_uint* addr, unsigned int expected);
+
+// Wakes up to `count` tasks waiting in park_wait() on `addr`. Returns the amount of tasks woken.
+size_t park_wake(atomic_uint* addr, size_t count);

include/zinnia/pipe.h

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/ident.h>
+#include <zinnia/vfs.h>
+
+errno_t pipe_create(
+    enum open_flags flags,
+    const struct ident* identity,
+    struct vfs_file** out_read,
+    struct vfs_file** out_write
+);

include/zinnia/poll.h

@@ -2,7 +2,11 @@
 
 #pragma once
 
-#include <zinnia/list.h>
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/mutex.h>
+#include <zinnia/spin.h>
+#include <zinnia/tailq.h>
 #include <stdint.h>
 #include <uapi/poll.h>
 
@@ -20,7 +24,39 @@ enum poll_flags : uint16_t {
     POLLWRBAND = __POLLWRBAND,
 };
 
-struct poll_event {
-    // TODO
-    list_link(struct poll_event*) next;
+struct task;
+struct poll_table;
+struct poll_event;
+
+struct poll_waiter {
+    struct poll_event* event;
+    struct poll_table* table;
+    tailq_link(struct poll_waiter) link;
+    struct poll_waiter* next;
 };
+
+struct poll_event {
+    struct spinlock lock;
+    tailq_head(struct poll_waiter) waiters;
+};
+
+struct poll_table {
+    struct task* task;
+    void (*notify)(struct poll_table* table);
+    atomic_bool triggered;
+    struct poll_waiter* waiters;
+    struct poll_waiter inline_waiter;
+    bool inline_used;
+    bool interruptible;
+    errno_t error;
+};
+
+void poll_event_init(struct poll_event* event);
+void poll_event_notify(struct poll_event* event);
+
+void poll_table_init(struct poll_table* table);
+void poll_table_init_callback(struct poll_table* table, void (*notify)(struct poll_table* table));
+void poll_table_fini(struct poll_table* table);
+void poll_wait(struct poll_table* table, struct poll_event* event);
+errno_t poll_table_wait(struct poll_table* table, const uint64_t* deadline_ns);
+errno_t poll_table_sleep(struct poll_table* table, mutex_t* lock, const uint64_t* deadline_ns);

include/zinnia/process.h

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/ident.h>
+#include <zinnia/mutex.h>
+#include <zinnia/refcount.h>
+#include <zinnia/spin.h>
+#include <zinnia/tailq.h>
+#include <zinnia/types.h>
+#include <zinnia/vfs.h>
+#include <stdint.h>
+#include <uapi/resource.h>
+
+constexpr size_t PROCESS_DEFAULT_FDS = 1024;
+constexpr size_t PROCESS_MAX_FDS = 1 << 20;
+
+struct task;
+struct vma;
+
+struct process_rlimit {
+    _Atomic(rlim_t) cur;
+    rlim_t max;
+};
+
+struct fd_table {
+    struct vfs_file** files;
+    uint64_t* open;
+    uint64_t* cloexec;
+    size_t capacity;
+    size_t next_free;
+};
+
+struct process {
+    struct refcount rc;
+    pid_t pid;
+
+    struct spinlock tasks_lock;
+    tailq_head(struct task) tasks;
+    size_t thread_count;
+
+    mutex_t lock;
+    struct vma* space;
+    struct vfs_path root;
+    struct vfs_path cwd;
+    struct ident identity;
+    mode_t umask;
+    int8_t nice;
+
+    struct process_rlimit limit_nofile;
+
+    mutex_t files_lock;
+    struct fd_table files;
+};
+
+struct process* process_kernel();
+void process_ref(struct process* proc);
+void process_unref(struct process* proc);
+
+uint8_t process_priority(struct process* proc);
+void process_attach_task(struct process* proc, struct task* task);
+void process_detach_task(struct process* proc, struct task* task);
+
+errno_t process_spawn_init(const char* path, const char** argv, const char** envp);
+
+size_t process_fd_limit(struct process* proc);
+
+// Allocate a file drescriptor for `file` on `proc`.
+errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloexec, int* out_fd);
+
+// Allocate a file descriptor for `file` on `proc`, starting at at least `min_fd`.
+errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int min_fd, bool cloexec, int* out_fd);
+
+// Replaces the current file indentified by `fd` in `proc` with a new `file`.
+errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, bool cloexec);
+errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec);
+errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec);
+struct vfs_file* process_fd_get(struct process* proc, int fd);
+errno_t process_fd_close(struct process* proc, int fd);

include/zinnia/sched.h

@@ -40,6 +40,7 @@ constexpr size_t SCHED_TIME_SLICE_NS = 10'000'000;
 constexpr uint8_t SCHED_FEEDBACK_DEPTH = 8;
 constexpr uint64_t SCHED_BOOST_INTERVAL_NS = 1'000'000'000;
 
+struct process;
 struct turnstile;
 
 constexpr size_t TASK_NAME_MAX = 128;
@@ -49,6 +50,8 @@ struct task {
     size_t id;
     char name[TASK_NAME_MAX];
     struct vma* space;
+    struct process* process;
+    tailq_link(struct task) process_link;
     enum task_state state;
     struct arch_task_context context;
     struct usercopy_region* usercopy_region;
@@ -105,7 +108,7 @@ struct scheduler {
 
 typedef void (*task_fn_t)(uintptr_t arg0);
 
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out);
+errno_t task_create(const char* name, struct process* proc, task_fn_t entry, uintptr_t arg0, struct task** out);
 void task_destroy(struct task* task);
 void task_lock(struct task* task);
 void task_unlock(struct task* task);
@@ -127,6 +130,8 @@ void sched_wake(struct task* task);
 // Same as sched_wake(), but the task must be locked with task_lock().
 void sched_wake_locked(struct task* task);
 
+void sched_notify(struct task* task);
+
 [[noreturn]]
 void sched_exit();
 

include/zinnia/signal.h

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+struct task;
+
+// TODO
+static inline bool signal_pending(struct task*) {
+    return false;
+}

include/zinnia/syscall_list.inc

@@ -8,10 +8,10 @@ SYSCALL_DECLARE(2, getuname, 1)
 SYSCALL_DECLARE(3, setuname, 1)
 SYSCALL_DECLARE(4, archctl, 2)
 // SYSCALL_DECLARE(5, reboot, -1)
-// SYSCALL_DECLARE(6, mmap, -1)
-// SYSCALL_DECLARE(7, munmap, -1)
-// SYSCALL_DECLARE(8, mprotect, -1)
-// SYSCALL_DECLARE(9, madvise, -1)
+SYSCALL_DECLARE(6, mmap, 6)
+SYSCALL_DECLARE(7, munmap, 2)
+SYSCALL_DECLARE(8, mprotect, 3)
+SYSCALL_DECLARE(9, madvise, 3)
 // SYSCALL_DECLARE(10, sigprocmask, -1)
 // SYSCALL_DECLARE(11, sigsuspend, -1)
 // SYSCALL_DECLARE(12, sigpending, -1)
@@ -29,47 +29,47 @@ SYSCALL_DECLARE(4, archctl, 2)
 // SYSCALL_DECLARE(24, getppid, -1)
 // SYSCALL_DECLARE(25, waitid, -1)
 // SYSCALL_DECLARE(26, waitpid, -1)
-// SYSCALL_DECLARE(27, pread, -1)
-// SYSCALL_DECLARE(28, readv, -1)
-// SYSCALL_DECLARE(29, pwrite, -1)
-// SYSCALL_DECLARE(30, writev, -1)
-// SYSCALL_DECLARE(31, seek, -1)
-// SYSCALL_DECLARE(32, ioctl, -1)
-// SYSCALL_DECLARE(33, openat, -1)
-// SYSCALL_DECLARE(34, close, -1)
-// SYSCALL_DECLARE(35, fstat, -1)
-// SYSCALL_DECLARE(36, fstatat, -1)
-// SYSCALL_DECLARE(37, statvfs, -1)
-// SYSCALL_DECLARE(38, fstatvfs, -1)
-// SYSCALL_DECLARE(39, faccessat, -1)
-// SYSCALL_DECLARE(40, fcntl, -1)
-// SYSCALL_DECLARE(41, ftruncate, -1)
-// SYSCALL_DECLARE(42, truncate, -1)
-// SYSCALL_DECLARE(43, fallocate, -1)
-// SYSCALL_DECLARE(44, fadvise, -1)
-// SYSCALL_DECLARE(45, utimensat, -1)
-// SYSCALL_DECLARE(46, mknodat, -1)
-// SYSCALL_DECLARE(47, getcwd, -1)
-// SYSCALL_DECLARE(48, chdir, -1)
-// SYSCALL_DECLARE(49, fchdir, -1)
-// SYSCALL_DECLARE(50, mkdirat, -1)
-// SYSCALL_DECLARE(51, rmdirat, -1)
-// SYSCALL_DECLARE(52, getdents, -1)
-// SYSCALL_DECLARE(53, renameat, -1)
-// SYSCALL_DECLARE(54, fchmod, -1)
-// SYSCALL_DECLARE(55, fchmodat, -1)
-// SYSCALL_DECLARE(56, fchownat, -1)
-// SYSCALL_DECLARE(57, linkat, -1)
-// SYSCALL_DECLARE(58, symlinkat, -1)
-// SYSCALL_DECLARE(59, unlinkat, -1)
-// SYSCALL_DECLARE(60, readlinkat, -1)
-// SYSCALL_DECLARE(61, flock, -1)
-// SYSCALL_DECLARE(62, ppoll, -1)
-// SYSCALL_DECLARE(63, dup, -1)
-// SYSCALL_DECLARE(64, dup3, -1)
-// SYSCALL_DECLARE(65, sync, -1)
-// SYSCALL_DECLARE(66, fsync, -1)
-// SYSCALL_DECLARE(67, fdatasync, -1)
+SYSCALL_DECLARE(27, pread, 4)
+SYSCALL_DECLARE(28, readv, 3)
+SYSCALL_DECLARE(29, pwrite, 4)
+SYSCALL_DECLARE(30, writev, 3)
+SYSCALL_DECLARE(31, seek, 3)
+SYSCALL_DECLARE(32, ioctl, 3)
+SYSCALL_DECLARE(33, openat, 4)
+SYSCALL_DECLARE(34, close, 1)
+SYSCALL_DECLARE(35, fstat, 2)
+SYSCALL_DECLARE(36, fstatat, 4)
+SYSCALL_DECLARE(37, statvfs, 2)
+SYSCALL_DECLARE(38, fstatvfs, 2)
+SYSCALL_DECLARE(39, faccessat, 4)
+SYSCALL_DECLARE(40, fcntl, 3)
+SYSCALL_DECLARE(41, ftruncate, 2)
+SYSCALL_DECLARE(42, truncate, 2)
+SYSCALL_DECLARE(43, fallocate, 3)
+SYSCALL_DECLARE(44, fadvise, 4)
+SYSCALL_DECLARE(45, utimensat, 4)
+SYSCALL_DECLARE(46, mknodat, 4)
+SYSCALL_DECLARE(47, getcwd, 2)
+SYSCALL_DECLARE(48, chdir, 1)
+SYSCALL_DECLARE(49, fchdir, 1)
+SYSCALL_DECLARE(50, mkdirat, 3)
+SYSCALL_DECLARE(51, rmdirat, 2)
+SYSCALL_DECLARE(52, getdents, 3)
+SYSCALL_DECLARE(53, renameat, 4)
+SYSCALL_DECLARE(54, fchmod, 2)
+SYSCALL_DECLARE(55, fchmodat, 4)
+SYSCALL_DECLARE(56, fchownat, 5)
+SYSCALL_DECLARE(57, linkat, 5)
+SYSCALL_DECLARE(58, symlinkat, 3)
+SYSCALL_DECLARE(59, unlinkat, 3)
+SYSCALL_DECLARE(60, readlinkat, 4)
+SYSCALL_DECLARE(61, flock, 2)
+SYSCALL_DECLARE(62, ppoll, 4)
+SYSCALL_DECLARE(63, dup, 1)
+SYSCALL_DECLARE(64, dup3, 3)
+SYSCALL_DECLARE(65, sync, 0)
+SYSCALL_DECLARE(66, fsync, 1)
+SYSCALL_DECLARE(67, fdatasync, 1)
 // SYSCALL_DECLARE(68, getgroups, -1)
 // SYSCALL_DECLARE(69, setgroups, -1)
 // SYSCALL_DECLARE(70, getsid, -1)
@@ -90,8 +90,8 @@ SYSCALL_DECLARE(4, archctl, 2)
 // SYSCALL_DECLARE(85, setresgid, -1)
 // SYSCALL_DECLARE(86, setreuid, -1)
 // SYSCALL_DECLARE(87, setregid, -1)
-// SYSCALL_DECLARE(88, umask, -1)
-// SYSCALL_DECLARE(89, pipe, -1)
+SYSCALL_DECLARE(88, umask, 1)
+SYSCALL_DECLARE(89, pipe, 2)
 // SYSCALL_DECLARE(90, futex_wait, -1)
 // SYSCALL_DECLARE(91, futex_wake, -1)
 // SYSCALL_DECLARE(92, thread_create, -1)
@@ -111,9 +111,9 @@ SYSCALL_DECLARE(4, archctl, 2)
 // SYSCALL_DECLARE(106, clock_set, -1)
 // SYSCALL_DECLARE(107, sleep, -1)
 // SYSCALL_DECLARE(108, yield, -1)
-// SYSCALL_DECLARE(109, chroot, -1)
-// SYSCALL_DECLARE(110, mount, -1)
-// SYSCALL_DECLARE(111, umount, -1)
+SYSCALL_DECLARE(109, chroot, 1)
+SYSCALL_DECLARE(110, mount, 4)
+SYSCALL_DECLARE(111, umount, 2)
 // SYSCALL_DECLARE(112, socket, -1)
 // SYSCALL_DECLARE(113, socketpair, -1)
 // SYSCALL_DECLARE(114, shutdown, -1)
@@ -145,5 +145,5 @@ SYSCALL_DECLARE(4, archctl, 2)
 // SYSCALL_DECLARE(140, timerfd_settime, -1)
 // SYSCALL_DECLARE(141, signalfd_create, -1)
 // SYSCALL_DECLARE(142, eventfd_create, -1)
-// SYSCALL_DECLARE(143, msync, -1)
+SYSCALL_DECLARE(143, msync, 3)
 // SYSCALL_DECLARE(144, sysconf, -1)

include/zinnia/vfs.h

@@ -4,9 +4,11 @@
 
 #include <zinnia/errno.h>
 #include <zinnia/ident.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/iovec.h>
 #include <zinnia/mman.h>
 #include <zinnia/mutex.h>
+#include <zinnia/poll.h>
 #include <zinnia/refcount.h>
 #include <zinnia/spin.h>
 #include <zinnia/tailq.h>
@@ -34,6 +36,7 @@ struct vfs_super_block;
 struct vfs_filesystem;
 struct vfs_node_ops;
 struct vfs_file_ops;
+struct pipe;
 
 enum open_flags : uint32_t {
     O_RDONLY = __O_RDONLY,
@@ -62,6 +65,13 @@ enum open_flags : uint32_t {
     O_ACCMODE = __O_ACCMODE,
 };
 
+enum access_flags : uint32_t {
+    F_OK = __F_OK,
+    X_OK = __X_OK,
+    W_OK = __W_OK,
+    R_OK = __R_OK,
+};
+
 enum mount_flags : uint32_t {
     MNT_RDONLY = __MNT_RDONLY,
     MNT_NOSUID = __MNT_NOSUID,
@@ -72,13 +82,6 @@ enum mount_flags : uint32_t {
     MNT_FORCE = __MNT_FORCE,
 };
 
-struct exec_info {
-    struct vma* space;
-    struct vmo* file_obj;
-    const char** argv;
-    const char** envp;
-};
-
 enum vfs_lookup_flags : uint32_t {
     VFS_LOOKUP_FOLLOW_SYMLINKS = 1 << 0,
     VFS_LOOKUP_MUST_EXIST = 1 << 1,
@@ -103,6 +106,7 @@ enum vfs_entry_state {
     VFS_ENTRY_NOT_PRESENT, // Entry is negative.
 };
 
+// A namecache entry.
 struct vfs_entry {
     struct refcount rc;
     // The name of this entry.
@@ -111,12 +115,12 @@ struct vfs_entry {
     mutex_t lock;
 
     enum vfs_entry_state state;
-    // The inode this entry points to when present.
+    // The inode this entry points to when present. Holds a reference.
     struct vfs_node* inode;
-    // The parent of this entry, weak reference.
+    // The parent of this entry, strong reference. `nullptr` for file system roots.
     struct vfs_entry* parent;
 
-    // All cached children of this entry.
+    // All cached children of this entry. Each child in this list holds a reference.
     tailq_head(struct vfs_entry) children;
     tailq_link(struct vfs_entry) child_link;
 
@@ -132,6 +136,22 @@ struct vfs_path {
 struct vfs_path vfs_path_clone(struct vfs_path path);
 void vfs_path_drop(struct vfs_path path);
 
+// Allocates a new, uncached entry with the given `name`. `parent` gets referenced if non-null.
+errno_t vfs_entry_new(const char* name, size_t name_len, struct vfs_entry* parent, struct vfs_entry** out);
+
+// Makes `entry` positive and points it to `node`. Takes a new reference to `node`.
+void vfs_entry_set_inode(struct vfs_entry* entry, struct vfs_node* node);
+
+// Returns a new reference to the inode of `entry`, or `nullptr` if the entry is negative.
+struct vfs_node* vfs_entry_get_inode(struct vfs_entry* entry);
+
+// Makes `entry` negative and drops its reference to the inode.
+void vfs_entry_set_negative(struct vfs_entry* entry);
+
+void vfs_entry_replace_children(struct vfs_entry* dst, struct vfs_entry* src);
+void vfs_entry_prune(struct vfs_entry* entry);
+
+// A mount point.
 struct vfs_mount {
     struct refcount rc;
     enum mount_flags flags;
@@ -142,9 +162,18 @@ struct vfs_mount {
 
 void vfs_mount_ref(struct vfs_mount* mount);
 void vfs_mount_unref(struct vfs_mount* mount);
+
+// Creates a unique instance of a mount point.
 errno_t vfs_mount(const char* fs_name, enum mount_flags flags, void* arg, struct vfs_mount** out);
 errno_t vfs_unmount(struct vfs_mount* mount);
 
+// Allocates a new mount with `root` as its root entry. Takes a new reference to `root`.
+errno_t vfs_mount_new(enum mount_flags flags, struct vfs_entry* root, struct vfs_mount** out);
+
+// Attaches `mount` on top of the entry referenced by `at`.
+errno_t vfs_path_mount(struct vfs_path at, struct vfs_mount* mount);
+errno_t vfs_path_unmount(struct vfs_path path);
+
 struct vfs_super_block_ops {
     errno_t (*sync)(struct vfs_super_block* sb);
     errno_t (*statvfs)(struct vfs_super_block* sb, struct __statvfs* out);
@@ -154,6 +183,7 @@ struct vfs_super_block_ops {
 struct vfs_super_block {
     struct refcount rc;
     const struct vfs_super_block_ops* ops;
+    enum mount_flags flags;
 };
 
 // TODO: Maybe not like this
@@ -161,6 +191,7 @@ struct vfs_mknod_target {
     enum vfs_node_type type;
     const struct vfs_node_ops* ops;
     void* priv;
+    dev_t rdev;
 };
 
 struct vfs_node_ops {
@@ -176,6 +207,8 @@ struct vfs_node_ops {
         struct {
             // Truncates the file down to `new_length` bytes.
             errno_t (*truncate)(struct vfs_node* node, uint64_t new_length);
+            // Returns a new reference to a memory object holding the file contents.
+            errno_t (*get_vmo)(struct vfs_node* node, struct vmo** out);
         } regular;
 
         // VNODE_SYMLINK
@@ -184,6 +217,11 @@ struct vfs_node_ops {
             errno_t (*read_link)(struct vfs_node* node, iovec_iter_t* iter, size_t* out_read);
         } symlink;
 
+        // VNODE_CHARDEV or VNODE_BLOCKDEV
+        struct {
+            errno_t (*open)(struct vfs_node* node, struct vfs_file* file);
+        } device;
+
         // VNODE_DIR
         struct {
             errno_t (*lookup)(struct vfs_node* node, struct vfs_path* child);
@@ -225,10 +263,10 @@ struct vfs_node_ops {
             );
             errno_t (*mknod)(
                 struct vfs_node* node,
+                struct vfs_path* path,
                 mode_t mode,
                 struct vfs_mknod_target* target,
-                const struct ident* identity,
-                struct vfs_node** out
+                const struct ident* identity
             );
             errno_t (*get_dir_entries)(
                 struct vfs_node* node,
@@ -246,7 +284,6 @@ struct vfs_node_ops {
 struct vfs_node {
     struct refcount rc;
     enum vfs_node_type type;
-    bool is_seekable;
 
     const struct vfs_node_ops* ops;
     const struct vfs_file_ops* file_ops;
@@ -254,9 +291,13 @@ struct vfs_node {
 
     mutex_t lock;
     void* priv;
+    struct pipe* pipe;
 
     ino_t id;
+    dev_t rdev;
     size_t size;
+    blkcnt_t blocks;
+    nlink_t nlink;
     mode_t mode;
     uid_t uid;
     gid_t gid;
@@ -280,7 +321,8 @@ struct vfs_file_ops {
         off_t offset,
         uintptr_t* out_addr
     );
-    bool (*seekable)(struct vfs_file* file);
+    enum poll_flags (*poll)(struct vfs_file* file, enum poll_flags events, struct poll_table* table);
+    void (*release)(struct vfs_file* file);
 };
 
 enum vfs_file_position {
@@ -303,7 +345,7 @@ struct vfs_file {
 
     // Absolute path resolved during open.
     char* absolute_path;
-    enum open_flags flags;
+    _Atomic(enum open_flags) flags;
 
     enum vfs_file_position position_kind;
     off_t position;
@@ -311,6 +353,9 @@ struct vfs_file {
 
     struct vfs_path path;
     bool has_path;
+
+    // Driver data.
+    void* priv;
 };
 
 // A file system driver.
@@ -319,6 +364,16 @@ struct vfs_filesystem {
     errno_t (*mount)(const struct vfs_filesystem* fs, enum mount_flags flags, void* arg, struct vfs_mount** out);
 };
 
+// Allocates a new inode in `sb` with `type`, `ops` and permission bits `mode`.
+// The inode takes a reference to `sb` if it's not `nullptr`.
+errno_t vfs_node_new(
+    struct vfs_super_block* sb,
+    enum vfs_node_type type,
+    const struct vfs_node_ops* ops,
+    mode_t mode,
+    struct vfs_node** out
+);
+
 void vfs_node_ref(struct vfs_node* node);
 void vfs_node_unref(struct vfs_node* node);
 void vfs_entry_ref(struct vfs_entry* entry);
@@ -331,8 +386,6 @@ void vfs_super_block_unref(struct vfs_super_block* sb);
 // Gets the kernel path representation of the absolute root.
 struct vfs_path vfs_get_root();
 
-void vfs_init();
-
 errno_t vfs_register_filesystem(const struct vfs_filesystem* fs);
 
 errno_t vfs_sync_all();
@@ -392,12 +445,46 @@ errno_t vfs_mknod(
     const char* path,
     mode_t mode,
     struct vfs_mknod_target* target,
+    const struct ident* identity,
+    struct vfs_node** out
+);
+
+errno_t vfs_rename(
+    struct vfs_path root,
+    struct vfs_path old_cwd,
+    const char* old_path,
+    struct vfs_path new_cwd,
+    const char* new_path,
     const struct ident* identity
 );
 
-// Checks if an access to a `node` with `flags` using `identity` is allowed.
+errno_t vfs_link(
+    struct vfs_path root,
+    struct vfs_path old_cwd,
+    const char* old_path,
+    struct vfs_path new_cwd,
+    const char* new_path,
+    bool follow_symlinks,
+    const struct ident* identity
+);
+
+void vfs_stat(struct vfs_node* node, struct __stat* out);
+errno_t vfs_statvfs(struct vfs_node* node, struct __statvfs* out);
+errno_t vfs_chmod(struct vfs_node* node, mode_t mode, const struct ident* identity);
+errno_t vfs_chown(struct vfs_node* node, uid_t uid, gid_t gid, const struct ident* identity);
+errno_t vfs_utimens(struct vfs_node* node, const timespec_t* times, const struct ident* identity);
+errno_t vfs_node_check_exec(struct vfs_node* node, const struct ident* identity);
+errno_t vfs_truncate(struct vfs_node* node, uint64_t length);
+errno_t vfs_read_link(struct vfs_node* node, iovec_iter_t* iter, size_t* out_read);
+
+// Checks if an access to a `node` with `access` using `identity` is allowed.
 // Returns 0 if the access is allowed.
-errno_t vfs_node_try_access(struct vfs_node* node, const struct ident* identity, enum open_flags flags, bool use_real);
+errno_t vfs_node_try_access(
+    struct vfs_node* node,
+    const struct ident* identity,
+    enum access_flags access,
+    bool use_real
+);
 
 // Opens a node at `path`, relative to `root` and `cwd`.
 // Returns the opened node as a file in `out`.
@@ -411,6 +498,16 @@ errno_t vfs_file_open(
     struct vfs_file** out
 );
 
+// Creates a new open file backed by `node`, which must not be `nullptr`.
+// If `path` is not `nullptr`, the file remembers it and takes a reference to it.
+errno_t vfs_file_new(struct vfs_node* node, const struct vfs_path* path, enum open_flags flags, struct vfs_file** out);
+
+// Changes the file's position mode to a stream.
+void vfs_stream_open(struct vfs_file* file);
+
+// Gets a new reference to a memory object holding the contents of `file`.
+errno_t vfs_file_get_object(struct vfs_file* file, struct vmo** out);
+
 // Reads from a `file` into a `buffer` from the current offset.
 // Returns the actual bytes read in `out_read`.
 errno_t vfs_file_read(struct vfs_file* file, iovec_iter_t* buffer, ssize_t* out_read);
@@ -447,6 +544,8 @@ errno_t vfs_file_mmap(
     uintptr_t* out_addr
 );
 
+enum poll_flags vfs_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table);
+
 // Directory listing.
 errno_t vfs_get_dir_entries(
     struct vfs_file* file,
@@ -456,6 +555,17 @@ errno_t vfs_get_dir_entries(
     size_t* out_count
 );
 
+// Generic directory listing implementation which walks the cached children of `entry`.
+errno_t vfs_dir_get_cached_entries(
+    struct vfs_node* node,
+    struct vfs_entry* entry,
+    off_t offset,
+    struct __dirent* buffer,
+    size_t buffer_len,
+    const struct ident* identity,
+    size_t* out_count
+);
+
 // Creates an inode belonging to the anonymous pseudo file system.
 errno_t vfs_create_anon_inode(
     const struct vfs_file_ops* file_ops,
@@ -464,3 +574,5 @@ errno_t vfs_create_anon_inode(
     const struct ident* identity,
     struct vfs_node** out
 );
+
+INIT_EXTERN(vfs_stage);

include/zinnia/virt.h

@@ -2,6 +2,9 @@
 
 #pragma once
 
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
 #include <zinnia/hashmap.h>
 #include <zinnia/types.h>
 #include <stddef.h>
@@ -11,15 +14,26 @@ enum page_type : uint8_t {
     PAGE_PHYS = 0, // Regular physical memory.
 };
 
+enum page_flags : uint32_t {
+    PAGE_BUSY = 1 << 0,
+    PAGE_WANTED = 1 << 1,
+    PAGE_DIRTY = 1 << 2,
+    PAGE_ANON = 1 << 3,
+};
+
 struct page {
     enum page_type type;
-    uint32_t flags;
-    size_t refcount;
+    _Atomic(uint32_t) flags;
+    _Atomic(size_t) holds;
     union {
         struct {
             struct page* next; // Pointer to the next chunk.
             size_t count;      // Amount of free pages.
         } freelist;
+        struct {
+            void* owner;
+            uint64_t index;
+        } obj;
     };
 };
 static_assert(0x1000 % sizeof(struct page) == 0, "struct must fit cleanly inside a page!");
@@ -29,6 +43,14 @@ static_assert(sizeof(struct page) <= 64, "struct must be smaller than 64 bytes!"
 extern struct page* vm_pfndb;
 
 uintptr_t vm_page_addr(struct page* page);
+struct page* vm_page_from_addr(phys_t addr);
+
+errno_t page_alloc(enum alloc_flags flags, struct page** out);
+void page_hold(struct page* page);
+void page_unhold(struct page* page);
+bool page_try_busy(struct page* page);
+void page_unbusy(struct page* page);
+void page_wait(struct page* page);
 
 // Allocator for virtual memory.
 struct vm_alloc {

include/zinnia/vma.h

@@ -2,21 +2,34 @@
 
 #pragma once
 
+#include <zinnia/amap.h>
 #include <zinnia/errno.h>
 #include <zinnia/mman.h>
 #include <zinnia/mutex.h>
 #include <zinnia/pmap.h>
-#include <zinnia/spin.h>
+#include <zinnia/rwlock.h>
 #include <zinnia/tailq.h>
 #include <zinnia/vmo.h>
 
+enum vma_flags : uint8_t {
+    VMA_SHARED = 1 << 0,
+    VMA_NEEDS_COPY = 1 << 1,
+};
+
+struct vma;
+
 struct vma_mapping {
     size_t start_page;
     size_t end_page;
     size_t offset_page;
     struct vmo* object;
+    struct amap* amap;
+    size_t amap_offset;
+    struct vma* vma;
     enum prot_flags prot;
+    enum vma_flags flags;
     tailq_link(struct vma_mapping) link;
+    tailq_link(struct vma_mapping) object_link;
 };
 
 // Virtual address space.
@@ -24,30 +37,44 @@ struct vma {
     // Architecture-specific virtual memory map implementation.
     struct pmap pmap;
     tailq_head(struct vma_mapping) mappings;
-    struct spinlock lock;
+    struct rwlock lock;
+    mutex_t map_lock;
+    struct refcount rc;
 };
 
 // Try to create a new, empty virtual address space.
 errno_t vma_new(struct vma** out);
 
-// Delete a virtual address space.
-void vma_free(struct vma* v);
+void vma_ref(struct vma* v);
+void vma_unref(struct vma* v);
+errno_t vma_fork(struct vma* src, struct vma** out);
 
 errno_t vma_map(
     struct vma* vma,
     struct vmo* vmo,
     uintptr_t addr,
     size_t len,
-    enum prot_flags flags,
+    enum prot_flags prot,
+    enum vma_flags flags,
     uintptr_t vmo_offset
 );
 
+errno_t vma_find_free(struct vma* v, uintptr_t hint, size_t len, uintptr_t* out);
+
+bool vma_range_mapped(struct vma* v, uintptr_t vaddr, size_t size);
+
 // Remove any mappings in the given address range.
 errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size);
 
 // Change the protection of an existing mapping.
 errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags);
 
+errno_t vma_write(struct vma* v, uintptr_t addr, const void* buf, size_t len);
+errno_t vma_zero(struct vma* v, uintptr_t addr, size_t len);
+errno_t vma_sync(struct vma* v, uintptr_t addr, size_t len);
+
+void vma_object_remove_page(struct vmo* object, uint64_t index, phys_t phys, bool protect);
+
 struct vma_fault_info {
     uintptr_t ip;
     uintptr_t addr;

include/zinnia/vmo.h

@@ -3,57 +3,63 @@
 #pragma once
 
 #include <zinnia/errno.h>
-#include <zinnia/list.h>
+#include <zinnia/iovec.h>
+#include <zinnia/mutex.h>
+#include <zinnia/pmap.h>
+#include <zinnia/radix.h>
 #include <zinnia/refcount.h>
-#include <zinnia/spin.h>
+#include <zinnia/tailq.h>
 #include <zinnia/virt.h>
 #include <stddef.h>
 #include <stdint.h>
 
-struct vmo {
-    struct refcount rc;
-    errno_t (*get_page)(struct vmo* vmo, uintptr_t offset_idx, struct page** out);
+struct vmo;
+struct vma_mapping;
+
+struct vmo_pager_ops {
+    errno_t (*get_pages)(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid);
+    errno_t (*put_pages)(struct vmo* vmo, uint64_t index, struct page** pages, size_t count);
+    errno_t (*lookup)(struct vmo* vmo, uint64_t index, phys_t* out);
+    size_t max_cluster;
     void (*destroy)(struct vmo* vmo);
 };
 
+struct vmo {
+    struct refcount rc;
+    const struct vmo_pager_ops* pager;
+    mutex_t lock;
+    struct radix_tree pages;
+    tailq_head(struct vma_mapping) mappings;
+    uint64_t ra_next;
+    size_t ra_window;
+    enum cache_mode cache;
+};
+
+enum vmo_access {
+    VMO_READ,
+    VMO_WRITE,
+};
+
+void vmo_init(struct vmo* vmo, const struct vmo_pager_ops* pager);
 void vmo_ref(struct vmo* obj);
 void vmo_unref(struct vmo* obj);
 
-errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read);
-errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written);
+errno_t vmo_new_anon(struct vmo** out);
+errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out);
+void vmo_put_page(struct page* page);
+
+errno_t vmo_read(struct vmo* obj, uint64_t offset, void* buf, size_t len, size_t* actual_read);
+errno_t vmo_write(struct vmo* obj, uint64_t offset, const void* buf, size_t len, size_t* actual_written);
+errno_t vmo_read_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_read);
+errno_t vmo_write_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_written);
 errno_t vmo_copy(
     struct vmo* target,
-    uintptr_t target_offset,
+    uint64_t target_offset,
     struct vmo* src,
-    uintptr_t src_offset,
+    uint64_t src_offset,
     size_t len,
     size_t* actual_copied
 );
 
-struct pager_ops {
-    errno_t (*get_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page** out);
-    errno_t (*put_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page* page);
-};
-
-// TODO: change to rbtree
-struct page_list {
-    uintptr_t offset;
-    struct page* value;
-    list_link(struct page_list*) next;
-};
-
-struct paged_vmo {
-    struct vmo object;
-    struct pager_ops source;
-    list_head(struct page_list*) cache;
-    struct spinlock lock;
-};
-
-errno_t vmo_new_paged(struct pager_ops ops, struct paged_vmo** out);
-errno_t vmo_new_phys(struct paged_vmo** out);
-
-// Drops all cached pages past `length` bytes.
-void vmo_truncate_paged(struct paged_vmo* paged, size_t length);
-
-// Creates a VMO backed by a specific physical address range (e.g. for MMIO).
-errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out);
+errno_t vmo_sync(struct vmo* vmo, uint64_t first, uint64_t last);
+void vmo_truncate(struct vmo* vmo, uint64_t length);

include/zinnia/waitq.h

@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: ISC
-
-#pragma once
-
-#include <zinnia/atomic.h>
-#include <zinnia/sched.h>
-#include <zinnia/spin.h>
-#include <zinnia/tailq.h>
-
-// A queue of tasks waiting for an event.
-struct waitq {
-    tailq_head(_Atomic(struct task)) waiters;
-    _Atomic(bool) woken;
-};
-
-void waitq_init(struct waitq* wq);
-void waitq_wait(struct waitq* wq);
-void waitq_wake_one(struct waitq* wq);
-void waitq_wake_all(struct waitq* wq);

include/zinnia/workqueue.h

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/tailq.h>
+#include <stdint.h>
+
+struct work {
+    void (*func)(struct work* work);
+    tailq_link(struct work) link;
+    atomic_bool pending;
+};
+
+struct workqueue;
+
+// Workqueue for regular prios
+extern struct workqueue* workqueue_system;
+// Workqueue for high prios
+extern struct workqueue* workqueue_high;
+
+void work_init(struct work* work, void (*func)(struct work* work));
+
+errno_t workqueue_create(const char* name, uint8_t priority, struct workqueue** out);
+
+void workqueue_destroy(struct workqueue* wq);
+
+bool work_queue(struct workqueue* wq, struct work* work);
+
+bool work_cancel(struct workqueue* wq, struct work* work);
+
+void workqueue_flush(struct workqueue* wq);
+
+INIT_EXTERN(workqueue_stage);

lib/condvar.c

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/atomic.h>
+#include <zinnia/condvar.h>
+#include <zinnia/park.h>
+#include <stdint.h>
+
+void condvar_init(struct condvar* cv) {
+    atomic_store(&cv->seq, 0, ATOMIC_RELAXED);
+    atomic_store(&cv->waiters, 0, ATOMIC_RELAXED);
+}
+
+void condvar_wait(struct condvar* cv, mutex_t* mutex) {
+    atomic_fetch_add(&cv->waiters, 1, ATOMIC_SEQ_CST);
+    uint32_t seq = atomic_load(&cv->seq, ATOMIC_SEQ_CST);
+    mutex_unlock(mutex);
+    park_wait(&cv->seq, seq);
+    atomic_fetch_sub(&cv->waiters, 1, ATOMIC_RELAXED);
+    mutex_lock(mutex);
+}
+
+static void wake(struct condvar* cv, size_t count) {
+    atomic_fetch_add(&cv->seq, 1, ATOMIC_SEQ_CST);
+    if (atomic_load(&cv->waiters, ATOMIC_SEQ_CST) != 0)
+        park_wake(&cv->seq, count);
+}
+
+void condvar_signal(struct condvar* cv) {
+    wake(cv, 1);
+}
+
+void condvar_broadcast(struct condvar* cv) {
+    wake(cv, SIZE_MAX);
+}

lib/console.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
 #include <zinnia/init.h>
 #include <zinnia/print.h>

lib/dpc.c

@@ -1,3 +0,0 @@
-#include <zinnia/dpc.h>
-
-// TODO

lib/event.c

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/atomic.h>
+#include <zinnia/event.h>
+#include <zinnia/park.h>
+#include <zinnia/percpu.h>
+#include <zinnia/sched.h>
+
+constexpr uint32_t EVENT_SET = (1u << 0);
+constexpr uint32_t EVENT_PARKED = (1u << 1); // At least one task might be parked on the event.
+
+void event_init(struct event* event) {
+    atomic_store(&event->state, 0, ATOMIC_RELAXED);
+}
+
+void event_set(struct event* event) {
+    uint32_t old = atomic_fetch_or(&event->state, EVENT_SET, ATOMIC_RELEASE);
+    if (!(old & EVENT_PARKED))
+        return;
+
+    atomic_fetch_and(&event->state, ~EVENT_PARKED, ATOMIC_RELAXED);
+    unpark_all(event, 0);
+}
+
+void event_reset(struct event* event) {
+    atomic_fetch_and(&event->state, ~EVENT_SET, ATOMIC_RELAXED);
+}
+
+bool event_is_set(struct event* event) {
+    return atomic_load(&event->state, ATOMIC_ACQUIRE) & EVENT_SET;
+}
+
+void event_wait(struct event* event) {
+    uint32_t state = atomic_load(&event->state, ATOMIC_ACQUIRE);
+    while (!(state & EVENT_SET)) {
+        if (!(state & EVENT_PARKED)) {
+            if (!atomic_compare_exchange_weak(
+                    &event->state,
+                    &state,
+                    state | EVENT_PARKED,
+                    ATOMIC_ACQUIRE,
+                    ATOMIC_ACQUIRE
+                ))
+                continue;
+        }
+
+        // Only sleeps if the event is not set, but parked at this point.
+        park_wait(&event->state, EVENT_PARKED);
+        state = atomic_load(&event->state, ATOMIC_ACQUIRE);
+    }
+}

lib/irq.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/assert.h>
 #include <zinnia/irq.h>
 #include <zinnia/list.h>

lib/meson.build

@@ -1,6 +1,8 @@
 kernel_sources += files(
     'clock.c',
+    'condvar.c',
     'console.c',
+    'event.c',
     'guid.c',
     'initgraph.c',
     'iovec.c',
@@ -9,8 +11,10 @@ kernel_sources += files(
     'module.c',
     'mutex.c',
     'panic.c',
+    'park.c',
     'percpu.c',
     'print.c',
+    'process.c',
     'radix.c',
     'random.c',
     'ringbuf.c',
@@ -21,4 +25,5 @@ kernel_sources += files(
     'string.c',
     'timer.c',
     'turnstile.c',
+    'workqueue.c',
 )

lib/park.c

@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
+#include <zinnia/irq.h>
+#include <zinnia/park.h>
+#include <zinnia/percpu.h>
+#include <zinnia/sched.h>
+#include <zinnia/signal.h>
+#include <zinnia/spin.h>
+#include <zinnia/timer.h>
+#include <zinnia/utils.h>
+
+constexpr size_t PARK_BUCKET_BITS = 8;
+constexpr size_t PARK_BUCKETS = (1 << PARK_BUCKET_BITS);
+
+enum park_waiter_state : unsigned int {
+    WAITER_QUEUED,
+    WAITER_CLAIMED,
+    WAITER_DONE,
+};
+
+struct park_waiter {
+    const void* key;
+    struct task* task;
+    uint8_t priority;
+    atomic_uint state;
+    struct park_result result;
+    struct park_waiter* prev;
+    struct park_waiter* next;
+    struct park_waiter* key_prev;
+    struct park_waiter* key_next;
+    struct park_waiter* wake_next;
+};
+
+struct park_bucket {
+    alignas(64) struct spinlock lock;
+    struct park_waiter* keys;
+};
+
+static struct park_bucket buckets[PARK_BUCKETS];
+
+static struct park_bucket* bucket_for(const void* key) {
+    uint64_t hash = ((uintptr_t)key >> 3) * 0x9E3779B97F4A7C15ull;
+    return &buckets[hash >> (64 - PARK_BUCKET_BITS)];
+}
+
+static struct park_waiter* queue_find(struct park_bucket* b, const void* key) {
+    struct park_waiter* head = b->keys;
+    while (head && head->key != key)
+        head = head->key_next;
+    return head;
+}
+
+static void queue_replace_head(struct park_bucket* b, struct park_waiter* old, struct park_waiter* new) {
+    new->key_prev = old->key_prev;
+    new->key_next = old->key_next;
+    if (new->key_prev)
+        new->key_prev->key_next = new;
+    else
+        b->keys = new;
+    if (new->key_next)
+        new->key_next->key_prev = new;
+}
+
+// Queues `w` behind all waiters on the same key that are at least as urgent.
+// With `front`, it's queued in front of waiters on the same key with the same priority instead.
+// The priority is only considered at the time of queueing.
+static void queue_insert(struct park_bucket* b, struct park_waiter* w, bool front) {
+    struct park_waiter* head = queue_find(b, w->key);
+    if (!head) {
+        w->prev = nullptr;
+        w->next = nullptr;
+        w->key_prev = nullptr;
+        w->key_next = b->keys;
+        if (b->keys)
+            b->keys->key_prev = w;
+        b->keys = w;
+        return;
+    }
+
+    struct park_waiter* last = nullptr;
+    struct park_waiter* pos = head;
+    while (pos && (pos->priority > w->priority || (!front && pos->priority == w->priority))) {
+        last = pos;
+        pos = pos->next;
+    }
+
+    w->prev = last;
+    w->next = pos;
+    if (pos)
+        pos->prev = w;
+    if (last)
+        last->next = w;
+    else
+        queue_replace_head(b, head, w);
+}
+
+static void queue_remove(struct park_bucket* b, struct park_waiter* w) {
+    if (w->next)
+        w->next->prev = w->prev;
+    if (w->prev) {
+        w->prev->next = w->next;
+        return;
+    }
+    if (w->next) {
+        queue_replace_head(b, w, w->next);
+        return;
+    }
+    if (w->key_prev)
+        w->key_prev->key_next = w->key_next;
+    else
+        b->keys = w->key_next;
+    if (w->key_next)
+        w->key_next->key_prev = w->key_prev;
+}
+
+// Must be called with the bucket locksd. `w` must already be removed from the bucket.
+static void claim(struct park_waiter* w, uintptr_t token) {
+    w->result = (struct park_result){.status = PARK_UNPARKED, .token = token};
+    w->wake_next = nullptr;
+    atomic_store(&w->state, WAITER_CLAIMED, ATOMIC_RELAXED);
+}
+
+static void wake_claimed(struct park_waiter* w) {
+    while (w) {
+        struct park_waiter* next = w->wake_next;
+        struct task* task = w->task;
+        task_lock(task);
+        atomic_store(&w->state, WAITER_DONE, ATOMIC_RELEASE);
+        sched_wake_locked(task);
+        task_unlock(task);
+        w = next;
+    }
+}
+
+static bool cancel(
+    struct park_bucket* b,
+    struct park_waiter* w,
+    enum park_status status,
+    void (*cancelled)(void* ctx, bool was_last),
+    void* ctx
+) {
+    spin_lock(&b->lock);
+    const bool queued = atomic_load(&w->state, ATOMIC_RELAXED) == WAITER_QUEUED;
+    if (queued) {
+        queue_remove(b, w);
+        if (cancelled)
+            cancelled(ctx, queue_find(b, w->key) == nullptr);
+        w->result = (struct park_result){.status = status};
+        atomic_store(&w->state, WAITER_DONE, ATOMIC_RELAXED);
+    }
+    spin_unlock(&b->lock);
+    return queued;
+}
+
+struct park_timer {
+    struct timer timer;
+    struct task* task;
+    atomic_bool expired;
+};
+
+static void park_timer_fn(struct timer* timer) {
+    struct park_timer* t = containerof(timer, struct park_timer, timer);
+    atomic_store(&t->expired, true, ATOMIC_RELEASE);
+    sched_wake(t->task);
+}
+
+struct park_result park_timeout(
+    const void* key,
+    bool (*validate)(void* ctx),
+    void (*cancelled)(void* ctx, bool was_last),
+    void* ctx,
+    enum park_flags flags,
+    const uint64_t* deadline_ns
+) {
+    struct task* self = percpu_get()->sched.current;
+    assert(self != nullptr, "Tried to park without a scheduler");
+    assert(irq_get_state(), "Tried to park with interrupts disabled");
+
+    struct park_waiter w = {
+        .key = key,
+        .task = self,
+        .priority = atomic_load(&self->priority, ATOMIC_RELAXED),
+    };
+    atomic_init(&w.state, WAITER_QUEUED);
+
+    struct park_bucket* b = bucket_for(key);
+    spin_lock(&b->lock);
+    if (validate && !validate(ctx)) {
+        spin_unlock(&b->lock);
+        return (struct park_result){.status = PARK_INVALID};
+    }
+    if (deadline_ns && clock_get_elapsed_ns() >= *deadline_ns) {
+        if (cancelled)
+            cancelled(ctx, queue_find(b, key) == nullptr);
+        spin_unlock(&b->lock);
+        return (struct park_result){.status = PARK_TIMED_OUT};
+    }
+    queue_insert(b, &w, flags & PARK_FRONT);
+    spin_unlock(&b->lock);
+
+    struct park_timer timer = {.task = self};
+    atomic_init(&timer.expired, false);
+    if (deadline_ns) {
+        timer_init(&timer.timer, park_timer_fn);
+        timer_arm(&timer.timer, *deadline_ns);
+    }
+
+    // sched_block() can return early, so check whether we were actually unparked.
+    // The task may migrate while blocked, the scheduler has to be looked up again every time.
+    while (atomic_load(&w.state, ATOMIC_ACQUIRE) != WAITER_DONE) {
+        enum park_status reason = PARK_UNPARKED;
+        if (atomic_load(&timer.expired, ATOMIC_ACQUIRE))
+            reason = PARK_TIMED_OUT;
+        else if ((flags & PARK_INTERRUPTIBLE) && signal_pending(self))
+            reason = PARK_INTERRUPTED;
+        if (reason != PARK_UNPARKED && cancel(b, &w, reason, cancelled, ctx))
+            break;
+        sched_block(&percpu_get()->sched);
+    }
+
+    if (deadline_ns)
+        timer_cancel(&timer.timer);
+    task_lock(self);
+    task_unlock(self);
+    return w.result;
+}
+
+struct park_result park(const void* key, bool (*validate)(void* ctx), void* ctx, enum park_flags flags) {
+    return park_timeout(key, validate, nullptr, ctx, flags, nullptr);
+}
+
+bool unpark_one(const void* key, uintptr_t (*callback)(void* ctx, const struct park_unpark_result* result), void* ctx) {
+    struct park_bucket* b = bucket_for(key);
+    spin_lock(&b->lock);
+
+    struct park_waiter* w = queue_find(b, key);
+    if (w)
+        queue_remove(b, w);
+
+    struct park_unpark_result result = {
+        .task = w ? w->task : nullptr,
+        .have_more = queue_find(b, key) != nullptr,
+    };
+    uintptr_t token = callback ? callback(ctx, &result) : 0;
+
+    if (w)
+        claim(w, token);
+
+    spin_unlock(&b->lock);
+    wake_claimed(w);
+    return w != nullptr;
+}
+
+static size_t unpark_n(const void* key, size_t count, uintptr_t token) {
+    struct park_bucket* b = bucket_for(key);
+    struct park_waiter* list = nullptr;
+    struct park_waiter** tail = &list;
+    size_t woken = 0;
+
+    spin_lock(&b->lock);
+    struct park_waiter* w;
+    while (woken < count && (w = queue_find(b, key))) {
+        queue_remove(b, w);
+        claim(w, token);
+        *tail = w;
+        tail = &w->wake_next;
+        woken++;
+    }
+    spin_unlock(&b->lock);
+
+    wake_claimed(list);
+    return woken;
+}
+
+size_t unpark_all(const void* key, uintptr_t token) {
+    return unpark_n(key, SIZE_MAX, token);
+}
+
+struct park_wait_ctx {
+    atomic_uint* addr;
+    unsigned int expected;
+};
+
+static bool park_wait_validate(void* ctx) {
+    struct park_wait_ctx* c = ctx;
+    return atomic_load(c->addr, ATOMIC_RELAXED) == c->expected;
+}
+
+bool park_wait(atomic_uint* addr, unsigned int expected) {
+    struct park_wait_ctx ctx = {addr, expected};
+    return park(addr, park_wait_validate, &ctx, 0).status != PARK_INVALID;
+}
+
+size_t park_wake(atomic_uint* addr, size_t count) {
+    return unpark_n(addr, count, 0);
+}

lib/print.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/clock.h>
 #include <zinnia/console.h>
 #include <zinnia/panic.h>

lib/process.c

@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/exec.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <zinnia/process.h>
+#include <zinnia/sched.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+#include <string.h>
+
+constexpr size_t PROCESS_INITIAL_FDS = 64;
+
+static struct process kernel_process;
+static atomic_bool kernel_process_ready;
+static atomic_int next_pid = 1;
+
+static size_t fd_words(size_t capacity) {
+    return capacity / 64;
+}
+
+static void fd_table_destroy(struct fd_table* t) {
+    mem_free(t->files);
+    mem_free(t->open);
+    mem_free(t->cloexec);
+    *t = (struct fd_table){};
+}
+
+static errno_t fd_table_resize(struct fd_table* t, size_t capacity) {
+    struct vfs_file** files = mem_alloc(capacity * sizeof(*files), 0);
+    uint64_t* open = mem_alloc(fd_words(capacity) * sizeof(*open), 0);
+    uint64_t* cloexec = mem_alloc(fd_words(capacity) * sizeof(*cloexec), 0);
+    if (!files || !open || !cloexec) {
+        mem_free(files);
+        mem_free(open);
+        mem_free(cloexec);
+        return ENOMEM;
+    }
+
+    if (t->capacity) {
+        memcpy(files, t->files, t->capacity * sizeof(*files));
+        memcpy(open, t->open, fd_words(t->capacity) * sizeof(*open));
+        memcpy(cloexec, t->cloexec, fd_words(t->capacity) * sizeof(*cloexec));
+    }
+    mem_free(t->files);
+    mem_free(t->open);
+    mem_free(t->cloexec);
+
+    t->files = files;
+    t->open = open;
+    t->cloexec = cloexec;
+    t->capacity = capacity;
+    return 0;
+}
+
+static errno_t fd_table_grow(struct fd_table* t, size_t fd) {
+    if (fd < t->capacity)
+        return 0;
+    size_t capacity = MAX(PROCESS_INITIAL_FDS, t->capacity);
+    while (capacity <= fd)
+        capacity *= 2;
+    return fd_table_resize(t, MIN(capacity, PROCESS_MAX_FDS));
+}
+
+static size_t fd_find_free(struct fd_table* t, size_t start) {
+    for (size_t w = start / 64; w < fd_words(t->capacity); w++) {
+        uint64_t used = t->open[w];
+        if (w == start / 64)
+            used |= (1ull << (start % 64)) - 1;
+        if (~used)
+            return w * 64 + __builtin_ctzll(~used);
+    }
+    return MAX(start, t->capacity);
+}
+
+static void fd_install(struct fd_table* t, size_t fd, struct vfs_file* file, bool cloexec) {
+    const uint64_t bit = 1ull << (fd % 64);
+    t->files[fd] = file;
+    t->open[fd / 64] |= bit;
+    if (cloexec)
+        t->cloexec[fd / 64] |= bit;
+    else
+        t->cloexec[fd / 64] &= ~bit;
+}
+
+static void fd_clear(struct fd_table* t, size_t fd) {
+    const uint64_t bit = 1ull << (fd % 64);
+    t->files[fd] = nullptr;
+    t->open[fd / 64] &= ~bit;
+    t->cloexec[fd / 64] &= ~bit;
+    if (fd < t->next_free)
+        t->next_free = fd;
+}
+
+static struct vfs_file* fd_lookup(struct process* proc, int fd) {
+    if (fd < 0 || (size_t)fd >= proc->files.capacity)
+        return nullptr;
+    return proc->files.files[fd];
+}
+
+static void process_init_common(struct process* proc) {
+    refcount_init(&proc->rc);
+    spin_init(&proc->tasks_lock);
+    tailq_init(&proc->tasks);
+    mutex_init(&proc->lock);
+    mutex_init(&proc->files_lock);
+    vec_init(&proc->identity.groups);
+}
+
+struct process* process_kernel() {
+    if (!atomic_load(&kernel_process_ready, ATOMIC_ACQUIRE)) {
+        process_init_common(&kernel_process);
+        kernel_process.pid = 0;
+        kernel_process.space = &kernel_vma;
+        atomic_store(&kernel_process_ready, true, ATOMIC_RELEASE);
+    }
+    return &kernel_process;
+}
+
+static void close_files(struct process* proc) {
+    struct fd_table* t = &proc->files;
+    mutex_lock(&proc->files_lock);
+    for (size_t w = 0; w < fd_words(t->capacity); w++) {
+        for (uint64_t bits = t->open[w]; bits; bits &= bits - 1)
+            vfs_file_unref(t->files[w * 64 + __builtin_ctzll(bits)]);
+    }
+    fd_table_destroy(t);
+    mutex_unlock(&proc->files_lock);
+}
+
+void process_ref(struct process* proc) {
+    refcount_inc(&proc->rc);
+}
+
+void process_unref(struct process* proc) {
+    if (proc == &kernel_process || !refcount_dec(&proc->rc))
+        return;
+    close_files(proc);
+    vfs_path_drop(proc->root);
+    vfs_path_drop(proc->cwd);
+    if (proc->space)
+        vma_unref(proc->space);
+    vec_destroy(&proc->identity.groups);
+    mem_free(proc);
+}
+
+uint8_t process_priority(struct process* proc) {
+    if (proc == &kernel_process)
+        return SCHED_PRIO_DEFAULT;
+
+    // Normalize the priority from "nice" ranges to our priorities.
+    return 16 + (19 - proc->nice) * 15 / 39;
+}
+
+void process_attach_task(struct process* proc, struct task* task) {
+    spin_lock(&proc->tasks_lock);
+    tailq_insert_tail(&proc->tasks, task, process_link);
+    proc->thread_count++;
+    spin_unlock(&proc->tasks_lock);
+}
+
+void process_detach_task(struct process* proc, struct task* task) {
+    spin_lock(&proc->tasks_lock);
+    tailq_remove(&proc->tasks, task, process_link);
+    spin_unlock(&proc->tasks_lock);
+}
+
+static void rlimit_init(struct process_rlimit* limit, rlim_t cur, rlim_t max) {
+    atomic_init(&limit->cur, cur);
+    limit->max = max;
+}
+
+static errno_t process_alloc(struct process** out) {
+    struct process* proc = mem_alloc(sizeof(struct process), 0);
+    if (!proc)
+        return ENOMEM;
+    process_init_common(proc);
+    proc->pid = atomic_fetch_add(&next_pid, 1, ATOMIC_RELAXED);
+    proc->root = vfs_get_root();
+    proc->cwd = vfs_get_root();
+    proc->umask = 022;
+    rlimit_init(&proc->limit_nofile, PROCESS_DEFAULT_FDS, PROCESS_MAX_FDS);
+
+    *out = proc;
+    return 0;
+}
+
+static errno_t load_image(
+    struct process* proc,
+    struct vmo* object,
+    const char* path,
+    const char** argv,
+    const char** envp,
+    struct vma** out_space,
+    uintptr_t* out_ip,
+    uintptr_t* out_sp
+) {
+    struct vma* space;
+    errno_t status = vma_new(&space);
+    if (status)
+        return status;
+
+    struct exec_info info = {
+        .space = space,
+        .file_obj = object,
+        .exec_path = path,
+        .argv = argv,
+        .envp = envp,
+    };
+    struct exec_format* format = exec_identify(object);
+    status = format ? format->load(proc, &info, out_ip, out_sp) : ENOEXEC;
+    if (status) {
+        vma_unref(space);
+        return status;
+    }
+
+    *out_space = space;
+    return 0;
+}
+
+errno_t process_spawn_init(const char* path, const char** argv, const char** envp) {
+    struct process* proc;
+    errno_t status = process_alloc(&proc);
+    if (status)
+        return status;
+
+    struct task_start_info* start = mem_alloc(sizeof(struct task_start_info), 0);
+    if (!start) {
+        process_unref(proc);
+        return ENOMEM;
+    }
+
+    struct vmo* object;
+    status = exec_open(proc, path, &object);
+    if (!status) {
+        status = load_image(proc, object, path, argv, envp, &proc->space, &start->ip, &start->sp);
+        vmo_unref(object);
+    }
+    if (status) {
+        mem_free(start);
+        process_unref(proc);
+        return status;
+    }
+
+    struct vfs_file* console;
+    status = vfs_file_open(proc->root, proc->cwd, "/dev/console", O_RDWR, 0, ident_kernel(), &console);
+    if (status) {
+        log_warn("process: Unable to open /dev/console for init (%i)\n", status);
+    } else {
+        for (int fd = 0; fd <= 2; fd++)
+            process_fd_replace(proc, fd, console, false);
+        vfs_file_unref(console);
+    }
+
+    struct task* task;
+    status = task_create(argv[0], proc, sched_to_user, (uintptr_t)start, &task);
+    process_unref(proc);
+    if (status) {
+        mem_free(start);
+        return status;
+    }
+
+    sched_add_task(&percpu_get()->sched, task);
+    return 0;
+}
+
+errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloexec, int* out_fd) {
+    return process_fd_alloc_min(proc, file, 0, cloexec, out_fd);
+}
+
+size_t process_fd_limit(struct process* proc) {
+    return MIN((size_t)PROCESS_MAX_FDS, (size_t)atomic_load(&proc->limit_nofile.cur, ATOMIC_RELAXED));
+}
+
+errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int min_fd, bool cloexec, int* out_fd) {
+    if (min_fd < 0 || (size_t)min_fd >= process_fd_limit(proc))
+        return EINVAL;
+
+    struct fd_table* t = &proc->files;
+    mutex_lock(&proc->files_lock);
+    const size_t fd = fd_find_free(t, MAX((size_t)min_fd, t->next_free));
+    if (fd >= process_fd_limit(proc)) {
+        mutex_unlock(&proc->files_lock);
+        return EMFILE;
+    }
+
+    errno_t status = fd_table_grow(t, fd);
+    if (status) {
+        mutex_unlock(&proc->files_lock);
+        return status;
+    }
+    fd_install(t, fd, file, cloexec);
+    if ((size_t)min_fd <= t->next_free)
+        t->next_free = fd + 1;
+    vfs_file_ref(file);
+    mutex_unlock(&proc->files_lock);
+
+    *out_fd = fd;
+    return 0;
+}
+
+errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, bool cloexec) {
+    if (fd < 0 || (size_t)fd >= process_fd_limit(proc))
+        return EBADF;
+
+    struct fd_table* t = &proc->files;
+    mutex_lock(&proc->files_lock);
+    errno_t status = fd_table_grow(t, fd);
+    if (status) {
+        mutex_unlock(&proc->files_lock);
+        return status;
+    }
+    struct vfs_file* displaced = t->files[fd];
+    fd_install(t, fd, file, cloexec);
+    vfs_file_ref(file);
+    mutex_unlock(&proc->files_lock);
+
+    if (displaced)
+        vfs_file_unref(displaced);
+    return 0;
+}
+
+errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) {
+    errno_t status = EBADF;
+    mutex_lock(&proc->files_lock);
+    if (fd_lookup(proc, fd)) {
+        *out_cloexec = (proc->files.cloexec[fd / 64] >> (fd % 64)) & 1;
+        status = 0;
+    }
+    mutex_unlock(&proc->files_lock);
+    return status;
+}
+
+errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) {
+    errno_t status = EBADF;
+    mutex_lock(&proc->files_lock);
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file) {
+        fd_install(&proc->files, fd, file, cloexec);
+        status = 0;
+    }
+    mutex_unlock(&proc->files_lock);
+    return status;
+}
+
+struct vfs_file* process_fd_get(struct process* proc, int fd) {
+    mutex_lock(&proc->files_lock);
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file)
+        vfs_file_ref(file);
+    mutex_unlock(&proc->files_lock);
+    return file;
+}
+
+errno_t process_fd_close(struct process* proc, int fd) {
+    mutex_lock(&proc->files_lock);
+    struct vfs_file* file = fd_lookup(proc, fd);
+    if (file)
+        fd_clear(&proc->files, fd);
+    mutex_unlock(&proc->files_lock);
+
+    if (!file)
+        return EBADF;
+    vfs_file_unref(file);
+    return 0;
+}

lib/wait.c

@@ -1,17 +0,0 @@
-#include <zinnia/waitq.h>
-
-void waitq_init(struct waitq* wq) {
-    tailq_init(&wq->waiters);
-}
-
-void waitq_wait(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_one(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_all(struct waitq* wq) {
-    // TODO
-}

lib/workqueue.c

@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/percpu.h>
+#include <zinnia/process.h>
+#include <zinnia/sched.h>
+#include <zinnia/spin.h>
+#include <zinnia/workqueue.h>
+
+struct workqueue {
+    struct spinlock lock;
+    tailq_head(struct work) queue;
+    struct task* worker;
+    bool idle;
+    bool stopping;
+    struct task* exit_waiter;
+    atomic_bool exited;
+};
+
+struct work_barrier {
+    struct work work;
+    struct task* waiter;
+    atomic_bool done;
+};
+
+struct workqueue* workqueue_system;
+struct workqueue* workqueue_high;
+
+void work_init(struct work* work, void (*func)(struct work* work)) {
+    work->func = func;
+    atomic_init(&work->pending, false);
+}
+
+static struct task* current_task() {
+    return percpu_get()->sched.current;
+}
+
+static void wait_for(atomic_bool* flag) {
+    while (!atomic_load(flag, ATOMIC_ACQUIRE))
+        sched_block(&percpu_get()->sched);
+}
+
+static void signal(atomic_bool* flag, struct task* waiter) {
+    atomic_store(flag, true, ATOMIC_RELEASE);
+    sched_wake(waiter);
+}
+
+static struct work* next_work(struct workqueue* wq, void (**out_func)(struct work* work)) {
+    spin_lock(&wq->lock);
+    while (true) {
+        struct work* work = tailq_first(&wq->queue);
+        if (work) {
+            tailq_remove(&wq->queue, work, link);
+            *out_func = work->func;
+            atomic_store(&work->pending, false, ATOMIC_RELEASE);
+            spin_unlock(&wq->lock);
+            return work;
+        }
+        if (wq->stopping) {
+            spin_unlock(&wq->lock);
+            return nullptr;
+        }
+
+        wq->idle = true;
+        spin_unlock(&wq->lock);
+        sched_block(&percpu_get()->sched);
+        spin_lock(&wq->lock);
+    }
+}
+
+// Worker task main loop.
+static void worker_fn(uintptr_t arg) {
+    struct workqueue* wq = (struct workqueue*)arg;
+
+    struct work* work;
+    void (*func)(struct work* work);
+    while ((work = next_work(wq, &func)))
+        func(work);
+
+    struct task* waiter = wq->exit_waiter;
+    signal(&wq->exited, waiter);
+    sched_exit();
+}
+
+errno_t workqueue_create(const char* name, uint8_t priority, struct workqueue** out) {
+    struct workqueue* wq = mem_alloc(sizeof(*wq), 0);
+    if (!wq)
+        return ENOMEM;
+
+    spin_init(&wq->lock);
+    tailq_init(&wq->queue);
+    atomic_init(&wq->exited, false);
+
+    errno_t status = task_create(name, process_kernel(), worker_fn, (uintptr_t)wq, &wq->worker);
+    if (status) {
+        mem_free(wq);
+        return status;
+    }
+
+    sched_set_priority(wq->worker, priority);
+    sched_add_task(&percpu_get()->sched, wq->worker);
+
+    *out = wq;
+    return 0;
+}
+
+static void wake_worker(struct workqueue* wq, bool stop) {
+    spin_lock(&wq->lock);
+    if (stop)
+        wq->stopping = true;
+    const bool was_idle = wq->idle;
+    wq->idle = false;
+    spin_unlock(&wq->lock);
+
+    if (was_idle)
+        sched_wake(wq->worker);
+}
+
+void workqueue_destroy(struct workqueue* wq) {
+    wq->exit_waiter = current_task();
+    wake_worker(wq, true);
+    wait_for(&wq->exited);
+    mem_free(wq);
+}
+
+bool work_queue(struct workqueue* wq, struct work* work) {
+    if (atomic_exchange(&work->pending, true, ATOMIC_ACQ_REL))
+        return false;
+
+    spin_lock(&wq->lock);
+    assert(!wq->stopping, "Queued work on a workqueue that is being destroyed");
+    tailq_insert_tail(&wq->queue, work, link);
+    const bool was_idle = wq->idle;
+    wq->idle = false;
+    spin_unlock(&wq->lock);
+
+    if (was_idle)
+        sched_wake(wq->worker);
+    return true;
+}
+
+bool work_cancel(struct workqueue* wq, struct work* work) {
+    bool removed = false;
+    spin_lock(&wq->lock);
+    for (struct work* it = tailq_first(&wq->queue); it; it = tailq_next(it, link)) {
+        if (it == work) {
+            tailq_remove(&wq->queue, work, link);
+            atomic_store(&work->pending, false, ATOMIC_RELEASE);
+            removed = true;
+            break;
+        }
+    }
+    spin_unlock(&wq->lock);
+    return removed;
+}
+
+static void barrier_fn(struct work* work) {
+    struct work_barrier* barrier = containerof(work, struct work_barrier, work);
+    struct task* waiter = barrier->waiter;
+    signal(&barrier->done, waiter);
+}
+
+void workqueue_flush(struct workqueue* wq) {
+    assert(current_task() != wq->worker, "Tried to flush a workqueue from its own worker");
+
+    struct work_barrier barrier = {.waiter = current_task()};
+    work_init(&barrier.work, barrier_fn);
+    atomic_init(&barrier.done, false);
+    work_queue(wq, &barrier.work);
+    wait_for(&barrier.done);
+}
+
+static void workqueue_stage_init(struct boot_info* info) {
+    errno_t status = workqueue_create("kworker", SCHED_PRIO_DEFAULT, &workqueue_system);
+    assert(status == 0, "Unable to create a workqueue");
+
+    status = workqueue_create("kworker/high", SCHED_PRIO_WORK_HIGH, &workqueue_high);
+    assert(status == 0, "Unable to create a workqueue");
+}
+
+INIT_TASK(workqueue_stage, "core.workqueue", workqueue_stage_init);
+INIT_DEPENDS(workqueue_stage, sched_stage);

mem/amap.c

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/amap.h>
+#include <zinnia/utils.h>
+#include <string.h>
+
+errno_t anon_new(struct page* src, struct vm_anon** out) {
+    struct vm_anon* anon = mem_alloc(sizeof(struct vm_anon), 0);
+    if (!anon)
+        return ENOMEM;
+
+    struct page* page;
+    errno_t status = page_alloc(src ? ALLOC_NOZERO : 0, &page);
+    if (status) {
+        mem_free(anon);
+        return status;
+    }
+
+    if (src)
+        memcpy(HHDM_PTR(vm_page_addr(page)), HHDM_PTR(vm_page_addr(src)), arch_mem_page_size());
+
+    atomic_fetch_or(&page->flags, PAGE_ANON, ATOMIC_RELAXED);
+    page->obj.owner = anon;
+    atomic_store(&anon->rc, 1, ATOMIC_RELAXED);
+    anon->page = page;
+    *out = anon;
+    return 0;
+}
+
+void anon_ref(struct vm_anon* anon) {
+    atomic_fetch_add(&anon->rc, 1, ATOMIC_RELAXED);
+}
+
+void anon_unref(struct vm_anon* anon) {
+    if (atomic_fetch_sub(&anon->rc, 1, ATOMIC_RELEASE) != 1)
+        return;
+    atomic_fence(ATOMIC_ACQUIRE);
+    page_unhold(anon->page);
+    mem_free(anon);
+}
+
+errno_t amap_new(struct amap** out) {
+    struct amap* amap = mem_alloc(sizeof(struct amap), 0);
+    if (!amap)
+        return ENOMEM;
+    refcount_init(&amap->rc);
+    mutex_init(&amap->lock);
+    radix_init(&amap->anons);
+    *out = amap;
+    return 0;
+}
+
+void amap_ref(struct amap* amap) {
+    refcount_inc(&amap->rc);
+}
+
+void amap_unref(struct amap* amap) {
+    if (!refcount_dec(&amap->rc))
+        return;
+
+    struct vm_anon* batch[32];
+    uint64_t slots[32];
+    size_t found;
+    while ((found = radix_gang_lookup(&amap->anons, 0, ARRAY_SIZE(batch), (void**)batch, slots)) != 0) {
+        for (size_t i = 0; i < found; i++) {
+            radix_remove(&amap->anons, slots[i]);
+            anon_unref(batch[i]);
+        }
+    }
+    radix_destroy(&amap->anons);
+    mem_free(amap);
+}
+
+bool amap_is_shared(struct amap* amap) {
+    return atomic_load(&amap->rc.count, ATOMIC_ACQUIRE) != 1;
+}
+
+errno_t amap_copy(struct amap* src, size_t offset, size_t num_slots, struct amap** out) {
+    struct amap* copy;
+    errno_t status = amap_new(&copy);
+    if (status)
+        return status;
+
+    struct vm_anon* batch[32];
+    uint64_t slots[32];
+    uint64_t next = offset;
+
+    mutex_lock(&src->lock);
+    while (status == 0) {
+        const size_t found = radix_gang_lookup(&src->anons, next, ARRAY_SIZE(batch), (void**)batch, slots);
+        if (found == 0)
+            break;
+        for (size_t i = 0; i < found && status == 0; i++) {
+            if (slots[i] >= offset + num_slots)
+                goto done;
+            status = radix_insert(&copy->anons, slots[i] - offset, batch[i]);
+            if (!status)
+                anon_ref(batch[i]);
+        }
+        next = slots[found - 1] + 1;
+    }
+done:
+    mutex_unlock(&src->lock);
+
+    if (status) {
+        amap_unref(copy);
+        return status;
+    }
+    *out = copy;
+    return 0;
+}

mem/meson.build

@@ -1,6 +1,7 @@
 kernel_sources += files(
-    'fault.c',
+    'amap.c',
     'init.c',
+    'page.c',
     'pmm.c',
     'slab.c',
     'stack.c',

mem/page.c

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/park.h>
+#include <zinnia/virt.h>
+
+struct page* vm_page_from_addr(phys_t addr) {
+    return &vm_pfndb[addr / arch_mem_page_size()];
+}
+
+errno_t page_alloc(enum alloc_flags flags, struct page** out) {
+    phys_t addr;
+    errno_t status = mem_phys_alloc(1, flags, &addr);
+    if (status)
+        return status;
+
+    struct page* page = vm_page_from_addr(addr);
+    page->type = PAGE_PHYS;
+    page->obj.owner = nullptr;
+    page->obj.index = 0;
+    atomic_store(&page->flags, 0, ATOMIC_RELAXED);
+    atomic_store(&page->holds, 1, ATOMIC_RELEASE);
+    *out = page;
+    return 0;
+}
+
+void page_hold(struct page* page) {
+    atomic_fetch_add(&page->holds, 1, ATOMIC_RELAXED);
+}
+
+void page_unhold(struct page* page) {
+    if (atomic_fetch_sub(&page->holds, 1, ATOMIC_RELEASE) != 1)
+        return;
+    atomic_fence(ATOMIC_ACQUIRE);
+    atomic_store(&page->flags, 0, ATOMIC_RELAXED);
+    page->obj.owner = nullptr;
+    mem_phys_free(vm_page_addr(page), 1);
+}
+
+bool page_try_busy(struct page* page) {
+    return !(atomic_fetch_or(&page->flags, PAGE_BUSY, ATOMIC_ACQUIRE) & PAGE_BUSY);
+}
+
+void page_unbusy(struct page* page) {
+    const uint32_t old = atomic_fetch_and(&page->flags, ~(PAGE_BUSY | PAGE_WANTED), ATOMIC_RELEASE);
+    if (old & PAGE_WANTED)
+        unpark_all(&page->flags, 0);
+}
+
+static bool still_busy(void* ctx) {
+    struct page* page = ctx;
+    uint32_t flags = atomic_load(&page->flags, ATOMIC_RELAXED);
+    while (flags & PAGE_BUSY) {
+        if (atomic_compare_exchange_weak(&page->flags, &flags, flags | PAGE_WANTED, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            return true;
+    }
+    return false;
+}
+
+void page_wait(struct page* page) {
+    park(&page->flags, still_busy, page, 0);
+}

mem/vma.c

@@ -1,11 +1,20 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
+#include <zinnia/amap.h>
 #include <zinnia/assert.h>
 #include <zinnia/errno.h>
 #include <zinnia/mman.h>
 #include <zinnia/pmap.h>
 #include <zinnia/utils.h>
 #include <zinnia/vma.h>
+#include <arch/mem.h>
 #include <stdckdint.h>
+#include <string.h>
+
+typedef tailq_head(struct vma_mapping) mapping_list_t;
+
+constexpr uintptr_t VMA_MMAP_BASE = 0x1'0000'0000;
 
 // Try to create a new, empty virtual address space.
 errno_t vma_new(struct vma** out) {
@@ -24,48 +33,252 @@ errno_t vma_new(struct vma** out) {
         goto err1;
 
     tailq_init(&vmspace->mappings);
+    rwlock_init(&vmspace->lock);
+    mutex_init(&vmspace->map_lock);
+    refcount_init(&vmspace->rc);
 
     *out = vmspace;
     return 0;
 
 err1:
     mem_free(vmspace);
-err0:
     return res;
 }
 
-// Delete a virtual address space.
-void vma_free(struct vma* v) {
-    struct vma_mapping* mapping;
-    while ((mapping = tailq_first(&v->mappings)) != nullptr) {
-        tailq_remove(&v->mappings, mapping, link);
-        vmo_unref(mapping->object);
-        mem_free(mapping);
+static void object_lock(struct vma_mapping* m) {
+    if (m->object)
+        mutex_lock(&m->object->lock);
+}
+
+static void object_unlock(struct vma_mapping* m) {
+    if (m->object)
+        mutex_unlock(&m->object->lock);
+}
+
+static void object_link(struct vma_mapping* m) {
+    if (!m->object)
+        return;
+    mutex_lock(&m->object->lock);
+    tailq_insert_tail(&m->object->mappings, m, object_link);
+    mutex_unlock(&m->object->lock);
+}
+
+static void mapping_free(struct vma_mapping* m) {
+    if (m->object) {
+        mutex_lock(&m->object->lock);
+        tailq_remove(&m->object->mappings, m, object_link);
+        mutex_unlock(&m->object->lock);
+        vmo_unref(m->object);
     }
+    if (m->amap)
+        amap_unref(m->amap);
+    mem_free(m);
+}
+
+static void free_mappings(mapping_list_t* list) {
+    struct vma_mapping* mapping;
+    while ((mapping = tailq_first(list)) != nullptr) {
+        tailq_remove(list, mapping, link);
+        mapping_free(mapping);
+    }
+}
+
+void vma_ref(struct vma* v) {
+    refcount_inc(&v->rc);
+}
+
+void vma_unref(struct vma* v) {
+    assert(v != &kernel_vma, "Tried to release the kernel address space");
+    if (!refcount_dec(&v->rc))
+        return;
+
+    free_mappings((mapping_list_t*)&v->mappings);
+    pmap_destroy(&v->pmap);
+    mem_free(v);
+}
+
+static struct vma_mapping* mapping_alloc() {
+    return mem_alloc(sizeof(struct vma_mapping), 0);
+}
+
+static void mapping_clone(
+    struct vma_mapping* dst,
+    const struct vma_mapping* src,
+    struct vma* vma,
+    size_t start,
+    size_t end
+) {
+    const size_t delta = start - src->start_page;
+    dst->start_page = start;
+    dst->end_page = end;
+    dst->offset_page = src->offset_page + delta;
+    dst->object = src->object;
+    dst->amap = src->amap;
+    dst->amap_offset = src->amap_offset + delta;
+    dst->vma = vma;
+    dst->prot = src->prot;
+    dst->flags = src->flags;
+    if (dst->object)
+        vmo_ref(dst->object);
+    if (dst->amap)
+        amap_ref(dst->amap);
+}
+
+static void unmap_pages(struct vma* v, size_t start, size_t end) {
+    const size_t page_size = arch_mem_page_size();
+    for (size_t p = start; p < end; p++)
+        pmap_unmap(&v->pmap, p * page_size);
+}
+
+static void carve(struct vma* v, size_t start, size_t end, struct vma_mapping** spare, mapping_list_t* dead) {
+    struct vma_mapping* m = tailq_first(&v->mappings);
+    while (m) {
+        struct vma_mapping* next = tailq_next(m, link);
+        if (start < m->end_page && m->start_page < end) {
+            unmap_pages(v, MAX(start, m->start_page), MIN(end, m->end_page));
+
+            if (m->start_page >= start && m->end_page <= end) {
+                tailq_remove(&v->mappings, m, link);
+                tailq_insert_tail(dead, m, link);
+            } else if (m->start_page < start && m->end_page > end) {
+                struct vma_mapping* tail = *spare;
+                assert(tail != nullptr, "Missing spare mapping");
+                *spare = nullptr;
+                mapping_clone(tail, m, v, end, m->end_page);
+                object_lock(m);
+                m->end_page = start;
+                object_unlock(m);
+                object_link(tail);
+                tailq_insert_tail(&v->mappings, tail, link);
+            } else if (m->start_page < start) {
+                object_lock(m);
+                m->end_page = start;
+                object_unlock(m);
+            } else {
+                const size_t delta = end - m->start_page;
+                object_lock(m);
+                m->offset_page += delta;
+                m->amap_offset += delta;
+                m->start_page = end;
+                object_unlock(m);
+            }
+        }
+        m = next;
+    }
+}
+
+static void split_at(struct vma* v, size_t page, struct vma_mapping** spare) {
+    struct vma_mapping* m;
+    for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (m->start_page < page && page < m->end_page)
+            break;
+    }
+    if (!m)
+        return;
+
+    struct vma_mapping* tail = *spare;
+    assert(tail != nullptr, "Missing spare mapping");
+    *spare = nullptr;
+    mapping_clone(tail, m, v, page, m->end_page);
+    object_lock(m);
+    m->end_page = page;
+    object_unlock(m);
+    object_link(tail);
+    tailq_insert_tail(&v->mappings, tail, link);
+}
+
+static errno_t page_range(uintptr_t addr, size_t len, size_t* start, size_t* end) {
+    const size_t page_size = arch_mem_page_size();
+    uintptr_t last;
+    if (addr % page_size != 0 || len == 0 || ckd_add(&last, addr, len - 1))
+        return EINVAL;
+    *start = addr / page_size;
+    *end = last / page_size + 1;
+    return 0;
+}
+
+static bool range_mapped(struct vma* v, size_t start, size_t end) {
+    size_t page = start;
+    while (page < end) {
+        struct vma_mapping* m;
+        for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+            if (page >= m->start_page && page < m->end_page)
+                break;
+        }
+        if (!m)
+            return false;
+        page = m->end_page;
+    }
+    return true;
+}
+
+bool vma_range_mapped(struct vma* v, uintptr_t vaddr, size_t size) {
+    size_t start, end;
+    if (page_range(vaddr, size, &start, &end))
+        return false;
+
+    rwlock_read_lock(&v->lock);
+    bool mapped = range_mapped(v, start, end);
+    rwlock_read_unlock(&v->lock);
+    return mapped;
+}
+
+errno_t vma_find_free(struct vma* v, uintptr_t hint, size_t len, uintptr_t* out) {
+    const size_t page_size = arch_mem_page_size();
+    if (len == 0)
+        return EINVAL;
+
+    uintptr_t base = MAX(hint, VMA_MMAP_BASE);
+    if (ckd_add(&base, base, page_size - 1))
+        return ENOMEM;
+    size_t candidate = base / page_size;
+    const size_t num_pages = len / page_size + (len % page_size != 0);
+
+    rwlock_read_lock(&v->lock);
+    while (true) {
+        size_t candidate_end;
+        if (ckd_add(&candidate_end, candidate, num_pages) || candidate_end > SIZE_MAX / page_size ||
+            !arch_mem_addr_is_user(candidate_end * page_size - 1)) {
+            rwlock_read_unlock(&v->lock);
+            return ENOMEM;
+        }
+
+        struct vma_mapping* m;
+        for (m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+            if (candidate < m->end_page && m->start_page < candidate_end)
+                break;
+        }
+        if (!m)
+            break;
+        candidate = m->end_page;
+    }
+    rwlock_read_unlock(&v->lock);
+
+    *out = candidate * page_size;
+    return 0;
 }
 
 errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
-    panic("TODO");
-}
+    size_t start, end;
+    errno_t status = page_range(vaddr, size, &start, &end);
+    if (status)
+        return status;
 
-errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
-    panic("TODO");
-}
+    struct vma_mapping* spare = mapping_alloc();
+    if (!spare)
+        return ENOMEM;
 
-static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
-    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
-        if (page >= m->start_page && page < m->end_page)
-            return m;
-    }
-    return nullptr;
-}
+    mapping_list_t dead;
+    tailq_init(&dead);
 
-static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) {
-    if (info->caused_by_write)
-        return prot & PROT_WRITE;
-    if (info->caused_by_fetch)
-        return prot & PROT_EXEC;
-    return prot & (PROT_READ | PROT_WRITE);
+    rwlock_write_lock(&v->lock);
+    carve(v, start, end, &spare, &dead);
+    pmap_shootdown(&v->pmap);
+    rwlock_write_unlock(&v->lock);
+
+    free_mappings(&dead);
+    mem_free(spare);
+    return 0;
 }
 
 static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
@@ -81,43 +294,344 @@ static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
     return pf;
 }
 
-bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
+static void reprotect_pages(struct vma* v, const struct vma_mapping* m) {
     const size_t page_size = arch_mem_page_size();
-    const size_t faulty_page = info->addr / page_size;
+    const bool none = !(m->prot & (PROT_READ | PROT_WRITE | PROT_EXEC));
+    const enum pte_flags flags = prot_to_pte(v, m->prot & ~PROT_WRITE);
+    for (size_t p = m->start_page; p < m->end_page; p++) {
+        if (none)
+            pmap_unmap(&v->pmap, p * page_size);
+        else
+            pmap_protect(&v->pmap, p * page_size, flags);
+    }
+}
 
-    spin_lock(&v->lock);
-    struct vma_mapping* mapping = find_mapping(v, faulty_page);
-    if (!mapping || !can_access(mapping->prot, info)) {
-        spin_unlock(&v->lock);
-        return false;
+errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
+    size_t start, end;
+    errno_t status = page_range(vaddr, size, &start, &end);
+    if (status)
+        return status;
+
+    struct vma_mapping* spares[2] = {mapping_alloc(), mapping_alloc()};
+    if (!spares[0] || !spares[1]) {
+        mem_free(spares[0]);
+        mem_free(spares[1]);
+        return ENOMEM;
     }
 
-    struct vmo* object = mapping->object;
-    const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
-    const enum prot_flags prot = mapping->prot;
-    vmo_ref(object);
-    spin_unlock(&v->lock);
+    rwlock_write_lock(&v->lock);
+    if (!range_mapped(v, start, end)) {
+        rwlock_write_unlock(&v->lock);
+        mem_free(spares[0]);
+        mem_free(spares[1]);
+        return ENOMEM;
+    }
+    split_at(v, start, &spares[0]);
+    split_at(v, end, &spares[1]);
+    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (m->start_page >= start && m->end_page <= end) {
+            object_lock(m);
+            m->prot = flags;
+            object_unlock(m);
+            reprotect_pages(v, m);
+        }
+    }
+    pmap_shootdown(&v->pmap);
+    rwlock_write_unlock(&v->lock);
 
-    struct page* page;
-    if (object->get_page(object, object_page, &page)) {
-        vmo_unref(object);
-        return false;
+    mem_free(spares[0]);
+    mem_free(spares[1]);
+    return 0;
+}
+
+void vma_object_remove_page(struct vmo* object, uint64_t index, phys_t phys, bool protect) {
+    const size_t page_size = arch_mem_page_size();
+    for (struct vma_mapping* m = tailq_first(&object->mappings); m; m = tailq_next(m, object_link)) {
+        if (index < m->offset_page || index - m->offset_page >= m->end_page - m->start_page)
+            continue;
+
+        struct pmap* pmap = &m->vma->pmap;
+        const uintptr_t va = (m->start_page + (index - m->offset_page)) * page_size;
+        phys_t current;
+        if (pmap_extract(pmap, va, &current) || current != phys)
+            continue;
+
+        if (protect)
+            pmap_protect(pmap, va, prot_to_pte(m->vma, m->prot & ~PROT_WRITE));
+        else
+            pmap_unmap(pmap, va);
+        pmap_shootdown(pmap);
+    }
+}
+
+static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
+    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (page >= m->start_page && page < m->end_page)
+            return m;
+    }
+    return nullptr;
+}
+
+enum fault_kind {
+    FAULT_READ,
+    FAULT_WRITE,
+    FAULT_EXEC,
+};
+
+static bool can_access(enum prot_flags prot, enum fault_kind kind) {
+    switch (kind) {
+    case FAULT_WRITE:
+        return prot & PROT_WRITE;
+    case FAULT_EXEC:
+        return prot & PROT_EXEC;
+    default:
+        return prot & (PROT_READ | PROT_WRITE);
+    }
+}
+
+static errno_t enter(struct vma* v, size_t page, phys_t phys, enum pte_flags flags, enum cache_mode cache) {
+    const uintptr_t va = page * arch_mem_page_size();
+    phys_t old;
+    const bool replace = pmap_extract(&v->pmap, va, &old) == 0;
+    errno_t status = pmap_map(&v->pmap, va, phys, flags, cache);
+    if (status)
+        return status;
+    pte_flush(va);
+    if (replace)
+        pmap_shootdown(&v->pmap);
+    return 0;
+}
+
+static errno_t prepare_amap(struct vma* v, size_t page) {
+    errno_t status = 0;
+    rwlock_write_lock(&v->lock);
+    struct vma_mapping* m = find_mapping(v, page);
+    if (m && !(m->flags & VMA_SHARED)) {
+        if (!m->amap) {
+            status = amap_new(&m->amap);
+            m->amap_offset = 0;
+            if (!status)
+                m->flags &= ~VMA_NEEDS_COPY;
+        } else if (m->flags & VMA_NEEDS_COPY) {
+            if (amap_is_shared(m->amap)) {
+                struct amap* copy;
+                status = amap_copy(m->amap, m->amap_offset, m->end_page - m->start_page, &copy);
+                if (!status) {
+                    amap_unref(m->amap);
+                    m->amap = copy;
+                    m->amap_offset = 0;
+                }
+            }
+            if (!status)
+                m->flags &= ~VMA_NEEDS_COPY;
+        }
+    }
+    rwlock_write_unlock(&v->lock);
+    return status;
+}
+
+static errno_t fault_anon(
+    struct vma* v,
+    struct vma_mapping* m,
+    struct vm_anon* anon,
+    size_t page,
+    size_t slot,
+    bool write,
+    phys_t* out
+) {
+    bool writable = !(m->flags & VMA_NEEDS_COPY) && atomic_load(&anon->rc, ATOMIC_ACQUIRE) == 1;
+    if (write && !writable) {
+        struct vm_anon* copy;
+        errno_t status = anon_new(anon->page, &copy);
+        if (status)
+            return status;
+        *radix_slot(&m->amap->anons, slot) = copy;
+        anon_unref(anon);
+        anon = copy;
+        writable = true;
     }
 
-    spin_lock(&v->lock);
-    mapping = find_mapping(v, faulty_page);
-    const bool still_valid = mapping && mapping->object == object &&
-        (faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info);
+    *out = vm_page_addr(anon->page);
+    return enter(v, page, *out, prot_to_pte(v, writable ? m->prot : m->prot & ~PROT_WRITE), CACHE_ANY);
+}
 
-    if (still_valid) {
-        errno_t status =
-            pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
-        assert(status == 0, "Failed to map a demand-loaded page");
+static errno_t fault_promote(
+    struct vma* v,
+    struct vma_mapping* m,
+    size_t page,
+    size_t slot,
+    size_t index,
+    phys_t* out
+) {
+    struct vmo* obj = m->object;
+    struct page* src = nullptr;
+    errno_t status = 0;
+    if (obj && obj->pager->lookup) {
+        phys_t phys;
+        status = obj->pager->lookup(obj, index, &phys);
+        if (!status)
+            src = vm_page_from_addr(phys);
+    } else if (obj) {
+        status = vmo_get_page(obj, index, VMO_READ, &src);
     }
-    spin_unlock(&v->lock);
+    if (status)
+        return status;
 
-    vmo_unref(object);
-    return true;
+    struct vm_anon* anon;
+    status = anon_new(src, &anon);
+    if (src && !obj->pager->lookup)
+        vmo_put_page(src);
+    if (status)
+        return status;
+
+    status = radix_insert(&m->amap->anons, slot, anon);
+    if (status) {
+        anon_unref(anon);
+        return status;
+    }
+
+    *out = vm_page_addr(anon->page);
+    return enter(v, page, *out, prot_to_pte(v, m->prot), CACHE_ANY);
+}
+
+static errno_t fault_object(struct vma* v, struct vma_mapping* m, size_t page, size_t index, bool write, phys_t* out) {
+    struct vmo* obj = m->object;
+    const bool shared = m->flags & VMA_SHARED;
+
+    if (obj->pager->lookup) {
+        errno_t status = obj->pager->lookup(obj, index, out);
+        if (status)
+            return status;
+        return enter(v, page, *out, prot_to_pte(v, shared ? m->prot : m->prot & ~PROT_WRITE), obj->cache);
+    }
+
+    struct page* pg;
+    errno_t status = vmo_get_page(obj, index, shared && write ? VMO_WRITE : VMO_READ, &pg);
+    if (status)
+        return status;
+
+    const bool writable = shared && (write || !obj->pager->put_pages);
+    mutex_lock(&obj->lock);
+    if (radix_lookup(&obj->pages, index) == pg) {
+        *out = vm_page_addr(pg);
+        status = enter(v, page, *out, prot_to_pte(v, writable ? m->prot : m->prot & ~PROT_WRITE), obj->cache);
+    } else {
+        status = EAGAIN;
+    }
+    mutex_unlock(&obj->lock);
+    vmo_put_page(pg);
+    return status;
+}
+
+static errno_t fault(struct vma* v, size_t page, enum fault_kind kind, bool force, phys_t* out) {
+    const bool write = kind == FAULT_WRITE;
+
+    while (true) {
+        rwlock_read_lock(&v->lock);
+        struct vma_mapping* m = find_mapping(v, page);
+        if (!m) {
+            rwlock_read_unlock(&v->lock);
+            return EFAULT;
+        }
+        if (!force && !can_access(m->prot, kind)) {
+            rwlock_read_unlock(&v->lock);
+            return EACCES;
+        }
+
+        const bool shared = m->flags & VMA_SHARED;
+        if (!shared && (write || !m->object) && (!m->amap || (m->flags & VMA_NEEDS_COPY))) {
+            rwlock_read_unlock(&v->lock);
+            errno_t status = prepare_amap(v, page);
+            if (status)
+                return status;
+            continue;
+        }
+
+        const size_t index = page - m->start_page + m->offset_page;
+        const size_t slot = page - m->start_page + m->amap_offset;
+        struct amap* amap = shared ? nullptr : m->amap;
+        errno_t status;
+
+        if (amap) {
+            mutex_lock(&amap->lock);
+            struct vm_anon* anon = radix_lookup(&amap->anons, slot);
+            if (anon)
+                status = fault_anon(v, m, anon, page, slot, write, out);
+            else if (write || !m->object)
+                status = fault_promote(v, m, page, slot, index, out);
+            else
+                status = fault_object(v, m, page, index, false, out);
+            mutex_unlock(&amap->lock);
+        } else {
+            status = fault_object(v, m, page, index, write, out);
+        }
+
+        rwlock_read_unlock(&v->lock);
+        if (status != EAGAIN)
+            return status;
+    }
+}
+
+bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
+    const enum fault_kind kind = info->caused_by_write ? FAULT_WRITE : info->caused_by_fetch ? FAULT_EXEC : FAULT_READ;
+    phys_t phys;
+    return fault(v, info->addr / arch_mem_page_size(), kind, false, &phys) == 0;
+}
+
+static errno_t write_range(struct vma* v, uintptr_t addr, const void* buf, size_t len) {
+    const size_t page_size = arch_mem_page_size();
+    size_t progress = 0;
+    while (progress < len) {
+        const uintptr_t cur = addr + progress;
+        const size_t misalign = cur % page_size;
+        const size_t chunk = MIN(page_size - misalign, len - progress);
+
+        phys_t phys;
+        errno_t status = fault(v, cur / page_size, FAULT_WRITE, true, &phys);
+        if (status)
+            return status;
+
+        uint8_t* dst = (uint8_t*)HHDM_PTR(phys) + misalign;
+        if (buf)
+            memcpy(dst, (const uint8_t*)buf + progress, chunk);
+        else
+            memset(dst, 0, chunk);
+        progress += chunk;
+    }
+    return 0;
+}
+
+errno_t vma_write(struct vma* v, uintptr_t addr, const void* buf, size_t len) {
+    return write_range(v, addr, buf, len);
+}
+
+errno_t vma_zero(struct vma* v, uintptr_t addr, size_t len) {
+    return write_range(v, addr, nullptr, len);
+}
+
+errno_t vma_sync(struct vma* v, uintptr_t addr, size_t len) {
+    size_t start, end;
+    errno_t status = page_range(addr, len, &start, &end);
+    if (status)
+        return status;
+
+    errno_t result = 0;
+    rwlock_read_lock(&v->lock);
+    if (!range_mapped(v, start, end)) {
+        rwlock_read_unlock(&v->lock);
+        return ENOMEM;
+    }
+    for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
+        if (!m->object || !(m->flags & VMA_SHARED) || m->end_page <= start || m->start_page >= end)
+            continue;
+        const size_t first = MAX(start, m->start_page) - m->start_page + m->offset_page;
+        const size_t last = MIN(end, m->end_page) - m->start_page + m->offset_page;
+        status = vmo_sync(m->object, first, last);
+        if (status && !result)
+            result = status;
+    }
+    rwlock_read_unlock(&v->lock);
+    return result;
 }
 
 errno_t vma_map(
@@ -125,34 +639,116 @@ errno_t vma_map(
     struct vmo* vmo,
     uintptr_t addr,
     size_t len,
-    enum prot_flags flags,
+    enum prot_flags prot,
+    enum vma_flags flags,
     uintptr_t vmo_offset
 ) {
-    if (!vma || !vmo)
+    if (!vma || (!vmo && (flags & VMA_SHARED)))
         return EINVAL;
 
     uintptr_t end;
-    if (ckd_add(&end, addr, len))
+    if (len == 0 || ckd_add(&end, addr, len))
         return ENOMEM;
 
     const size_t page_size = arch_mem_page_size();
     if (addr % page_size != vmo_offset % page_size)
         return EINVAL;
 
-    struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
-    if (!mapping)
+    struct vma_mapping* mapping = mapping_alloc();
+    struct vma_mapping* spare = mapping_alloc();
+    if (!mapping || !spare) {
+        mem_free(mapping);
+        mem_free(spare);
         return ENOMEM;
+    }
 
-    vmo_ref(vmo);
-    mapping->start_page = addr / page_size;
-    mapping->end_page = ALIGN_UP(end, page_size) / page_size;
-    mapping->offset_page = vmo_offset / page_size;
-    mapping->object = vmo;
-    mapping->prot = flags;
+    *mapping = (struct vma_mapping){
+        .start_page = addr / page_size,
+        .end_page = ALIGN_UP(end, page_size) / page_size,
+        .offset_page = vmo_offset / page_size,
+        .object = vmo,
+        .vma = vma,
+        .prot = prot,
+        .flags = flags & VMA_SHARED,
+    };
+    if (vmo)
+        vmo_ref(vmo);
 
-    spin_lock(&vma->lock);
-    tailq_insert_head(&vma->mappings, mapping, link);
-    spin_unlock(&vma->lock);
+    mapping_list_t dead;
+    tailq_init(&dead);
 
+    rwlock_write_lock(&vma->lock);
+    carve(vma, mapping->start_page, mapping->end_page, &spare, &dead);
+    pmap_shootdown(&vma->pmap);
+    object_link(mapping);
+    tailq_insert_tail(&vma->mappings, mapping, link);
+    rwlock_write_unlock(&vma->lock);
+
+    free_mappings(&dead);
+    mem_free(spare);
+    return 0;
+}
+
+static bool protect_anons(struct vma* v, struct vma_mapping* m) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t num_slots = m->end_page - m->start_page;
+    const enum pte_flags flags = prot_to_pte(v, m->prot & ~PROT_WRITE);
+    struct vm_anon* batch[32];
+    uint64_t slots[32];
+    uint64_t next = m->amap_offset;
+    bool changed = false;
+
+    mutex_lock(&m->amap->lock);
+    while (true) {
+        const size_t found = radix_gang_lookup(&m->amap->anons, next, ARRAY_SIZE(batch), (void**)batch, slots);
+        if (found == 0)
+            break;
+        for (size_t i = 0; i < found; i++) {
+            if (slots[i] - m->amap_offset >= num_slots)
+                goto done;
+            const uintptr_t va = (m->start_page + (slots[i] - m->amap_offset)) * page_size;
+            if (pmap_protect(&v->pmap, va, flags) == 0)
+                changed = true;
+        }
+        next = slots[found - 1] + 1;
+    }
+done:
+    mutex_unlock(&m->amap->lock);
+    return changed;
+}
+
+errno_t vma_fork(struct vma* src, struct vma** out) {
+    struct vma* dst;
+    errno_t status = vma_new(&dst);
+    if (status)
+        return status;
+
+    bool changed = false;
+    rwlock_write_lock(&src->lock);
+    for (struct vma_mapping* m = tailq_first(&src->mappings); m; m = tailq_next(m, link)) {
+        struct vma_mapping* copy = mapping_alloc();
+        if (!copy) {
+            if (changed)
+                pmap_shootdown(&src->pmap);
+            rwlock_write_unlock(&src->lock);
+            vma_unref(dst);
+            return ENOMEM;
+        }
+
+        if (!(m->flags & VMA_SHARED) && m->amap) {
+            m->flags |= VMA_NEEDS_COPY;
+            if ((m->prot & PROT_WRITE) && protect_anons(src, m))
+                changed = true;
+        }
+
+        mapping_clone(copy, m, dst, m->start_page, m->end_page);
+        object_link(copy);
+        tailq_insert_tail(&dst->mappings, copy, link);
+    }
+    if (changed)
+        pmap_shootdown(&src->pmap);
+    rwlock_write_unlock(&src->lock);
+
+    *out = dst;
     return 0;
 }

mem/vmo.c

@@ -1,75 +1,250 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/alloc.h>
 #include <zinnia/errno.h>
-#include <zinnia/list.h>
+#include <zinnia/print.h>
 #include <zinnia/utils.h>
+#include <zinnia/vma.h>
 #include <zinnia/vmo.h>
+#include <string.h>
+
+constexpr size_t RA_MIN_PAGES = 8;
+constexpr size_t RA_MAX_PAGES = 64;
+
+void vmo_init(struct vmo* vmo, const struct vmo_pager_ops* pager) {
+    refcount_init(&vmo->rc);
+    vmo->pager = pager;
+    mutex_init(&vmo->lock);
+    radix_init(&vmo->pages);
+    tailq_init(&vmo->mappings);
+    vmo->ra_next = UINT64_MAX;
+    vmo->ra_window = RA_MIN_PAGES;
+    vmo->cache = CACHE_ANY;
+}
 
 void vmo_ref(struct vmo* obj) {
     refcount_inc(&obj->rc);
 }
 
+static void release_pages(struct vmo* vmo) {
+    struct page* batch[32];
+    uint64_t indices[32];
+    size_t found;
+    while ((found = radix_gang_lookup(&vmo->pages, 0, ARRAY_SIZE(batch), (void**)batch, indices)) != 0) {
+        for (size_t i = 0; i < found; i++) {
+            radix_remove(&vmo->pages, indices[i]);
+            page_unhold(batch[i]);
+        }
+    }
+    radix_destroy(&vmo->pages);
+}
+
 void vmo_unref(struct vmo* obj) {
-    if (refcount_dec(&obj->rc))
-        obj->destroy(obj);
+    if (!refcount_dec(&obj->rc))
+        return;
+
+    if (obj->pager->put_pages) {
+        errno_t status = vmo_sync(obj, 0, UINT64_MAX);
+        if (status)
+            log_warn("vmo: Dropping dirty pages after failed writeback (%i)\n", status);
+    }
+    release_pages(obj);
+    obj->pager->destroy(obj);
 }
 
-errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
+static void mark_dirty(struct vmo* vmo, struct page* page, uint64_t index) {
+    if (!vmo->pager->put_pages)
+        return;
+    atomic_fetch_or(&page->flags, PAGE_DIRTY, ATOMIC_RELAXED);
+    radix_tag_set(&vmo->pages, index, RADIX_TAG_DIRTY);
+}
+
+static void wait_busy(struct vmo* vmo, struct page* page) {
+    page_hold(page);
+    mutex_unlock(&vmo->lock);
+    page_wait(page);
+    page_unhold(page);
+    mutex_lock(&vmo->lock);
+}
+
+static size_t cluster_size(struct vmo* vmo, uint64_t index) {
+    const size_t max = MIN(vmo->pager->max_cluster, RA_MAX_PAGES);
+    if (!vmo->pager->get_pages || max <= 1)
+        return 1;
+
+    vmo->ra_window = index == vmo->ra_next ? MIN(vmo->ra_window * 2, max) : MIN(RA_MIN_PAGES, max);
+    size_t count = vmo->ra_window;
+    if (index > UINT64_MAX - count)
+        count = UINT64_MAX - index;
+    vmo->ra_next = index + count;
+
+    for (size_t i = 1; i < count; i++) {
+        if (radix_lookup(&vmo->pages, index + i))
+            return i;
+    }
+    return MAX(count, 1);
+}
+
+static errno_t fill(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) {
+    struct page* pages[RA_MAX_PAGES];
+    size_t count = cluster_size(vmo, index);
+    const enum alloc_flags flags = vmo->pager->get_pages ? ALLOC_NOZERO : 0;
+
+    for (size_t i = 0; i < count; i++) {
+        struct page* page;
+        errno_t status = page_alloc(flags, &page);
+        if (!status) {
+            page->obj.owner = vmo;
+            page->obj.index = index + i;
+            atomic_store(&page->flags, PAGE_BUSY, ATOMIC_RELAXED);
+            status = radix_insert(&vmo->pages, index + i, page);
+            if (status)
+                page_unhold(page);
+        }
+        if (status) {
+            if (i == 0) {
+                mutex_unlock(&vmo->lock);
+                return status;
+            }
+            count = i;
+            break;
+        }
+        pages[i] = page;
+    }
+
+    page_hold(pages[0]);
+    mutex_unlock(&vmo->lock);
+
+    size_t valid = count;
+    errno_t status = 0;
+    if (vmo->pager->get_pages)
+        status = vmo->pager->get_pages(vmo, index, pages, count, &valid);
+
+    mutex_lock(&vmo->lock);
+    for (size_t i = 0; i < count; i++) {
+        if (status || i >= valid) {
+            radix_remove(&vmo->pages, index + i);
+            page_unbusy(pages[i]);
+            page_unhold(pages[i]);
+            continue;
+        }
+        if (i == 0 && access == VMO_WRITE)
+            mark_dirty(vmo, pages[i], index);
+        page_unbusy(pages[i]);
+    }
+    mutex_unlock(&vmo->lock);
+
+    if (status || valid == 0) {
+        page_unhold(pages[0]);
+        return status ? status : ENXIO;
+    }
+
+    *out = pages[0];
+    return 0;
+}
+
+errno_t vmo_get_page(struct vmo* vmo, uint64_t index, enum vmo_access access, struct page** out) {
+    if (vmo->pager->lookup)
+        return ENOTSUP;
+
+    mutex_lock(&vmo->lock);
+    while (true) {
+        struct page* page = radix_lookup(&vmo->pages, index);
+        if (!page)
+            return fill(vmo, index, access, out);
+
+        if (atomic_load(&page->flags, ATOMIC_ACQUIRE) & PAGE_BUSY) {
+            wait_busy(vmo, page);
+            continue;
+        }
+
+        page_hold(page);
+        if (access == VMO_WRITE)
+            mark_dirty(vmo, page, index);
+        mutex_unlock(&vmo->lock);
+        *out = page;
+        return 0;
+    }
+}
+
+void vmo_put_page(struct page* page) {
+    page_unhold(page);
+}
+
+static errno_t copy_data(
+    struct vmo* obj,
+    uint64_t offset,
+    size_t len,
+    enum vmo_access access,
+    void* buf,
+    iovec_iter_t* iter,
+    size_t* out
+) {
     const size_t page_size = arch_mem_page_size();
     size_t progress = 0;
     errno_t status = 0;
 
     while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
+        const size_t misalign = (offset + progress) % page_size;
+        const uint64_t index = (offset + progress) / page_size;
+        const size_t chunk = MIN(page_size - misalign, len - progress);
 
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
+        struct page* page = nullptr;
+        phys_t phys;
+        if (obj->pager->lookup) {
+            status = obj->pager->lookup(obj, index, &phys);
+        } else {
+            status = vmo_get_page(obj, index, access, &page);
+            if (!status)
+                phys = vm_page_addr(page);
+        }
+        if (status)
             break;
 
-        uintptr_t page_addr = vm_page_addr(p);
+        uint8_t* ptr = (uint8_t*)HHDM_PTR(phys) + misalign;
+        size_t done = chunk;
+        if (iter && access == VMO_WRITE)
+            status = iovec_iter_copy_from(iter, ptr, chunk, &done);
+        else if (iter)
+            status = iovec_iter_copy_to(iter, ptr, chunk, &done);
+        else if (access == VMO_WRITE)
+            memcpy(ptr, (uint8_t*)buf + progress, chunk);
+        else
+            memcpy((uint8_t*)buf + progress, ptr, chunk);
 
-        memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
-        progress += copy_size;
+        if (page)
+            vmo_put_page(page);
+        progress += done;
+        if (status || done < chunk)
+            break;
     }
 
-    if (actual_read)
-        *actual_read = progress;
+    if (out)
+        *out = progress;
     return status;
 }
 
-errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
+errno_t vmo_read(struct vmo* obj, uint64_t offset, void* buf, size_t len, size_t* actual_read) {
+    return copy_data(obj, offset, len, VMO_READ, buf, nullptr, actual_read);
+}
 
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
+errno_t vmo_write(struct vmo* obj, uint64_t offset, const void* buf, size_t len, size_t* actual_written) {
+    return copy_data(obj, offset, len, VMO_WRITE, (void*)buf, nullptr, actual_written);
+}
 
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
+errno_t vmo_read_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_read) {
+    return copy_data(obj, offset, len, VMO_READ, nullptr, iter, actual_read);
+}
 
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_written)
-        *actual_written = progress;
-    return status;
+errno_t vmo_write_iter(struct vmo* obj, uint64_t offset, iovec_iter_t* iter, size_t len, size_t* actual_written) {
+    return copy_data(obj, offset, len, VMO_WRITE, nullptr, iter, actual_written);
 }
 
 errno_t vmo_copy(
     struct vmo* target,
-    uintptr_t target_offset,
+    uint64_t target_offset,
     struct vmo* src,
-    uintptr_t src_offset,
+    uint64_t src_offset,
     size_t len,
     size_t* actual_copied
 ) {
@@ -80,25 +255,27 @@ errno_t vmo_copy(
     while (progress < len) {
         const size_t target_misalign = (progress + target_offset) % page_size;
         const size_t src_misalign = (progress + src_offset) % page_size;
-        const size_t target_page_index = (progress + target_offset) / page_size;
-        const size_t src_page_index = (progress + src_offset) / page_size;
-
         const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
 
         struct page* tpage;
-        status = target->get_page(target, target_page_index, &tpage);
-        if (status != 0)
+        status = vmo_get_page(target, (progress + target_offset) / page_size, VMO_WRITE, &tpage);
+        if (status)
             break;
 
         struct page* spage;
-        status = src->get_page(src, src_page_index, &spage);
-        if (status != 0)
+        status = vmo_get_page(src, (progress + src_offset) / page_size, VMO_READ, &spage);
+        if (status) {
+            vmo_put_page(tpage);
             break;
+        }
 
-        uintptr_t taddr = vm_page_addr(tpage);
-        uintptr_t saddr = vm_page_addr(spage);
-
-        memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
+        memcpy(
+            (uint8_t*)HHDM_PTR(vm_page_addr(tpage)) + target_misalign,
+            (uint8_t*)HHDM_PTR(vm_page_addr(spage)) + src_misalign,
+            copy_size
+        );
+        vmo_put_page(spage);
+        vmo_put_page(tpage);
         progress += copy_size;
     }
 
@@ -107,157 +284,118 @@ errno_t vmo_copy(
     return status;
 }
 
-static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    // Try to find the requested page in the cache.
-    struct page_list* iter;
-    spin_lock(&paged->lock);
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
-            continue;
-
-        // We found a cached page!
-        *out = iter->value;
-        spin_unlock(&paged->lock);
+errno_t vmo_sync(struct vmo* vmo, uint64_t first, uint64_t end) {
+    if (!vmo->pager->put_pages)
         return 0;
-    }
-    spin_unlock(&paged->lock);
 
-    // Get page from out pager if we don't have one already.
-    struct page* new_page;
-    errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
-    if (status)
-        return status;
+    struct page* run[RA_MAX_PAGES];
+    const size_t max = MAX(MIN(vmo->pager->max_cluster, RA_MAX_PAGES), 1);
+    errno_t result = 0;
 
-    struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
-    if (!entry) {
-        paged->source.put_page(&paged->source, offset_idx, new_page);
-        return ENOMEM;
-    }
-
-    entry->offset = offset_idx;
-    entry->value = new_page;
-
-    spin_lock(&paged->lock);
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
+    mutex_lock(&vmo->lock);
+    while (first < end) {
+        struct page* found;
+        uint64_t index;
+        if (!radix_gang_lookup_tag(&vmo->pages, first, RADIX_TAG_DIRTY, 1, (void**)&found, &index) || index >= end)
+            break;
+        if (!page_try_busy(found)) {
+            wait_busy(vmo, found);
             continue;
+        }
 
-        *out = iter->value;
-        spin_unlock(&paged->lock);
-        mem_free(entry);
-        paged->source.put_page(&paged->source, offset_idx, new_page);
-        return 0;
+        size_t count = 1;
+        run[0] = found;
+        while (count < max && index + count < end && index + count != 0) {
+            struct page* next = radix_lookup(&vmo->pages, index + count);
+            if (!next || !radix_tag_get(&vmo->pages, index + count, RADIX_TAG_DIRTY) || !page_try_busy(next))
+                break;
+            run[count++] = next;
+        }
+
+        for (size_t i = 0; i < count; i++) {
+            radix_tag_clear(&vmo->pages, index + i, RADIX_TAG_DIRTY);
+            radix_tag_set(&vmo->pages, index + i, RADIX_TAG_WRITEBACK);
+            atomic_fetch_and(&run[i]->flags, ~PAGE_DIRTY, ATOMIC_RELAXED);
+            page_hold(run[i]);
+            vma_object_remove_page(vmo, index + i, vm_page_addr(run[i]), true);
+        }
+        mutex_unlock(&vmo->lock);
+
+        errno_t status = vmo->pager->put_pages(vmo, index, run, count);
+
+        mutex_lock(&vmo->lock);
+        for (size_t i = 0; i < count; i++) {
+            radix_tag_clear(&vmo->pages, index + i, RADIX_TAG_WRITEBACK);
+            if (status)
+                mark_dirty(vmo, run[i], index + i);
+            page_unbusy(run[i]);
+            page_unhold(run[i]);
+        }
+        if (status && !result)
+            result = status;
+        first = index + count;
+        if (first == 0)
+            break;
     }
-
-    // Add page to the object cache.
-    list_insert_head(&paged->cache, entry, next);
-    spin_unlock(&paged->lock);
-
-    *out = new_page;
-    return 0;
+    mutex_unlock(&vmo->lock);
+    return result;
 }
 
-static void paged_destroy(struct vmo* vmo) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    struct page_list* iter = list_first(&paged->cache);
-    while (iter) {
-        struct page_list* next = list_next(iter, next);
-        paged->source.put_page(&paged->source, iter->offset, iter->value);
-        mem_free(iter);
-        iter = next;
-    }
-
-    mem_free(paged);
-}
-
-errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
-    struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
-    if (!result)
-        return ENOMEM;
-
-    refcount_init(&result->object.rc);
-    result->source = pager;
-    result->object.get_page = paged_get_page;
-    result->object.destroy = paged_destroy;
-
-    *out = result;
-    return 0;
-}
-
-static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
-    phys_t addr;
-    errno_t status = mem_phys_alloc(1, 0, &addr);
-    if (status != 0)
-        return status;
-    size_t idx = addr / arch_mem_page_size();
-    *out = &vm_pfndb[idx];
-    return 0;
-}
-
-static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
-    return mem_phys_free(vm_page_addr(page), 1);
-}
-
-static const struct pager_ops phys_pager = {
-    .get_page = phys_get_page,
-    .put_page = phys_put_page,
-};
-
-errno_t vmo_new_phys(struct paged_vmo** out) {
-    return vmo_new_paged(phys_pager, out);
-}
-
-void vmo_truncate_paged(struct paged_vmo* paged, size_t length) {
+void vmo_truncate(struct vmo* vmo, uint64_t length) {
     const size_t page_size = arch_mem_page_size();
-    const size_t keep_pages = ROUND_UP(length, page_size);
+    uint64_t first = length / page_size + (length % page_size != 0);
 
-    spin_lock(&paged->lock);
-    struct page_list** pp = &list_first(&paged->cache);
-    while (*pp) {
-        struct page_list* entry = *pp;
-        if (entry->offset >= keep_pages) {
-            *pp = list_next(entry, next);
-            paged->source.put_page(&paged->source, entry->offset, entry->value);
-            mem_free(entry);
-        } else {
-            pp = &list_next(entry, next);
+    mutex_lock(&vmo->lock);
+    while (true) {
+        struct page* page;
+        uint64_t index;
+        if (!radix_gang_lookup(&vmo->pages, first, 1, (void**)&page, &index))
+            break;
+        if (!page_try_busy(page)) {
+            wait_busy(vmo, page);
+            continue;
+        }
+        vma_object_remove_page(vmo, index, vm_page_addr(page), false);
+        radix_remove(&vmo->pages, index);
+        page_unbusy(page);
+        page_unhold(page);
+        first = index + 1;
+    }
+
+    if (length % page_size != 0) {
+        while (true) {
+            struct page* tail = radix_lookup(&vmo->pages, length / page_size);
+            if (!tail)
+                break;
+            if (atomic_load(&tail->flags, ATOMIC_ACQUIRE) & PAGE_BUSY) {
+                wait_busy(vmo, tail);
+                continue;
+            }
+            memset((uint8_t*)HHDM_PTR(vm_page_addr(tail)) + length % page_size, 0, page_size - length % page_size);
+            mark_dirty(vmo, tail, length / page_size);
+            break;
         }
     }
-    spin_unlock(&paged->lock);
+    mutex_unlock(&vmo->lock);
 }
 
-struct phys_range_vmo {
-    struct vmo object;
-    uintptr_t start;
-    size_t len;
+struct anon_vmo {
+    struct vmo vmo;
 };
 
-static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
-    if (offset_idx >= p->len / arch_mem_page_size())
-        return EINVAL;
-    *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
-    return 0;
+static void anon_destroy(struct vmo* vmo) {
+    mem_free(containerof(vmo, struct anon_vmo, vmo));
 }
 
-static void phys_range_destroy(struct vmo* vmo) {
-    mem_free(containerof(vmo, struct phys_range_vmo, object));
-}
+static const struct vmo_pager_ops anon_pager = {
+    .destroy = anon_destroy,
+};
 
-errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
-    struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
-    if (!vmo)
+errno_t vmo_new_anon(struct vmo** out) {
+    struct anon_vmo* anon = mem_alloc(sizeof(struct anon_vmo), 0);
+    if (!anon)
         return ENOMEM;
-
-    refcount_init(&vmo->object.rc);
-    vmo->object.get_page = phys_range_get_page;
-    vmo->object.destroy = phys_range_destroy;
-    vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
-    vmo->len = ALIGN_UP(length, arch_mem_page_size());
-
-    *out = &vmo->object;
+    vmo_init(&anon->vmo, &anon_pager);
+    *out = &anon->vmo;
     return 0;
 }

meson.build

@@ -13,6 +13,7 @@ project(
 
 if get_option('headers_only')
     install_subdir('include/uapi', install_dir: get_option('includedir'))
+    install_subdir('include/sys', install_dir: get_option('includedir'))
     pkg = import('pkgconfig')
     pkg.generate(name: 'zinnia', description: 'Zinnia user API')
     subdir_done()

sched/core.c

@@ -4,6 +4,7 @@
 #include <zinnia/assert.h>
 #include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/stack.h>
@@ -25,7 +26,9 @@ void sched_to_user(uintptr_t info_ptr) {
 }
 
 void sched_to_user_context(uintptr_t ctx) {
-    arch_sched_jump_to_context((struct arch_context*)ctx);
+    struct arch_context context = *(struct arch_context*)ctx;
+    mem_free((void*)ctx);
+    arch_sched_jump_to_context(&context);
 }
 
 static struct spinlock task_cache_lock;
@@ -54,20 +57,25 @@ static void task_release(struct task* task) {
     spin_unlock(&task_cache_lock);
 }
 
-static void task_init_common(struct task* task, const char* name, struct vma* space) {
-    task->space = space;
+static void task_init_common(struct task* task, const char* name, struct process* proc) {
+    task->process = proc;
+    task->space = proc->space;
+    process_ref(proc);
+    if (task->space != &kernel_vma)
+        vma_ref(task->space);
     spin_init(&task->lock);
     atomic_store(&task->lockp, &task->lock, ATOMIC_RELAXED);
     strncpy(task->name, name, sizeof(task->name));
-    task->base_prio = SCHED_PRIO_DEFAULT;
+    task->base_prio = process_priority(proc);
     task->time_slice = SCHED_TIME_SLICE_NS;
     task->penalty = 0;
     task->feedback_epoch = 0;
-    atomic_store(&task->priority, SCHED_PRIO_DEFAULT, ATOMIC_RELAXED);
+    atomic_store(&task->priority, task->base_prio, ATOMIC_RELAXED);
     tailq_init(&task->pi_lenders);
+    process_attach_task(proc, task);
 }
 
-errno_t task_create_current(const char* name, struct vma* space, struct task** out) {
+errno_t task_create_current(const char* name, struct process* proc, struct task** out) {
     struct task* task = task_alloc();
     if (task == nullptr)
         return ENOMEM;
@@ -78,14 +86,14 @@ errno_t task_create_current(const char* name, struct vma* space, struct task** o
         return ENOMEM;
     }
 
-    task_init_common(task, name, space);
+    task_init_common(task, name, proc);
     *out = task;
     return 0;
 }
 
 constexpr size_t KERNEL_STACK_SIZE = 16 * 1024;
 
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
+errno_t task_create(const char* name, struct process* proc, task_fn_t entry, uintptr_t arg0, struct task** out) {
     if (!out)
         return EINVAL;
 
@@ -119,7 +127,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
         return s;
     }
 
-    task_init_common(new_task, name, space);
+    task_init_common(new_task, name, proc);
     *out = new_task;
     return 0;
 }
@@ -129,6 +137,10 @@ void task_destroy(struct task* task) {
     turnstile_free(task->turnstile);
     arch_task_fini(&task->context);
     stack_destroy(&task->kernel_stack);
+    process_detach_task(task->process, task);
+    if (task->space != &kernel_vma)
+        vma_unref(task->space);
+    process_unref(task->process);
     task_release(task);
 }
 

sched/internal.h

@@ -4,6 +4,6 @@
 
 #include <zinnia/sched.h>
 
-errno_t task_create_current(const char* name, struct vma* space, struct task** out);
+errno_t task_create_current(const char* name, struct process* proc, struct task** out);
 
 bool sched_update_priority_locked(struct task* task);

sched/sched_dumb.c

@@ -8,6 +8,7 @@
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/spin.h>
 #include <zinnia/stack.h>
@@ -57,12 +58,12 @@ void sched_init(struct scheduler* sched) {
         tailq_init(&sched->run_queue[i]);
 
     struct task* idle_task;
-    assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!");
+    assert(task_create("idle", process_kernel(), idle_func, 0, &idle_task) == 0, "Unable to create idle task!");
     idle_task->last_rq = sched;
     sched->idle_task = idle_task;
 
     struct task* boot_task;
-    assert(task_create_current("boot", &kernel_vma, &boot_task) == 0, "Unable to create boot task!");
+    assert(task_create_current("boot", process_kernel(), &boot_task) == 0, "Unable to create boot task!");
     boot_task->state = TASK_STATE_RUNNING;
     boot_task->last_rq = sched;
     atomic_store(&boot_task->on_cpu, true, ATOMIC_RELAXED);
@@ -326,6 +327,18 @@ void sched_wake(struct task* task) {
     task_unlock(task);
 }
 
+void sched_notify(struct task* task) {
+    task_lock(task);
+    if (task->state == TASK_STATE_BLOCKED) {
+        enqueue(select_cpu(task), task);
+    } else {
+        atomic_store(&task->wake_pending, true, ATOMIC_RELEASE);
+        if (task->state == TASK_STATE_RUNNING && task->last_rq && task->last_rq != &percpu_get()->sched)
+            kick(task->last_rq);
+    }
+    task_unlock(task);
+}
+
 bool sched_update_priority_locked(struct task* task) {
     struct scheduler* sched = task->last_rq;
     if (sched == nullptr) {

syscalls/archctl.c

@@ -1,6 +0,0 @@
-#include <zinnia/syscall.h>
-#include <arch/archctl.h>
-
-struct syscall_ret sys_archctl(uintptr_t op, uintptr_t arg) {
-    return arch_archctl(op, arg);
-}

syscalls/dispatch.c

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
 #include <zinnia/print.h>

syscalls/exit.c

@@ -1,12 +0,0 @@
-#include <zinnia/panic.h>
-#include <zinnia/syscall.h>
-
-struct syscall_ret sys_exit(int err) {
-    struct syscall_ret ret = {};
-
-    ret.error = ENOSYS;
-
-    panic("sys::exit() not implemented!\n");
-
-    return ret;
-}

syscalls/internal.h

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/percpu.h>
+#include <zinnia/process.h>
+#include <zinnia/sched.h>
+#include <zinnia/syscall.h>
+
+static inline struct process* self() {
+    return percpu_get()->sched.current->process;
+}

syscalls/memory.c

@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/atomic.h>
+#include <zinnia/mman.h>
+#include <zinnia/vfs.h>
+#include <zinnia/vma.h>
+#include <zinnia/vmo.h>
+#include <arch/mem.h>
+#include <stdckdint.h>
+#include <uapi/fcntl.h>
+#include "internal.h"
+
+static struct vma* space() {
+    return percpu_get()->sched.current->space;
+}
+
+constexpr enum prot_flags PROT_MASK = (PROT_READ | PROT_WRITE | PROT_EXEC);
+
+static bool is_user_range(uintptr_t addr, size_t len) {
+    uintptr_t last;
+    if (len == 0 || ckd_add(&last, addr, len - 1))
+        return false;
+    return arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(last);
+}
+
+static errno_t map_anon(struct vma* vma, uintptr_t addr, size_t len, uint32_t prot, uint32_t flags) {
+    if (!(flags & MAP_SHARED))
+        return vma_map(vma, nullptr, addr, len, prot, 0, 0);
+
+    struct vmo* anon;
+    errno_t status = vmo_new_anon(&anon);
+    if (status)
+        return status;
+    status = vma_map(vma, anon, addr, len, prot, VMA_SHARED, 0);
+    vmo_unref(anon);
+    return status;
+}
+
+static errno_t get_map_file(int fd, uint32_t prot, uint32_t flags, struct vfs_file** out) {
+    struct vfs_file* file = process_fd_get(self(), fd);
+    if (!file)
+        return EBADF;
+    if (!(atomic_load(&file->flags, ATOMIC_RELAXED) & O_RDONLY) ||
+        ((flags & MAP_SHARED) && (prot & PROT_WRITE) && !(atomic_load(&file->flags, ATOMIC_RELAXED) & O_WRONLY))) {
+        vfs_file_unref(file);
+        return EACCES;
+    }
+    *out = file;
+    return 0;
+}
+
+struct syscall_ret sys_mmap(uintptr_t addr, size_t len, uint32_t prot, uint32_t flags, int fd, off_t offset) {
+    struct syscall_ret ret = {};
+
+    const size_t page_size = arch_mem_page_size();
+    const uint32_t kind = flags & (MAP_SHARED | MAP_PRIVATE);
+
+    if (len == 0 || (prot & ~PROT_MASK) || (kind != MAP_SHARED && kind != MAP_PRIVATE)) {
+        ret.error = EINVAL;
+        return ret;
+    }
+    if (offset < 0 || (uint64_t)offset % page_size != 0) {
+        ret.error = EINVAL;
+        return ret;
+    }
+    if ((flags & MAP_FIXED) && (addr % page_size != 0 || !is_user_range(addr, len))) {
+        ret.error = EINVAL;
+        return ret;
+    }
+
+    struct vfs_file* file = nullptr;
+    if (!(flags & MAP_ANON)) {
+        ret.error = get_map_file(fd, prot, flags, &file);
+        if (ret.error)
+            return ret;
+    }
+
+    struct vma* vma = space();
+    mutex_lock(&vma->map_lock);
+
+    if (!(flags & MAP_FIXED))
+        ret.error = vma_find_free(vma, addr, len, &addr);
+
+    if (!ret.error) {
+        if (file)
+            ret.error = vfs_file_mmap(file, vma, addr, len, prot, flags, offset, &addr);
+        else
+            ret.error = map_anon(vma, addr, len, prot, flags);
+    }
+
+    mutex_unlock(&vma->map_lock);
+    if (file)
+        vfs_file_unref(file);
+    if (!ret.error)
+        ret.value = addr;
+    return ret;
+}
+
+struct syscall_ret sys_mprotect(uintptr_t addr, size_t len, uint32_t prot) {
+    struct syscall_ret ret = {};
+
+    if ((prot & ~PROT_MASK) || !is_user_range(addr, len)) {
+        ret.error = EINVAL;
+        return ret;
+    }
+
+    struct vma* vma = space();
+    mutex_lock(&vma->map_lock);
+    ret.error = vma_protect(vma, addr, len, prot);
+    mutex_unlock(&vma->map_lock);
+    return ret;
+}
+
+struct syscall_ret sys_munmap(uintptr_t addr, size_t len) {
+    struct syscall_ret ret = {};
+
+    if (!is_user_range(addr, len)) {
+        ret.error = EINVAL;
+        return ret;
+    }
+
+    struct vma* vma = space();
+    mutex_lock(&vma->map_lock);
+    ret.error = vma_unmap(vma, addr, len);
+    mutex_unlock(&vma->map_lock);
+    return ret;
+}
+
+struct syscall_ret sys_madvise(uintptr_t addr, size_t len, int advice) {
+    struct syscall_ret ret = {};
+
+    if (addr % arch_mem_page_size() != 0)
+        ret.error = EINVAL;
+    return ret;
+}
+
+struct syscall_ret sys_msync(uintptr_t addr, size_t len, int flags) {
+    struct syscall_ret ret = {};
+
+    if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) {
+        ret.error = EINVAL;
+        return ret;
+    }
+    if ((flags & MS_ASYNC) && (flags & MS_SYNC)) {
+        ret.error = EINVAL;
+        return ret;
+    }
+    if (addr % arch_mem_page_size() != 0 || !is_user_range(addr, len)) {
+        ret.error = EINVAL;
+        return ret;
+    }
+    if (flags & MS_ASYNC) {
+        if (!vma_range_mapped(space(), addr, len))
+            ret.error = ENOMEM;
+        return ret;
+    }
+    ret.error = vma_sync(space(), addr, len);
+    return ret;
+}

syscalls/meson.build

@@ -1,7 +1,7 @@
 kernel_sources += files(
-    'archctl.c',
     'dispatch.c',
-    'exit.c',
-    'syslog.c',
-    'uname.c',
+    'memory.c',
+    'process.c',
+    'system.c',
+    'vfs.c',
 )