todo

.clang-format

@@ -51,12 +51,16 @@ SpaceInEmptyBlock: false
 SpacesBeforeTrailingComments: 1
 IncludeBlocks: Merge
 IncludeCategories:
-  - Regex: "^<zinnia/.*>"
+- Regex: "^<zinnia/.*>"
-    Priority: 1
+  Priority: 1
-  - Regex: "^<kernel/.*>"
+- Regex: "^<common/.*>"
-    Priority: 2
+  Priority: 2
-  - Regex: '^<.*\.h>'
+- Regex: "^<kernel/.*>"
-    Priority: 3
+  Priority: 3
-  - Regex: '^".*\.h"'
+- Regex: "^<vdso/.*>"
-    Priority: 4
+  Priority: 4
+- Regex: '^<.*\.h>'
+  Priority: 5
+- Regex: '^".*\.h"'
+  Priority: 6
 SortIncludes: CaseInsensitive

.clangd

@@ -1,2 +1,2 @@
 Style:
-  AngledHeaders: ["zinnia/.*", " kernel/.*"]
+  AngledHeaders: [ "zinnia/.*", " kernel/.*" ]

.gitignore

@@ -1,13 +0,0 @@
-# IDE Files
-.vscode/
-.idea/
-.cache/
-
-# Build artifacts
-*.o
-*.a
-*.so
-compile_commands.json
-
-# Default build dir
-build/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "subprojects/uacpi"]
+	path = subprojects/uacpi
+	url = https://github.com/uACPI/uacpi

README.md

@@ -1,40 +1,23 @@
-<img src="zinnia.svg" width="10%"/>
+# Zinnia
 
 ![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=blue)
 ![GitHub Repo stars](https://img.shields.io/github/stars/zinnia-os/zinnia?style=flat)
-![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/zinnia-os/zinnia/ci.yml)
 ![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/zinnia-os/zinnia?style=flat)
 
-# zinnia
+Zinnia is a lightweight Unix-like microkernel targeting 64-bit devices.
 
-zinnia is a lightweight Unix-like microkernel targeting 64-bit devices.
+## Building Zinnia
 
-> [!IMPORTANT]
+To build the kernel and the servers you need `meson` and a C23-compatible
-> Please note that this project is a work in progress.
+GNU-like compiler targeting Zinnia.
-> Some parts may not work at all and/or will drastically change over time.
-
-## Building
-
-> [!NOTE]
-> This repository contains only the kernel and drivers.
-> If you want to get a bootable image, you might want to check out
-> **https://github.com/zinnia-os/bootstrap** instead.
-
-To build the kernel you need `meson` and a C23-compatible GNU-like compiler.
 Currently supported are GCC and Clang toolchains.
 
 To configure, run:
 ```sh
-meson setup $build_dir
+meson setup -Dbuild_kernel=true -Dbuild_servers=true $build_dir
 ```
 
 And to build:
 ```sh
 meson compile -C $build_dir
 ```
-
-To cross-compile, you should follow the Meson cross-compilation guide.
-
-## Debugging
-
-Follow the debugging guide at **https://github.com/zinnia-os/bootstrap**

include/common/compiler.h

@@ -1,7 +1,14 @@
+// SPDX-License-Identifier: MIT
+
 #pragma once
 
-// These hints may improve code generation, but are optional and can be stubbed.
+// Attributes that affect data layout.
-#if defined(__GNUC__) || defined(__clang__)
+#define __section(x) gnu::section(x)
+#define __aligned(x) gnu::aligned(x)
+#define __packed     gnu::packed
+#define __naked      gnu::naked
+
+// These hints may improve code generation.
 #define __cold              gnu::cold
 #define __hot               gnu::hot
 #define __used              gnu::used
@@ -9,17 +16,6 @@
 #define __format(like, ...) gnu::format(like, __VA_ARGS__)
 #define __likely(x)         __builtin_expect(!!(x), 1)
 #define __unlikely(x)       __builtin_expect(!!(x), 0)
-#define __always_inline     gnu::always_inline
-#else
-#define __cold
-#define __hot
-#define __used
-#define __unused
-#define __format(like, ...)
-#define __likely(x)   x
-#define __unlikely(x) x
-#define __always_inline
-#endif
 
 // Only clang has these hints, GCC will throw a warning here.
 // They also don't support the C23 attribute format.
@@ -30,17 +26,3 @@
 #define __user
 #define __phys
 #endif
-
-// Attributes that affect data layout. Not optional.
-#if defined(__GNUC__) || defined(__clang__)
-#define __weak       gnu::weak
-#define __section(x) gnu::section(x)
-#define __aligned(x) gnu::aligned(x)
-#define __packed     gnu::packed
-#define __naked      gnu::naked
-#else
-#error "Unsupported compiler!"
-#endif
-
-// Intrinsics
-#define __unreachable() __builtin_unreachable()

include/common/elf.h

@@ -0,0 +1,375 @@
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <stdint.h>
+
+constexpr char ELF_MAG[4] = {0x7F, 'E', 'L', 'F'};
+
+typedef uint64_t elf64_addr_t;
+typedef uint64_t elf64_off_t;
+typedef uint32_t elf32_addr_t;
+typedef uint32_t elf32_off_t;
+
+enum : uint8_t {
+    EI_MAG0 = 0,
+    EI_MAG1 = 1,
+    EI_MAG2 = 2,
+    EI_MAG3 = 3,
+    EI_CLASS = 4,
+    EI_DATA = 5,
+    EI_VERSION = 6,
+    EI_OSABI = 7,
+    EI_ABIVERSION = 8,
+    EI_PAD = 9,
+    EI_NIDENT = 16,
+};
+
+enum : uint16_t {
+    EM_X86_64 = 62,
+    EM_AARCH64 = 183,
+    EM_RISCV = 243,
+    EM_LOONGARCH = 258,
+};
+
+enum : uint32_t {
+    ELFCLASS32 = 1,
+    ELFCLASS64 = 2,
+};
+
+enum : uint32_t {
+    ELFDATA2LSB = 1,
+    ELFDATA2MSB = 2,
+};
+
+enum : uint32_t {
+    EV_NONE = 0,
+    EV_CURRENT = 1,
+    EV_NUM = 2,
+};
+
+enum : uint32_t {
+    ELFOSABI_SYSV = 0,
+    ELFOSABI_HPUX = 1,
+    ELFOSABI_STANDALONE = 255,
+};
+
+enum : uint32_t {
+    ET_NONE = 0,
+    ET_REL = 1,
+    ET_EXEC = 2,
+    ET_DYN = 3,
+    ET_CORE = 4,
+};
+
+enum : uint32_t {
+    PT_NULL = 0x00000000,
+    PT_LOAD = 0x00000001,
+    PT_DYNAMIC = 0x00000002,
+    PT_INTERP = 0x00000003,
+    PT_NOTE = 0x00000004,
+    PT_SHLIB = 0x00000005,
+    PT_PHDR = 0x00000006,
+    PT_TLS = 0x00000007,
+};
+
+enum : uint32_t {
+    PF_X = 0x00000001,
+    PF_W = 0x00000002,
+    PF_R = 0x00000004,
+};
+
+enum : uint32_t {
+    DT_NULL = 0,
+    DT_NEEDED = 1,
+    DT_PLTRELSZ = 2,
+    DT_PLTGOT = 3,
+    DT_HASH = 4,
+    DT_STRTAB = 5,
+    DT_SYMTAB = 6,
+    DT_RELA = 7,
+    DT_RELASZ = 8,
+    DT_RELAENT = 9,
+    DT_STRSZ = 10,
+    DT_SYMENT = 11,
+    DT_INIT = 12,
+    DT_FINI = 13,
+    DT_SONAME = 14,
+    DT_RPATH = 15,
+    DT_SYMBOLIC = 16,
+    DT_REL = 17,
+    DT_RELSZ = 18,
+    DT_RELENT = 19,
+    DT_PLTREL = 20,
+    DT_DEBUG = 21,
+    DT_TEXTREL = 22,
+    DT_JMPREL = 23,
+    DT_BIND_NOW = 24,
+    DT_INIT_ARRAY = 25,
+    DT_FINI_ARRAY = 26,
+    DT_INIT_ARRAYSZ = 27,
+    DT_FINI_ARRAYSZ = 28,
+    DT_LOOS = 0x60000000,
+    DT_HIOS = 0x6FFFFFFF,
+    DT_LOPROC = 0x70000000,
+    DT_HIPROC = 0x7FFFFFFF,
+};
+
+enum : uint32_t {
+    AT_NULL = 0,
+    AT_IGNORE = 1,
+    AT_EXECFD = 2,
+    AT_PHDR = 3,
+    AT_PHENT = 4,
+    AT_PHNUM = 5,
+    AT_PAGESZ = 6,
+    AT_BASE = 7,
+    AT_FLAGS = 8,
+    AT_ENTRY = 9,
+    AT_NOTELF = 10,
+    AT_UID = 11,
+    AT_EUID = 12,
+    AT_GID = 13,
+    AT_EGID = 14,
+    AT_HWCAP = 16,
+    AT_SECURE = 23,
+    AT_RANDOM = 25,
+    AT_EXECFN = 31,
+    AT_SYSINFO_EHDR = 33,
+    AT_INIT_HANDLE = 0x1000,
+};
+
+struct elf64_ehdr {
+    uint8_t e_ident[EI_NIDENT];
+    uint16_t e_type;
+    uint16_t e_machine;
+    uint32_t e_version;
+    elf64_addr_t e_entry;
+    elf64_off_t e_phoff;
+    elf64_off_t e_shoff;
+    uint32_t e_flags;
+    uint16_t e_ehsize;
+    uint16_t e_phentsize;
+    uint16_t e_phnum;
+    uint16_t e_shentsize;
+    uint16_t e_shnum;
+    uint16_t e_shstrndx;
+};
+
+struct elf32_ehdr {
+    uint8_t e_ident[EI_NIDENT];
+    uint16_t e_type;
+    uint16_t e_machine;
+    uint32_t e_version;
+    elf32_addr_t e_entry;
+    elf32_off_t e_phoff;
+    elf32_off_t e_shoff;
+    uint32_t e_flags;
+    uint16_t e_ehsize;
+    uint16_t e_phentsize;
+    uint16_t e_phnum;
+    uint16_t e_shentsize;
+    uint16_t e_shnum;
+    uint16_t e_shstrndx;
+};
+
+struct elf64_phdr {
+    uint32_t p_type;
+    uint32_t p_flags;
+    elf64_off_t p_offset;
+    elf64_addr_t p_vaddr;
+    elf64_addr_t p_paddr;
+    uint64_t p_filesz;
+    uint64_t p_memsz;
+    uint64_t p_align;
+};
+
+struct elf32_phdr {
+    uint32_t p_type;
+    elf32_off_t p_offset;
+    elf32_addr_t p_vaddr;
+    elf32_addr_t p_paddr;
+    uint32_t p_filesz;
+    uint32_t p_memsz;
+    uint32_t p_flags;
+    uint32_t p_align;
+};
+
+struct elf64_dyn {
+    int64_t d_tag;
+    union {
+        uint64_t d_val;
+        elf64_addr_t d_ptr;
+    } d_un;
+};
+
+struct elf32_dyn {
+    int32_t r_offset;
+    union {
+        uint32_t d_val;
+        elf32_addr_t d_ptr;
+    } d_un;
+};
+
+struct elf64_nhdr {
+    uint64_t n_namesz;
+    uint64_t n_descsz;
+    uint64_t n_type;
+};
+
+struct elf32_nhdr {
+    uint32_t n_namesz;
+    uint32_t n_descsz;
+    uint32_t n_type;
+};
+
+struct elf64_auxv {
+    uint64_t atype;
+    uint64_t avalue;
+};
+
+struct elf32_auxv {
+    uint32_t atype;
+    uint32_t avalue;
+};
+
+struct elf_hash {
+    uint32_t n_buckets;
+    uint32_t n_chain;
+};
+
+struct elf32_sym {
+    elf32_off_t st_name;
+    elf32_addr_t st_value;
+    elf32_off_t st_size;
+    uint8_t st_info;
+    uint8_t st_other;
+    uint16_t st_shndx;
+};
+
+struct elf64_sym {
+    uint32_t st_name;
+    uint8_t st_info;
+    uint8_t st_other;
+    uint16_t st_shndx;
+    uint64_t st_value;
+    uint64_t st_size;
+};
+
+static uint8_t ELF64_ST_BIND(uint8_t info) {
+    return info >> 4;
+}
+
+static inline uint8_t ELF64_ST_TYPE(uint8_t info) {
+    return info & 0x0F;
+}
+
+static inline uint8_t ELF64_ST_INFO(uint8_t bind, uint8_t type) {
+    return (bind << 4) | type;
+}
+
+typedef struct {
+    uint16_t si_boundto;
+    uint16_t si_flags;
+} elf64_syminfo;
+
+static inline uint8_t ELF32_ST_BIND(uint8_t info) {
+    return info >> 4;
+}
+
+static inline uint8_t ELF32_ST_TYPE(uint8_t info) {
+    return info & 0xF;
+}
+
+static inline uint8_t ELF32_ST_INFO(uint8_t bind, uint8_t type) {
+    return (bind << 4) | (type & 0xF);
+}
+
+enum {
+    STB_GLOBAL = 1,
+    STB_WEAK = 2,
+    STB_GNU_UNIQUE = 10,
+    STB_LOPROC = 13,
+    STB_HIPROC = 15
+};
+
+enum {
+    STT_NOTYPE = 0,
+    STT_OBJECT = 1,
+    STT_FUNC = 2,
+    STT_TLS = 6,
+    STT_GNU_IFUNC = 10,
+    STT_LOPROC = 13,
+    STT_HIPROC = 15
+};
+
+struct elf32_rela {
+    elf32_addr_t r_offset;
+    uint32_t r_info;
+    int32_t r_addend;
+};
+
+struct elf64_rela {
+    elf64_addr_t r_offset;
+    uint64_t r_info;
+    int64_t r_addend;
+};
+
+struct elf32_rel {
+    elf32_addr_t r_offset;
+    uint32_t r_info;
+};
+
+struct elf64_rel {
+    elf64_addr_t r_offset;
+    uint64_t r_info;
+};
+
+static inline uint32_t ELF32_R_SYM(uint32_t info) {
+    return info >> 8;
+}
+
+static inline uint32_t ELF32_R_TYPE(uint32_t info) {
+    return info & 0xF;
+}
+
+static inline uint32_t ELF64_R_SYM(uint64_t info) {
+    return info >> 32;
+}
+
+static inline uint32_t ELF64_R_TYPE(uint64_t info) {
+    return info & 0xFFFFFFFF;
+}
+
+// Architecture specific bits.
+
+#ifdef __x86_64__
+#define ELF_ARCH_CLASS   ELFCLASS64
+#define ELF_ARCH_DATA    ELFDATA2LSB
+#define ELF_ARCH_MACHINE EM_X86_64
+
+#define ELF_R_SYM   ELF64_R_SYM
+#define ELF_R_TYPE  ELF64_R_TYPE
+#define ELF_ST_BIND ELF64_ST_BIND
+#define ELF_ST_TYPE ELF64_ST_TYPE
+
+#define elf_ehdr elf64_ehdr
+#define elf_phdr elf64_phdr
+#define elf_dyn  elf64_dyn
+#define elf_addr elf64_addr
+#define elf_off  elf64_off
+#define elf_nhdr elf64_nhdr
+#define elf_auxv elf64_auxv
+#define elf_sym  elf64_sym
+#define elf_rela elf64_rela
+#define elf_rel  elf64_rel
+
+enum {
+    R_NONE = 0,
+    R_COPY = 5,
+    R_GLOB_DAT = 6,
+    R_JUMP_SLOT = 7,
+    R_RELATIVE = 8,
+    R_IRELATIVE = 37,
+};
+#endif

include/common/utils.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: MIT
+
 #pragma once
 
 #define ARRAY_SIZE(array) (sizeof(array) / sizeof(array[0]))
@@ -11,7 +13,23 @@
 
 #define BIT_TEST(value, bit) (((value) & (1 << bit)) == (1 << bit))
 
-#define CONTAINER_OF(value, p, field) ((p*)((char*)(&(value)) + offsetof(p, field)))
+#if defined(__has_builtin)
+#if __has_builtin(__builtin_types_compatible_p)
+// Get pointer to parent object given pointer to a struct or union member.
+#define containerof(ptr, type, member) \
+    ({ \
+        static_assert( \
+            __builtin_types_compatible_p(typeof(*(ptr)), typeof(((type*)0)->member)), \
+            "Incompatible types for containerof" \
+        ); \
+        (type*)((size_t)ptr - offsetof(type, member)); \
+    })
+#endif
+#endif
+#ifndef containerof
+// Get pointer to parent object given pointer to a struct or union member.
+#define containerof(ptr, type, member) (type*)((size_t)ptr - offsetof(type, member))
+#endif
 
 #define CONCAT(A, B)  _CONCAT(A, B)
 #define _CONCAT(A, B) A##B

include/ipc/common.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <zinnia/channel.h>
+#include <zinnia/handle.h>
+#include <common/utils.h>
+#include <stdint.h>
+
+#define ZN_IPC_NUM(group, num) ((uint32_t)(group) << 24u) | ((uint32_t)(num) & 0xffffffu)
+
+#define ZN_IPC(name, group, num, num_handles) \
+    name##_inner; \
+    constexpr uint32_t name##_num = ZN_IPC_NUM(group, num); \
+    typedef struct { \
+        uint32_t id; \
+        zn_handle_t handles[num_handles]; \
+        name##_inner inner; \
+    } name
+
+#define zn_ipc_write(channel, obj) \
+    zn_channel_write(channel, (obj)->handles, &(obj)->inner, ARRAY_SIZE((obj)->handles), sizeof((obj)->inner))
+
+#define zn_ipc_read(channel, obj) \
+    zn_channel_read( \
+        channel, \
+        (obj)->handles, \
+        &(obj)->inner, \
+        ARRAY_SIZE((obj)->handles), \
+        sizeof((obj)->inner), \
+        nullptr, \
+        nullptr \
+    )
+
+#define zn_ipc_new(name) \
+    (zn_ipc_fw_rsdp) { \
+        .id = name##_num \
+    }

include/zinnia/archctl.h

@@ -1,19 +1,31 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_ARCHCTL_H
 #define ZINNIA_ARCHCTL_H
 
 #include <zinnia/status.h>
-#include <stddef.h>
+#include <zinnia/syscall.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 typedef enum {
     // Does nothing.
     ZN_ARCHCTL_NONE = 0,
-#ifdef __x86_64__
+    // On x86_64, sets the FSBASE MSR.
-    // On x86_64, sets the FSBASE register to the value.
     ZN_ARCHCTL_SET_FSBASE = 1,
-#endif
+    // On x86_64, sets the GSBASE MSR.
+    ZN_ARCHCTL_SET_GSBASE = 2,
 } zn_archctl_t;
 
 // Performs an architecture-dependent operation identified by `op`.
-zn_status_t zn_archctl(zn_archctl_t op, size_t value);
+static inline zn_status_t zn_archctl(zn_archctl_t op, void* value) {
+    return zn_syscall2(op, value, ZN_SYSCALL_ARCHCTL);
+}
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif

include/zinnia/channel.h

@@ -1,35 +1,67 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_CHANNEL_H
 #define ZINNIA_CHANNEL_H
 
 #include <zinnia/handle.h>
 #include <zinnia/status.h>
+#include <zinnia/syscall.h>
 #include <stddef.h>
 
-// Handle which always points to a channel connected to the init server.
+#ifdef __cplusplus
-#define ZN_INIT_CHANNEL ((zn_handle_t)(-1))
+extern "C" {
+#endif
 
 enum zn_channel_flags {
-    // Allow sending messages even if one endpoint is not connected.
+    ZN_CHANNEL_NONE = 0,
     ZN_CHANNEL_NONBLOCK = 1 << 0,
 };
 
 // Creates a new channel.
-zn_status_t zn_channel_create(enum zn_channel_flags flags, zn_handle_t* endpoint0, zn_handle_t* endpoint1);
+static inline zn_status_t zn_channel_create(
+    enum zn_channel_flags flags,
+    zn_handle_t* endpoint0,
+    zn_handle_t* endpoint1
+) {
+    return zn_syscall3(flags, endpoint0, endpoint1, ZN_SYSCALL_CHANNEL_CREATE);
+}
 
-// Maps the message buffer in the address space and returns its base address.
+// Notifies the peer that there is new data available.
-// There may only be one message buffer per channel and process.
+// The kernel only copies the first `num_handles` handles and `num_bytes` bytes.
-zn_status_t zn_channel_open(
+static inline zn_status_t zn_channel_write(
     zn_handle_t channel,
+    const zn_handle_t* handles,
+    const void* bytes,
+    size_t num_handles,
+    size_t num_bytes
+) {
+    return zn_syscall5(channel, handles, bytes, num_handles, num_bytes, ZN_SYSCALL_CHANNEL_WRITE);
+}
+
+// Waits for a new message to appear in the channel.
+static inline zn_status_t zn_channel_read(
+    zn_handle_t channel,
+    zn_handle_t* handles,
+    void* bytes,
     size_t num_handles,
     size_t num_bytes,
-    zn_handle_t** out_handle_buf,
+    size_t* read_handles,
-    void** out_data_buf
+    size_t* read_bytes
-);
+) {
+    return zn_syscall7(
+        channel,
+        handles,
+        bytes,
+        num_handles,
+        num_bytes,
+        read_handles,
+        read_bytes,
+        ZN_SYSCALL_CHANNEL_READ
+    );
+}
 
-// Waits for a new message to appear in the channel buffer.
+#ifdef __cplusplus
-zn_status_t zn_channel_read(zn_handle_t channel);
+}
-
+#endif
-// Writes a message to the
-zn_status_t zn_channel_write(zn_handle_t channel);
 
 #endif

include/zinnia/futex.h

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: MIT
+
+#ifndef ZINNIA_FUTEX_H
+#define ZINNIA_FUTEX_H
+
+#include <zinnia/status.h>
+#include <zinnia/syscall.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Blocks the calling thread if *addr == expected.
+// Returns immediately if the value at addr has changed.
+static inline zn_status_t zn_futex_wait(int32_t* addr, int32_t expected) {
+    return zn_syscall2(addr, expected, ZN_SYSCALL_FUTEX_WAIT);
+}
+
+// Wakes up to `count` threads waiting on the given futex address.
+static inline zn_status_t zn_futex_wake(int32_t* addr, int32_t count) {
+    return zn_syscall2(addr, count, ZN_SYSCALL_FUTEX_WAKE);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/zinnia/handle.h

@@ -1,26 +1,46 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_HANDLE_H
 #define ZINNIA_HANDLE_H
 
 #include <zinnia/rights.h>
 #include <zinnia/status.h>
+#include <zinnia/syscall.h>
 #include <stdint.h>
 
-// A generic object handle.
+#ifdef __cplusplus
-typedef uint32_t zn_handle_t;
+extern "C" {
+#endif
 
-#define ZN_HANDLE_INVALID ((zn_handle_t)(0))
+// A generic object handle.
+typedef int32_t zn_handle_t;
+
+#define ZN_HANDLE_INVALID     ((zn_handle_t)(0))
+#define ZN_HANDLE_THIS_NS     ((zn_handle_t)(-1))
+#define ZN_HANDLE_THIS_VAS    ((zn_handle_t)(-2))
+#define ZN_HANDLE_THIS_THREAD ((zn_handle_t)(-3))
+#define ZN_HANDLE_ZERO_VMO    ((zn_handle_t)(-4))
 
 // Checks an object handle for validity.
-zn_status_t zn_handle_validate(zn_handle_t handle);
+zn_status_t zn_handle_validate(zn_handle_t handle) {
+    return zn_syscall1(handle, ZN_SYSCALL_HANDLE_VALIDATE);
+}
 
-// Drops a an object handle.
+// Drops an object handle. All further references using this handle are invalid.
-// All further references using this handle are invalid. The numerical value
+// The numerical value may become valid again, but it is an error to keep using it.
-// may become valid again, but it is an error to keep using it.
 // The underlying object gets freed only once no other handle references it.
-zn_status_t zn_handle_drop(zn_handle_t handle);
+zn_status_t zn_handle_drop(zn_handle_t handle) {
+    return zn_syscall1(handle, ZN_SYSCALL_HANDLE_DROP);
+}
 
 // Creates a new handle pointing to the same object as pointed to
 // by the original handle. The new handle may obtain new rights.
-zn_status_t zn_handle_clone(zn_handle_t handle, zn_rights_t cloned_rights, zn_handle_t* cloned);
+zn_status_t zn_handle_clone(zn_handle_t handle, zn_rights_t cloned_rights, zn_handle_t* cloned) {
+    return zn_syscall3(handle, cloned_rights, cloned, ZN_SYSCALL_HANDLE_CLONE);
+}
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif

include/zinnia/ipc/posix.h

@@ -1,15 +0,0 @@
-#ifndef ZINNIA_IPC_POSIX_H
-#define ZINNIA_IPC_POSIX_H
-
-#include <stdint.h>
-
-enum {
-    ZNI_POSIX_OPEN_REQ,
-    ZNI_POSIX_OPEN_RES,
-};
-
-struct zni_posix_open {
-    uint32_t id;
-};
-
-#endif

include/zinnia/log.h

@@ -1,11 +0,0 @@
-#ifndef ZINNIA_LOG_H
-#define ZINNIA_LOG_H
-
-#include <zinnia/status.h>
-
-// Panics and returns an error status to the parent process.
-void zn_panic(zn_status_t status);
-
-void zn_log(const char* message);
-
-#endif

include/zinnia/mem.h

@@ -1,32 +1,65 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_MEM_H
 #define ZINNIA_MEM_H
 
 #include <zinnia/handle.h>
 #include <zinnia/status.h>
 #include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 // Virtual memory flags.
 enum zn_vm_flags {
-    ZN_VM_READ = 1 << 0,
+    // Mapping is readable.
-    ZN_VM_WRITE = 1 << 1,
+    ZN_VM_MAP_READ = 1 << 0,
-    ZN_VM_EXEC = 1 << 2,
+    // Mapping is writable.
-    ZN_VM_SHARED = 1 << 3,
+    ZN_VM_MAP_WRITE = 1 << 1,
+    // Mapping is executable.
+    ZN_VM_MAP_EXEC = 1 << 2,
+    // Shared memory.
+    ZN_VM_MAP_SHARED = 1 << 3,
+
+    // Map at a fixed address, replacing whatever is in the way.
+    ZN_VM_MAP_FIXED = 1 << 8,
+    // Map at a fixed address, failing if anything is in the way.
+    ZN_VM_MAP_FIXED_NOREPLACE = 1 << 9,
+    // Map the entire range immediately.
+    ZN_VM_MAP_POPULATE = 1 << 10,
 };
 
-enum zn_cache_type {
+// Creates a new virtual memory object.
-    // Generic memory
+static inline zn_status_t zn_vmo_create(size_t length, zn_handle_t* out) {
-    ZN_CACHE_NORMAL,
+    return zn_syscall2(length, out, ZN_SYSCALL_VMO_CREATE);
-    // Write combining
+}
-    ZN_CACHE_WC,
-    // Memory-mapped IO
-    ZN_CACHE_MMIO,
-};
 
-// Allocates memory
+// Creates a new virtual memory object which points to a contiguous phyiscal memory region.
-zn_status_t zn_mem_alloc(size_t length, zn_handle_t* out);
+static inline zn_status_t zn_vmo_create_phys(uintptr_t phys_addr, size_t length, zn_handle_t* out) {
+    return zn_syscall3(phys_addr, length, out, ZN_SYSCALL_VMO_CREATE_PHYS);
+}
 
-zn_status_t zn_mem_map(zn_handle_t mem, void* addr, size_t len, enum zn_vm_flags flags);
+// Maps `bytes` of `vmo` at `vmo_offset` to `addr` in `vas`.
+static inline zn_status_t zn_vmo_map(
+    zn_handle_t vmo,
+    zn_handle_t vas,
+    uintptr_t vmo_offset,
+    uintptr_t* addr,
+    size_t bytes,
+    enum zn_vm_flags flags
+) {
+    return zn_syscall6(vmo, vas, vmo_offset, addr, bytes, flags, ZN_SYSCALL_VMO_MAP);
+}
 
-zn_status_t zn_mem_unmap(zn_handle_t mem, void* addr, size_t len, enum zn_vm_flags flags);
+// Creates a new virtual address space.
+static inline zn_status_t zn_vas_create(zn_handle_t* out) {
+    return zn_syscall1(out, ZN_SYSCALL_VAS_CREATE);
+}
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif

include/zinnia/namespace.h

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: MIT
+
+#ifndef ZINNIA_NAMESPACE_H
+#define ZINNIA_NAMESPACE_H
+
+#include <zinnia/handle.h>
+#include <zinnia/status.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Creates a new namespace.
+static inline zn_status_t zn_ns_create(zn_handle_t* out) {
+    return ZN_ERR_UNSUPPORTED;
+}
+
+// Moves a handle to another namespace.
+// The handle must have the `ZN_RIGHT_MOVE` right.
+static inline zn_status_t zn_ns_move(zn_handle_t handle, zn_handle_t target_ns) {
+    return ZN_ERR_UNSUPPORTED;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/zinnia/random.h

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: MIT
+
+#ifndef ZINNIA_RANDOM_H
+#define ZINNIA_RANDOM_H
+
+#include <zinnia/status.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ZN_RANDOM_MAX_ENTROPY 0x1000
+
+// Stores random bytes in a buffer.
+zn_status_t zn_random_get(void* addr, size_t len);
+
+// Adds entropy to the RNG.
+zn_status_t zn_random_entropy(const void* addr, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/zinnia/resource.h

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: MIT
+
+#ifndef ZINNIA_RESOURCE_H
+#define ZINNIA_RESOURCE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/zinnia/rights.h

@@ -1,6 +1,12 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_RIGHTS_H
 #define ZINNIA_RIGHTS_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef enum {
     // Can be read from.
     ZN_RIGHT_READ = 1u << 0,
@@ -10,7 +16,7 @@ typedef enum {
     ZN_RIGHT_EXECUTE = 1u << 2,
     // Can be mapped in an address space.
     ZN_RIGHT_MAP = 1u << 3,
-    // Can be moved to another process.
+    // Can be moved to another universe.
     ZN_RIGHT_MOVE = 1u << 4,
     // Can be cloned.
     ZN_RIGHT_CLONE = 1u << 5,
@@ -21,6 +27,11 @@ typedef enum {
     ZN_RIGHTS_IDENTICAL = 1u << 31,
     ZN_RIGHTS_COMMON = (ZN_RIGHT_MOVE | ZN_RIGHT_CLONE),
     ZN_RIGHTS_RW = (ZN_RIGHT_READ | ZN_RIGHT_WRITE),
+    ZN_RIGHTS_ALL = ~0u,
 } zn_rights_t;
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

include/zinnia/status.h

@@ -1,35 +1,43 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_STATUS_H
 #define ZINNIA_STATUS_H
 
-// Status values that may be returned by the kernel.
+#ifdef __cplusplus
-// A status value of 0 means everything is okay.
+extern "C" {
-// A negative value indicates an error from the kernel.
+#endif
-// Positive values are available to user processes.
+
+// Status values that may be returned by syscalls or other calls.
+// A status value of 0 means no error occured.
+// A negative value indicates a common error, usually from the kernel.
 typedef enum {
     ZN_OK = 0,
     // An internal error occured.
     ZN_ERR_INTERNAL = -1,
     // Syscall number is not a recognized syscall.
     ZN_ERR_BAD_SYSCALL = -2,
-    // This operation is not supported.
+    // This operation is not supported or implemented yet.
     ZN_ERR_UNSUPPORTED = -3,
     // System does not have enough free memory for this operation.
     ZN_ERR_NO_MEMORY = -4,
     // Process can not own any more handles.
     ZN_ERR_NO_HANDLES = -5,
     // One or more of the provided arguments is not valid.
-    ZN_ERR_BAD_ARG = -6,
+    ZN_ERR_INVALID = -6,
     // Argument is outside of the range for valid values.
     ZN_ERR_BAD_RANGE = -7,
     // Object handle does not name a valid object or correct type.
-    ZN_ERR_BAD_OBJECT = -8,
+    ZN_ERR_BAD_HANDLE = -8,
     // Object has insufficient permissions for this operation.
     ZN_ERR_BAD_PERMS = -9,
     // Buffer is not large enough or doesn't point to a valid memory region.
     ZN_ERR_BAD_BUFFER = -10,
+    // The resource already exists.
+    ZN_ERR_ALREADY_EXISTS = -11,
 } zn_status_t;
 
-// Returns a string describing the error code.
+#ifdef __cplusplus
-const char* zn_status_to_string(zn_status_t status);
+}
+#endif
 
 #endif

include/zinnia/syscall.h

@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: MIT
+
+#ifndef ZINNIA_SYSCALL_H
+#define ZINNIA_SYSCALL_H
+
+#include <zinnia/status.h>
+#include <zinnia/syscall_numbers.h>
+#include <stdint.h>
+
+typedef uintptr_t zn_arg_t;
+
+#ifdef __x86_64__
+#define ASM_REG_NUM "rax"
+#define ASM_REG_RET "rax"
+#define ASM_REG_A0  "rdi"
+#define ASM_REG_A1  "rsi"
+#define ASM_REG_A2  "rdx"
+#define ASM_REG_A3  "r9"
+#define ASM_REG_A4  "r8"
+#define ASM_REG_A5  "r10"
+#define ASM_REG_A6  "r12"
+#define ASM_REG_A7  "r13"
+#define ASM_SYSCALL "syscall"
+#define ASM_CLOBBER "rcx", "r11"
+#elif defined(__aarch64__)
+#define ASM_REG_NUM "x8"
+#define ASM_REG_RET "x0"
+#define ASM_REG_A0  "x0"
+#define ASM_REG_A1  "x1"
+#define ASM_REG_A2  "x2"
+#define ASM_REG_A3  "x3"
+#define ASM_REG_A4  "x4"
+#define ASM_REG_A5  "x5"
+#define ASM_REG_A6  "x6"
+#define ASM_REG_A7  "x7"
+#define ASM_SYSCALL "svc 0"
+#define ASM_CLOBBER
+#elif defined(__riscv) && (__riscv_xlen == 64)
+#define ASM_REG_NUM "t3"
+#define ASM_REG_RET "a0"
+#define ASM_REG_A0  "a0"
+#define ASM_REG_A1  "a1"
+#define ASM_REG_A2  "a2"
+#define ASM_REG_A3  "a3"
+#define ASM_REG_A4  "a4"
+#define ASM_REG_A5  "a5"
+#define ASM_REG_A5  "a6"
+#define ASM_REG_A5  "a7"
+#define ASM_SYSCALL "ecall"
+#define ASM_CLOBBER
+#elif defined(__loongarch64)
+#define ASM_REG_NUM "a7"
+#define ASM_REG_RET "a0"
+#define ASM_REG_A0  "a0"
+#define ASM_REG_A1  "a1"
+#define ASM_REG_A2  "a2"
+#define ASM_REG_A3  "a3"
+#define ASM_REG_A4  "a4"
+#define ASM_REG_A5  "a5"
+// TODO: Define ABI
+#define ASM_REG_A6  TODO
+#define ASM_REG_A7  TODO
+#define ASM_SYSCALL "syscall 0"
+#define ASM_CLOBBER
+#else
+#error "Unsupported architecture!"
+#endif
+
+static inline zn_status_t zn_syscall0(zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum) : "memory", ASM_CLOBBER);
+    return value;
+}
+
+static inline zn_status_t zn_syscall1(zn_arg_t a0, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall1(a0, num) zn_syscall1((zn_arg_t)a0, num)
+
+static inline zn_status_t zn_syscall2(zn_arg_t a0, zn_arg_t a1, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0), "r"(r1) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall2(a0, a1, num) zn_syscall2((zn_arg_t)a0, (zn_arg_t)a1, num)
+
+static inline zn_status_t zn_syscall3(zn_arg_t a0, zn_arg_t a1, zn_arg_t a2, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0), "r"(r1), "r"(r2) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall3(a0, a1, a2, num) zn_syscall3((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, num)
+
+static inline zn_status_t zn_syscall4(zn_arg_t a0, zn_arg_t a1, zn_arg_t a2, zn_arg_t a3, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall4(a0, a1, a2, a3, num) zn_syscall4((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, num)
+
+static inline zn_status_t zn_syscall5(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall5(a0, a1, a2, a3, a4, num) \
+    zn_syscall5((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, (zn_arg_t)a4, num)
+
+static inline zn_status_t zn_syscall6(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_arg_t a5,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    register zn_arg_t r5 asm(ASM_REG_A5) = a5;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall6(a0, a1, a2, a3, a4, a5, num) \
+    zn_syscall6((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, (zn_arg_t)a4, (zn_arg_t)a5, num)
+
+static inline zn_status_t zn_syscall7(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_arg_t a5,
+    zn_arg_t a6,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    register zn_arg_t r5 asm(ASM_REG_A5) = a5;
+    register zn_arg_t r6 asm(ASM_REG_A6) = a6;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r6)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall7(a0, a1, a2, a3, a4, a5, a6, num) \
+    zn_syscall7((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, (zn_arg_t)a4, (zn_arg_t)a5, (zn_arg_t)a6, num)
+
+static inline zn_status_t zn_syscall8(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_arg_t a5,
+    zn_arg_t a6,
+    zn_arg_t a7,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    register zn_arg_t r5 asm(ASM_REG_A5) = a5;
+    register zn_arg_t r6 asm(ASM_REG_A6) = a6;
+    register zn_arg_t r7 asm(ASM_REG_A7) = a7;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall8(a0, a1, a2, a3, a4, a5, a6, a7, num) \
+    zn_syscall8( \
+        (zn_arg_t)a0, \
+        (zn_arg_t)a1, \
+        (zn_arg_t)a2, \
+        (zn_arg_t)a3, \
+        (zn_arg_t)a4, \
+        (zn_arg_t)a5, \
+        (zn_arg_t)a6, \
+        (zn_arg_t)a7, \
+        num \
+    )
+
+#endif

include/zinnia/syscall_numbers.h

@@ -1,27 +1,38 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_SYSCALL_NUMBERS_H
 #define ZINNIA_SYSCALL_NUMBERS_H
 
-enum {
+typedef enum {
-    ZN_SYSCALL_PANIC,
-
     ZN_SYSCALL_LOG,
-
     ZN_SYSCALL_ARCHCTL,
+    ZN_SYSCALL_PAGE_SIZE,
+    ZN_SYSCALL_TIME,
 
     ZN_SYSCALL_HANDLE_VALIDATE,
     ZN_SYSCALL_HANDLE_DROP,
     ZN_SYSCALL_HANDLE_CLONE,
 
-    ZN_SYSCALL_MEM_ALLOC,
+    ZN_SYSCALL_VAS_CREATE,
-    ZN_SYSCALL_MEM_FREE,
+
+    ZN_SYSCALL_VMO_CREATE,
+    ZN_SYSCALL_VMO_CREATE_PHYS,
+    ZN_SYSCALL_VMO_MAP,
 
     ZN_SYSCALL_CHANNEL_CREATE,
-    ZN_SYSCALL_CHANNEL_OPEN,
+    ZN_SYSCALL_CHANNEL_WAIT,
     ZN_SYSCALL_CHANNEL_READ,
     ZN_SYSCALL_CHANNEL_WRITE,
 
     ZN_SYSCALL_THREAD_CREATE,
-    ZN_SYSCALL_THREAD_DESTROY,
+    ZN_SYSCALL_THREAD_START,
-};
+    ZN_SYSCALL_THREAD_EXIT,
+
+    ZN_SYSCALL_RANDOM_GET,
+    ZN_SYSCALL_RANDOM_ENTROPY,
+
+    ZN_SYSCALL_FUTEX_WAIT,
+    ZN_SYSCALL_FUTEX_WAKE,
+} zn_syscall_t;
 
 #endif

include/zinnia/syscall_stubs.h

@@ -0,0 +1,246 @@
+#pragma once
+
+#include <zinnia/status.h>
+#include <kernel/syscall_numbers.h>
+#include <stdint.h>
+
+#ifdef __x86_64__
+typedef uint64_t zn_arg_t;
+#elif defined(__aarch64__)
+typedef uint64_t zn_arg_t;
+#elif defined(__riscv) && (__riscv_xlen == 64)
+typedef uint64_t zn_arg_t;
+#elif defined(__loongarch64)
+typedef uint64_t zn_arg_t;
+#else
+#error "Unsupported architecture!"
+#endif
+
+#ifdef __x86_64__
+#define ASM_REG_NUM "rax"
+#define ASM_REG_RET "rax"
+#define ASM_REG_A0  "rdi"
+#define ASM_REG_A1  "rsi"
+#define ASM_REG_A2  "rdx"
+#define ASM_REG_A3  "r9"
+#define ASM_REG_A4  "r8"
+#define ASM_REG_A5  "r10"
+#define ASM_REG_A6  "r12"
+#define ASM_REG_A7  "r13"
+#define ASM_SYSCALL "syscall"
+#define ASM_CLOBBER "rcx", "r11"
+#elif defined(__aarch64__)
+#define ASM_REG_NUM "x8"
+#define ASM_REG_RET "x0"
+#define ASM_REG_A0  "x0"
+#define ASM_REG_A1  "x1"
+#define ASM_REG_A2  "x2"
+#define ASM_REG_A3  "x3"
+#define ASM_REG_A4  "x4"
+#define ASM_REG_A5  "x5"
+#define ASM_REG_A6  "x6"
+#define ASM_REG_A7  "x7"
+#define ASM_SYSCALL "svc 0"
+#define ASM_CLOBBER
+#elif defined(__riscv) && (__riscv_xlen == 64)
+#define ASM_REG_NUM "t3"
+#define ASM_REG_RET "a0"
+#define ASM_REG_A0  "a0"
+#define ASM_REG_A1  "a1"
+#define ASM_REG_A2  "a2"
+#define ASM_REG_A3  "a3"
+#define ASM_REG_A4  "a4"
+#define ASM_REG_A5  "a5"
+#define ASM_REG_A5  "a6"
+#define ASM_REG_A5  "a7"
+#define ASM_SYSCALL "ecall"
+#define ASM_CLOBBER
+#elif defined(__loongarch64)
+#define ASM_REG_NUM "a7"
+#define ASM_REG_RET "a0"
+#define ASM_REG_A0  "a0"
+#define ASM_REG_A1  "a1"
+#define ASM_REG_A2  "a2"
+#define ASM_REG_A3  "a3"
+#define ASM_REG_A4  "a4"
+#define ASM_REG_A5  "a5"
+// TODO: Define ABI
+#define ASM_REG_A6  TODO
+#define ASM_REG_A7  TODO
+#define ASM_SYSCALL "syscall 0"
+#define ASM_CLOBBER
+#else
+#error "Unsupported architecture!"
+#endif
+
+// These symbols should not be used in the kernel, ever.
+#ifdef IN_VDSO
+
+static inline zn_status_t zn_syscall0(zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum) : "memory", ASM_CLOBBER);
+    return value;
+}
+
+static inline zn_status_t zn_syscall1(zn_arg_t a0, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall1(a0, num) zn_syscall1((zn_arg_t)a0, num)
+
+static inline zn_status_t zn_syscall2(zn_arg_t a0, zn_arg_t a1, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0), "r"(r1) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall2(a0, a1, num) zn_syscall2((zn_arg_t)a0, (zn_arg_t)a1, num)
+
+static inline zn_status_t zn_syscall3(zn_arg_t a0, zn_arg_t a1, zn_arg_t a2, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0), "r"(r1), "r"(r2) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall3(a0, a1, a2, num) zn_syscall3((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, num)
+
+static inline zn_status_t zn_syscall4(zn_arg_t a0, zn_arg_t a1, zn_arg_t a2, zn_arg_t a3, zn_syscall_t num) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    asm volatile(ASM_SYSCALL : "=r"(value) : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3) : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall4(a0, a1, a2, a3, num) zn_syscall4((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, num)
+
+static inline zn_status_t zn_syscall5(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall5(a0, a1, a2, a3, a4, num) \
+    zn_syscall5((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, (zn_arg_t)a4, num)
+
+static inline zn_status_t zn_syscall6(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_arg_t a5,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    register zn_arg_t r5 asm(ASM_REG_A5) = a5;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall6(a0, a1, a2, a3, a4, a5, num) \
+    zn_syscall6((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, (zn_arg_t)a4, (zn_arg_t)a5, num)
+
+static inline zn_status_t zn_syscall7(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_arg_t a5,
+    zn_arg_t a6,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    register zn_arg_t r5 asm(ASM_REG_A5) = a5;
+    register zn_arg_t r6 asm(ASM_REG_A6) = a6;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r6)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall7(a0, a1, a2, a3, a4, a5, a6, num) \
+    zn_syscall7((zn_arg_t)a0, (zn_arg_t)a1, (zn_arg_t)a2, (zn_arg_t)a3, (zn_arg_t)a4, (zn_arg_t)a5, (zn_arg_t)a6, num)
+
+static inline zn_status_t zn_syscall8(
+    zn_arg_t a0,
+    zn_arg_t a1,
+    zn_arg_t a2,
+    zn_arg_t a3,
+    zn_arg_t a4,
+    zn_arg_t a5,
+    zn_arg_t a6,
+    zn_arg_t a7,
+    zn_syscall_t num
+) {
+    register zn_syscall_t rnum asm(ASM_REG_NUM) = num;
+    register zn_status_t value asm(ASM_REG_RET);
+    register zn_arg_t r0 asm(ASM_REG_A0) = a0;
+    register zn_arg_t r1 asm(ASM_REG_A1) = a1;
+    register zn_arg_t r2 asm(ASM_REG_A2) = a2;
+    register zn_arg_t r3 asm(ASM_REG_A3) = a3;
+    register zn_arg_t r4 asm(ASM_REG_A4) = a4;
+    register zn_arg_t r5 asm(ASM_REG_A5) = a5;
+    register zn_arg_t r6 asm(ASM_REG_A6) = a6;
+    register zn_arg_t r7 asm(ASM_REG_A7) = a7;
+    asm volatile(ASM_SYSCALL
+                 : "=r"(value)
+                 : "r"(rnum), "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7)
+                 : "memory", ASM_CLOBBER);
+    return value;
+}
+#define zn_syscall8(a0, a1, a2, a3, a4, a5, a6, a7, num) \
+    zn_syscall8( \
+        (zn_arg_t)a0, \
+        (zn_arg_t)a1, \
+        (zn_arg_t)a2, \
+        (zn_arg_t)a3, \
+        (zn_arg_t)a4, \
+        (zn_arg_t)a5, \
+        (zn_arg_t)a6, \
+        (zn_arg_t)a7, \
+        num \
+    )
+
+#endif

include/zinnia/system.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: MIT
+
+#ifndef ZINNIA_SYSTEM_H
+#define ZINNIA_SYSTEM_H
+
+#include <zinnia/status.h>
+#include <string.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Logs a message through the kernel.
+// Messages longer than an unspecified size will get truncated by the kernel.
+void zn_log(const char* message, size_t len);
+
+// Returns the page size of the system.
+size_t zn_page_size();
+
+zn_status_t zn_powerctl();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/zinnia/thread.h

@@ -1,14 +1,38 @@
+// SPDX-License-Identifier: MIT
+
 #ifndef ZINNIA_THREAD_H
 #define ZINNIA_THREAD_H
 
 #include <zinnia/handle.h>
 #include <zinnia/status.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 enum zn_thread_flags {
-    ZN_THREAD_NONE,
+    ZN_THREAD_NONE = 0,
 };
 
-// Creates a new thread.
+// Creates a new thread in the given namespace.
-zn_status_t zn_thread_create(const char* name, zn_handle_t address_space, enum zn_thread_flags flags);
+zn_status_t zn_thread_create(
+    zn_handle_t ns,
+    const char* name,
+    size_t name_len,
+    enum zn_thread_flags flags,
+    zn_handle_t* out
+);
+
+// Starts executing a thread.
+zn_status_t zn_thread_start(zn_handle_t thread, uintptr_t ip, uintptr_t sp, uintptr_t arg);
+
+// Stops execution of the calling thread.
+void zn_thread_exit(void);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif

kernel/arch/x86_64/apic.c

@@ -0,0 +1,108 @@
+#include <zinnia/mem.h>
+#include <kernel/clock.h>
+#include <kernel/mmio.h>
+#include <kernel/percpu.h>
+#include <kernel/pmap.h>
+#include <kernel/vas.h>
+#include <kernel/virt.h>
+#include <assert.h>
+#include <stdint.h>
+#include <x86_64/apic.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+
+static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
+    if (lapic->xapic_regs) {
+        if (reg == APIC_REG_ICR) {
+            uint64_t lo, hi;
+            lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR);
+            hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI);
+            return (hi << 32) | lo;
+        } else {
+            return mmio_read32(lapic->xapic_regs + reg);
+        }
+    } else {
+        return asm_rdmsr(0x800 + (reg >> 4));
+    }
+}
+
+static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
+    if (lapic->xapic_regs) {
+        if (reg == APIC_REG_ICR) {
+            mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32);
+            mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value);
+        } else {
+            mmio_write32(lapic->xapic_regs + reg, (uint32_t)value);
+        }
+    } else {
+        asm_wrmsr(0x800 + (reg >> 4), value);
+    }
+}
+
+void lapic_init(struct local_apic* lapic) {
+    uint64_t apic_msr = asm_rdmsr(0x1B);
+    apic_msr |= 1 << 11; // Enable APIC.
+
+    struct cpuid cpuid = asm_cpuid(1, 0);
+    if (cpuid.ecx & CPUID_1C_X2APIC)
+        apic_msr |= 1 << 10;
+    else {
+        uintptr_t vaddr = vm_alloc(0x1000);
+        pmap_map(&kernel_vas.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        lapic->xapic_regs = (void*)vaddr;
+        assert(lapic->xapic_regs, "Failed to allocate virtual memory!\n");
+    }
+
+    asm_wrmsr(0x1B, apic_msr);
+
+    // Reset the TPR.
+    lapic_write_reg(lapic, APIC_REG_TPR, 0);
+    // Enable APIC bit in the SIVR.
+    lapic_write_reg(lapic, APIC_REG_SIVR, lapic_read_reg(lapic, APIC_REG_SIVR) | 0x100);
+
+    if (lapic->xapic_regs) {
+        lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
+        // Logical destination = LAPIC ID.
+        lapic_write_reg(lapic, APIC_REG_LDR, lapic_read_reg(lapic, APIC_REG_ID));
+    }
+
+    // TODO: Parse MADT and setup NMI sources.
+
+    // Tell the APIC timer to divide by 16.
+    lapic_write_reg(lapic, APIC_REG_DCR, 3);
+    // Set the timer to the highest value possible.
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0xFFFF'FFFF);
+
+    // Sleep for 10 milliseconds.
+    clock_spin_ns(10'000'000);
+
+    lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
+    lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
+}
+
+void lapic_eoi(struct local_apic* lapic) {
+    lapic_write_reg(lapic, APIC_REG_EOI, 0);
+}
+
+#define PIC1_COMMAND_PORT 0x20
+#define PIC1_DATA_PORT    0x21
+#define PIC2_COMMAND_PORT 0xA0
+#define PIC2_DATA_PORT    0xA1
+
+void pic_disable() {
+    // We initialize the PIC properly, but completely disable it and use the APIC in favor of it.
+    // Remap IRQs so they start at 0x20 since interrupts 0x00..0x1F are used by CPU exceptions.
+    asm_outb(PIC1_COMMAND_PORT, 0x11);
+    asm_outb(PIC1_DATA_PORT, 0x20);
+    asm_outb(PIC1_DATA_PORT, 0x04);
+    asm_outb(PIC1_DATA_PORT, 0x01);
+    asm_outb(PIC1_DATA_PORT, 0xFF);
+
+    // Same for the slave PIC.
+    asm_outb(PIC2_COMMAND_PORT, 0x11);
+    asm_outb(PIC2_DATA_PORT, 0x28);
+    asm_outb(PIC2_DATA_PORT, 0x02);
+    asm_outb(PIC2_DATA_PORT, 0x01);
+    asm_outb(PIC2_DATA_PORT, 0xFF);
+}

kernel/arch/x86_64/gdt.c

@@ -1,36 +1,53 @@
 #include <kernel/percpu.h>
 #include <kernel/syscalls.h>
-#include <asm.h>
-#include <defs.h>
-#include <gdt.h>
 #include <stddef.h>
 #include <string.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+#include <x86_64/tss.h>
 
-struct gdt main_gdt = {
+static const struct gdt main_gdt = {
-    .null = 0,
+    .null = {.val = 0},
-    .kernel_code32 = 0x00cf9b000000ffff,
+    .kernel_code32 = {.val = 0x00cf9b000000ffff},
-    .kernel_data32 = 0x00cf93000000ffff,
+    .kernel_data32 = {.val = 0x00cf93000000ffff},
-    .kernel_code64 = 0x00a09b0000000000,
+    .kernel_code64 = {.val = 0x00a09b0000000000},
-    .kernel_data64 = 0x0000930000000000,
+    .kernel_data64 = {.val = 0x0000930000000000},
-    .user_code = 0x00cffb000000ffff,
+    .user_code32 = {.val = 0x00cffb000000ffff},
-    .user_data = 0x0000f30000000000,
+    .user_data = {.val = 0x0000f30000000000},
-    .user_code64 = 0x0000fb0000000000,
+    .user_code64 = {.val = 0x00a0fb0000000000},
-    .tss = {0x0000890000000000, 0},
+    .tss = {
+        .limit0 = sizeof(struct tss) - 1,
+        .access = 0x89, // Present | DPL = 0 | System | 64-bit TSS
+        .limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F,
+        .reserved = 0,
+    },
 };
 
-void gdt_init() {
+void gdt_load() {
-    // Set initial values.
+    struct gdt* gdt = &percpu_get()->arch.gdt;
-    memcpy(&percpu_get()->arch.gdt, &main_gdt, sizeof(struct gdt));
+    memcpy(gdt, &main_gdt, sizeof(main_gdt));
+
+    struct tss* tss = &percpu_get()->arch.tss;
+    tss->iopb = sizeof(struct tss);
+
+    uintptr_t tss_addr = (uintptr_t)tss;
+    gdt->tss.base0 = tss_addr;
+    gdt->tss.base1 = tss_addr >> 16;
+    gdt->tss.base2 = tss_addr >> 24;
+    gdt->tss.base3 = tss_addr >> 32;
 
     struct gdtr gdtr = {
         .limit = sizeof(struct gdt) - 1,
-        .base = &percpu_get()->arch.gdt,
+        .base = gdt,
     };
 
-    asm volatile("lgdt [%0]" ::"r"(&gdtr));
+    asm volatile("lgdt %0" ::"m"(gdtr));
 
     // Save the contents of MSR_GS_BASE, as they get cleared by a write to `gs`.
-    virt_t gs = asm_rdmsr(MSR_GS_BASE);
+    uintptr_t gs = asm_rdmsr(MSR_GS_BASE);
+
+    // Flush and reload the segment registers.
     asm volatile(
         "push %0\n"
         "lea rax, [rip + 1f]\n"
@@ -47,5 +64,14 @@ void gdt_init() {
         : "i"(offsetof(struct gdt, kernel_code64)), "i"(offsetof(struct gdt, kernel_data64))
         : "rax", "memory"
     );
+
     asm_wrmsr(MSR_GS_BASE, gs);
+
+    asm volatile(
+        "mov ax, %c0\n"
+        "ltr ax"
+        :
+        : "i"(offsetof(struct gdt, tss))
+        : "rax", "memory"
+    );
 }

kernel/arch/x86_64/hpet.c

@@ -0,0 +1,75 @@
+#include <common/utils.h>
+#include <kernel/clock.h>
+#include <kernel/cmdline.h>
+#include <kernel/pmap.h>
+#include <kernel/print.h>
+#include <kernel/vas.h>
+#include <kernel/virt.h>
+#include <assert.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <uacpi/uacpi.h>
+
+static bool use_hpet = true;
+static void hpet_option(bool use) {
+    use_hpet = use;
+}
+CMDLINE_OPTION("hpet", hpet_option);
+
+struct hpet {
+    struct clock clock;
+    volatile uint64_t* regs;
+    uint32_t period;
+};
+
+#define HPET_CAP     (0x0 / sizeof(uint64_t))
+#define HPET_CFG     (0x10 / sizeof(uint64_t))
+#define HPET_COUNTER (0xF0 / sizeof(uint64_t))
+
+static void hpet_reset(struct clock* c) {
+    struct hpet* hpet = containerof(c, struct hpet, clock);
+    hpet->regs[HPET_COUNTER] = 0;
+}
+
+static uint64_t hpet_get_elapsed_ns(struct clock* c) {
+    struct hpet* hpet = containerof(c, struct hpet, clock);
+    uint64_t counter = hpet->regs[HPET_COUNTER];
+    return (counter * (uint64_t)hpet->period / 1'000'000);
+}
+
+static struct hpet hpet_clock = {
+    .clock = {
+        .name = "hpet",
+        .priority = 128,
+        .reset = hpet_reset,
+        .get_elapsed_ns = hpet_get_elapsed_ns,
+    },
+};
+
+static char table_buf[0x1000];
+
+void hpet_init() {
+    if (!use_hpet)
+        return;
+
+    uacpi_setup_early_table_access(table_buf, sizeof(table_buf));
+    uacpi_table table = {0};
+    assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
+
+    struct acpi_hpet* hpet = table.ptr;
+
+    uintptr_t vaddr = vm_alloc(0x1000);
+    pmap_map(&kernel_vas.pmap, vaddr, hpet->address.address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+    hpet_clock.regs = (void*)vaddr;
+
+    uacpi_table_unref(&table);
+    kprintf("HPET regs at %p\n", hpet_clock.regs);
+
+    hpet_clock.regs[HPET_CFG] = hpet_clock.regs[HPET_CFG] | 1;
+
+    uint64_t caps = hpet_clock.regs[HPET_CAP];
+    hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
+
+    assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");
+}

kernel/arch/x86_64/idt.c

@@ -1,7 +1,209 @@
-#include <kernel/compiler.h>
+#include <common/compiler.h>
+#include <kernel/irq.h>
+#include <kernel/panic.h>
+#include <kernel/percpu.h>
+#include <kernel/print.h>
+#include <kernel/sched.h>
+#include <assert.h>
+#include <bits/sched.h>
+#include <stddef.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+
+struct [[__packed]] idt_entry {
+    uint16_t base0;
+    uint16_t selector;
+    uint8_t ist;
+    uint8_t attrib;
+    uint16_t base1;
+    uint32_t base2;
+    uint32_t reserved;
+};
+
+static_assert(sizeof(struct idt_entry) == 16);
 
 struct [[__packed]] idtr {
     uint16_t limit;
-    struct idt* base;
+    struct idt_entry* base;
 };
+
+static_assert(sizeof(struct idtr) == 10);
+
+void interrupt_handler(struct arch_context* ctx) {
+    struct percpu* cpu = percpu_get();
+    bool old = irq_set_interrupted(true);
+
+    uint64_t isr = ctx->isr;
+    switch (isr) {
+    case IDT_PF:
+        uint64_t cr2;
+        asm volatile("mov %0, cr2" : "=r"(cr2));
+        panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
+    case IDT_IPI_RESCHED:
+        arch_sched_preempt_disable();
+        if (arch_sched_preempt_enable()) {
+            lapic_eoi(&cpu->arch.lapic);
+            sched_reschedule(&cpu->sched);
+        }
+        break;
+    case 0x20 ...(0x20 + ARRAY_SIZE(cpu->arch.irq_lines)):
+        struct irq_line* line = cpu->arch.irq_lines[isr - 0x20];
+        assert(line, "Unhandled interrupt on ISR %lu\n", isr);
+        // TODO: irq_raise(line);
+        break;
+    default:
+        panic("Unhandled exception on ISR %lu\n", isr);
+    }
+
+    irq_set_interrupted(old);
+}
+
+#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
+
+[[__naked]]
+void interrupt_return() {
+    asm volatile(
+        "pop r15\n"
+        "pop r14\n"
+        "pop r13\n"
+        "pop r12\n"
+        "pop r11\n"
+        "pop r10\n"
+        "pop r9\n"
+        "pop r8\n"
+        "pop rsi\n"
+        "pop rdi\n"
+        "pop rbp\n"
+        "pop rdx\n"
+        "pop rcx\n"
+        "pop rbx\n"
+        "pop rax\n"
+        // Change GS back if we came from user mode.
+        "cmp word ptr [rsp + %c0], %c1\n"
+        "je 2f\n"
+        "swapgs\n"
+        "2:\n"
+        // Skip .error and .isr fields.
+        "add rsp, 0x10\n"
+        "iretq\n"
+        :
+        : "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64))
+    );
+}
+
+[[__naked]]
+void interrupt_stub_internal() {
+    asm volatile(
+        // Load the kernel GS base if we're coming from user space.
+        "cmp word ptr [rsp + %c0], %c1\n"
+        "je 2f\n"
+        "swapgs\n"
+        "2:\n"
+        "push rax\n"
+        "push rbx\n"
+        "push rcx\n"
+        "push rdx\n"
+        "push rbp\n"
+        "push rdi\n"
+        "push rsi\n"
+        "push r8\n"
+        "push r9\n"
+        "push r10\n"
+        "push r11\n"
+        "push r12\n"
+        "push r13\n"
+        "push r14\n"
+        "push r15\n"
+        "cld\n"
+        // Zero out the base pointer since we can't trust it.
+        "xor rbp, rbp\n"
+        // Load the frame as first argument.
+        "mov rdi, rsp\n"
+        "call %c2\n"
+        "jmp %c3\n"
+        :
+        : "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64)), "i"(interrupt_handler), "i"(interrupt_return)
+    );
+}
+
+// clang-format off
+#define REPEAT_256(M) \
+    M(0)   M(1)   M(2)   M(3)   M(4)   M(5)   M(6)   M(7)   \
+    M(8)   M(9)   M(10)  M(11)  M(12)  M(13)  M(14)  M(15)  \
+    M(16)  M(17)  M(18)  M(19)  M(20)  M(21)  M(22)  M(23)  \
+    M(24)  M(25)  M(26)  M(27)  M(28)  M(29)  M(30)  M(31)  \
+    M(32)  M(33)  M(34)  M(35)  M(36)  M(37)  M(38)  M(39)  \
+    M(40)  M(41)  M(42)  M(43)  M(44)  M(45)  M(46)  M(47)  \
+    M(48)  M(49)  M(50)  M(51)  M(52)  M(53)  M(54)  M(55)  \
+    M(56)  M(57)  M(58)  M(59)  M(60)  M(61)  M(62)  M(63)  \
+    M(64)  M(65)  M(66)  M(67)  M(68)  M(69)  M(70)  M(71)  \
+    M(72)  M(73)  M(74)  M(75)  M(76)  M(77)  M(78)  M(79)  \
+    M(80)  M(81)  M(82)  M(83)  M(84)  M(85)  M(86)  M(87)  \
+    M(88)  M(89)  M(90)  M(91)  M(92)  M(93)  M(94)  M(95)  \
+    M(96)  M(97)  M(98)  M(99)  M(100) M(101) M(102) M(103) \
+    M(104) M(105) M(106) M(107) M(108) M(109) M(110) M(111) \
+    M(112) M(113) M(114) M(115) M(116) M(117) M(118) M(119) \
+    M(120) M(121) M(122) M(123) M(124) M(125) M(126) M(127) \
+    M(128) M(129) M(130) M(131) M(132) M(133) M(134) M(135) \
+    M(136) M(137) M(138) M(139) M(140) M(141) M(142) M(143) \
+    M(144) M(145) M(146) M(147) M(148) M(149) M(150) M(151) \
+    M(152) M(153) M(154) M(155) M(156) M(157) M(158) M(159) \
+    M(160) M(161) M(162) M(163) M(164) M(165) M(166) M(167) \
+    M(168) M(169) M(170) M(171) M(172) M(173) M(174) M(175) \
+    M(176) M(177) M(178) M(179) M(180) M(181) M(182) M(183) \
+    M(184) M(185) M(186) M(187) M(188) M(189) M(190) M(191) \
+    M(192) M(193) M(194) M(195) M(196) M(197) M(198) M(199) \
+    M(200) M(201) M(202) M(203) M(204) M(205) M(206) M(207) \
+    M(208) M(209) M(210) M(211) M(212) M(213) M(214) M(215) \
+    M(216) M(217) M(218) M(219) M(220) M(221) M(222) M(223) \
+    M(224) M(225) M(226) M(227) M(228) M(229) M(230) M(231) \
+    M(232) M(233) M(234) M(235) M(236) M(237) M(238) M(239) \
+    M(240) M(241) M(242) M(243) M(244) M(245) M(246) M(247) \
+    M(248) M(249) M(250) M(251) M(252) M(253) M(254) M(255)
+// clang-format on
+
+#define INTERRUPT_STUB(n) \
+    [[__naked]] \
+    static void interrupt_stub_##n() { \
+        asm volatile( \
+            ".if (%c0 == 8 || (%c0 >= 10 && %c0 <= 14) || %c0 == 17 || %c0 == 21 || %c0 == 29 || %c0 == 30)\n" \
+            ".else\n" \
+            "push 0\n" \
+            ".endif\n" \
+            "push %c0\n" \
+            "jmp %c1\n" \
+            : \
+            : "i"(n), "i"(interrupt_stub_internal) \
+        ); \
+    }
+
+#define INTERRUPT_SETUP(n) \
+    do { \
+        uint64_t addr = (uint64_t)interrupt_stub_##n; \
+        idt[n].base0 = addr & 0xFFFF; \
+        idt[n].base1 = (addr >> 16) & 0xFFFF; \
+        idt[n].base2 = (addr >> 32) & 0xFFFF'FFFF; \
+        idt[n].ist = 0; \
+        idt[n].attrib = (1 << 7) | 0xE; \
+        idt[n].selector = offsetof(struct gdt, kernel_code64); \
+        idt[n].reserved = 0; \
+    } while (0);
+
+REPEAT_256(INTERRUPT_STUB)
+
+static struct idt_entry idt[256] = {};
+
+void idt_init() {
+    REPEAT_256(INTERRUPT_SETUP)
+}
+
+void idt_load() {
+    struct idtr idtr = {
+        .limit = sizeof(idt) - 1,
+        .base = idt,
+    };
+
+    asm volatile("lidt %0" ::"m"(idtr));
+}

kernel/arch/x86_64/include/bits/elf.h

@@ -1,13 +0,0 @@
-#pragma once
-
-#define ELF_ARCH_CLASS   ELFCLASS64
-#define ELF_ARCH_DATA    ELFDATA2LSB
-#define ELF_ARCH_MACHINE EM_X86_64
-
-#define elf_hdr  elf64_hdr
-#define elf_phdr elf64_phdr
-#define elf_dyn  elf64_dyn
-#define elf_addr elf64_addr
-#define elf_off  elf64_off
-#define elf_nhdr elf64_nhdr
-#define elf_auxv elf64_auxv

kernel/arch/x86_64/include/bits/mem.h

@@ -1,41 +0,0 @@
-#pragma once
-
-#include <kernel/types.h>
-#include <stddef.h>
-#include <stdint.h>
-
-static inline size_t arch_mem_page_bits() {
-    return 12;
-}
-
-static inline size_t arch_mem_level_bits() {
-    return 9;
-}
-
-static inline size_t arch_mem_num_levels() {
-    return 4;
-}
-
-static inline virt_t arch_mem_hhdm_addr() {
-    return 0xFFFF'8000'0000'0000;
-}
-
-static inline virt_t arch_mem_pfndb_addr() {
-    return 0xFFFF'A000'0000'0000;
-}
-
-static inline virt_t arch_mem_mapping_addr() {
-    return 0xFFFF'C000'0000'0000;
-}
-
-typedef uint64_t pte_t;
-enum pte_flags;
-enum cache_mode;
-struct page_table;
-
-void arch_mem_pte_clear(pte_t* pte);
-pte_t arch_mem_pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
-bool arch_mem_pte_is_present(pte_t* pte);
-bool arch_mem_pte_is_dir(pte_t* pte);
-phys_t arch_mem_pte_address(pte_t* pte);
-void arch_mem_pt_set(struct page_table* pt);

kernel/arch/x86_64/include/bits/mmu.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include <stdint.h>
+
+typedef uint64_t pte_t;

kernel/arch/x86_64/include/bits/percpu.h

@@ -1,16 +1,18 @@
 #pragma once
 
-#include <gdt.h>
+#include <kernel/irq.h>
-#include <stdint.h>
+#include <stddef.h>
-
+#include <x86_64/apic.h>
-static inline struct percpu* arch_percpu_get() {
+#include <x86_64/gdt.h>
-    struct percpu* result;
+#include <x86_64/tss.h>
-    asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
-    return result;
-}
 
 struct arch_percpu {
-    uint32_t lapic_id;
     struct gdt gdt;
     struct tss tss;
+    struct local_apic lapic;
+    struct irq_line* irq_lines[128];
+    size_t fpu_size;
+    void (*fpu_save)(void*);
+    void (*fpu_restore)(const void*);
+    bool can_smap;
 };

kernel/arch/x86_64/include/bits/sched.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <bits/types.h>
 #include <stdint.h>
 
 struct arch_task_context {
@@ -9,22 +10,27 @@ struct arch_task_context {
     uint64_t fs_base, gs_base;
 };
 
+#define ARCH_ABI(typ, reg, ...) \
+    union { \
+        typ reg, __VA_ARGS__; \
+    }
+
 struct arch_context {
     uint64_t r15;
     uint64_t r14;
-    uint64_t r13;
+    ARCH_ABI(uint64_t, r13, a7);
-    uint64_t r12;
+    ARCH_ABI(uint64_t, r12, a6);
     uint64_t r11;
-    uint64_t r10;
+    ARCH_ABI(uint64_t, r10, a5);
-    uint64_t r9;
+    ARCH_ABI(uint64_t, r9, a3);
-    uint64_t r8;
+    ARCH_ABI(uint64_t, r8, a4);
-    uint64_t rsi;
+    ARCH_ABI(uint64_t, rsi, a1);
-    uint64_t rdi;
+    ARCH_ABI(uint64_t, rdi, a0);
     uint64_t rbp;
-    uint64_t rdx;
+    ARCH_ABI(uint64_t, rdx, a2, ret1);
     uint64_t rcx;
     uint64_t rbx;
-    uint64_t rax;
+    ARCH_ABI(uint64_t, rax, num, ret0);
     uint64_t isr;   // Pushed onto the stack by the interrupt handler stubs.
     uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code.
     uint64_t rip;   // The rest is pushed onto the stack by the CPU during an interrupt.
@@ -33,3 +39,17 @@ struct arch_context {
     uint64_t rsp;
     uint64_t ss;
 };
+
+static_assert(sizeof(struct arch_context) == 22 * sizeof(uint64_t));
+
+#define ARCH_CTX_NUM  rax
+#define ARCH_CTX_A0   rdi
+#define ARCH_CTX_A1   rsi
+#define ARCH_CTX_A2   rdx
+#define ARCH_CTX_A3   r9
+#define ARCH_CTX_A4   r8
+#define ARCH_CTX_A5   r10
+#define ARCH_CTX_A6   r12
+#define ARCH_CTX_A7   r13
+#define ARCH_CTX_RET0 rax
+#define ARCH_CTX_RET1 rdx

kernel/arch/x86_64/include/bits/types.h

@@ -3,5 +3,4 @@
 #include <stdint.h>
 
 typedef uint64_t phys_t;
-typedef uint64_t virt_t;
 typedef uint64_t reg_t;

kernel/arch/x86_64/include/bits/usercopy.h

@@ -1,9 +0,0 @@
-#pragma once
-
-#include <kernel/compiler.h>
-#include <stddef.h>
-#include <stdint.h>
-
-bool arch_usercopy_read(uint8_t* dst, const __user uint8_t* src, size_t len);
-bool arch_usercopy_write(__user uint8_t* dst, const uint8_t* src, size_t len);
-bool arch_usercopy_strlen(const __user uint8_t* str, size_t max, size_t* len);

kernel/arch/x86_64/include/gdt.h

@@ -1,28 +0,0 @@
-#pragma once
-
-#include <kernel/compiler.h>
-#include <stdint.h>
-#include "tss.h"
-
-struct [[__packed]] gdt {
-    uint64_t null;
-    uint64_t kernel_code32;
-    uint64_t kernel_data32;
-    uint64_t kernel_code64;
-    uint64_t kernel_data64;
-    uint64_t user_code;
-    uint64_t user_data;
-    uint64_t user_code64;
-    uint64_t tss[2];
-};
-
-struct [[__packed]] gdtr {
-    uint16_t limit;
-    struct gdt* base;
-};
-
-// Initializes a GDT on the local core.
-void gdt_init();
-
-// Sets the Task State Segment in the given GDT.
-void gdt_set_tss(struct gdt* gdt, struct tss* tss);

kernel/arch/x86_64/include/idt.h

@@ -1 +0,0 @@
-#pragma once

kernel/arch/x86_64/include/x86_64/apic.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <stdint.h>
+
+struct local_apic {
+    uint32_t id;
+    uint32_t ticks_per_10ms;
+    void* xapic_regs;
+};
+
+enum apic_delivery_mode {
+    APIC_DELIVERY_FIXED = 0b000,
+    APIC_DELIVERY_LOWESTPRIO = 0b001,
+    APIC_DELIVERY_SMI = 0b010,
+    APIC_DELIVERY_NMI = 0b100,
+    APIC_DELIVERY_INIT = 0b101,
+    APIC_DELIVERY_STARTUP = 0b110,
+};
+
+enum apic_dest_mode {
+    APIC_DEST_PHYSICAL = 0,
+    APIC_DEST_LOGICAL = 1,
+};
+
+enum apic_delivery_status {
+    APIC_DELIVERY_STATUS_IDLE = 0,
+    APIC_DELIVERY_STATUS_PENDING = 1,
+};
+
+enum apic_level {
+    APIC_LEVEL_DEASSERT = 0,
+    APIC_LEVEL_ASSERT = 1,
+};
+
+enum apic_trigger_mode {
+    APIC_TRIGGER_EDGE = 0,
+    APIC_TRIGGER_LEVEL = 1,
+};
+
+enum apic_regs {
+    APIC_REG_ID = 0x20,
+    APIC_REG_TPR = 0x80,
+    APIC_REG_EOI = 0xB0,
+    APIC_REG_LDR = 0xD0,
+    APIC_REG_DFR = 0xE0,
+    APIC_REG_SIVR = 0xF0,
+    APIC_REG_ESR = 0x280,
+    APIC_REG_ICR = 0x300,
+    APIC_REG_ICR_HI = 0x310,
+    APIC_REG_LVT_TR = 0x320,
+    APIC_REG_ICR_TIMER = 0x380,
+    APIC_REG_CCR = 0x390,
+    APIC_REG_DCR = 0x3E0,
+};
+
+void lapic_init(struct local_apic* lapic);
+void lapic_eoi(struct local_apic* lapic);
+void pic_disable();

kernel/arch/x86_64/include/x86_64/asm.h

@@ -2,6 +2,16 @@
 
 #include <stdint.h>
 
+struct cpuid {
+    uint32_t eax, ebx, ecx, edx;
+};
+
+static inline struct cpuid asm_cpuid(uint32_t leaf, uint32_t subleaf) {
+    struct cpuid out = {0};
+    asm volatile("cpuid" : "=a"(out.eax), "=b"(out.ebx), "=c"(out.ecx), "=d"(out.edx) : "0"(leaf), "2"(subleaf));
+    return out;
+}
+
 static inline uint64_t asm_rdmsr(uint32_t msr) {
     uint32_t eax;
     uint32_t edx;
@@ -9,6 +19,13 @@ static inline uint64_t asm_rdmsr(uint32_t msr) {
     return ((uint64_t)edx << 32) | eax;
 }
 
+static inline uint64_t asm_rdtsc() {
+    uint32_t eax;
+    uint32_t edx;
+    asm volatile("rdtsc" : "=a"(eax), "=d"(edx) : : "memory");
+    return ((uint64_t)edx << 32) | eax;
+}
+
 // Writes a 64-bit value to a given MSR.
 static inline void asm_wrmsr(uint32_t msr, uint64_t val) {
     uint32_t eax = (uint32_t)val;
@@ -26,25 +43,25 @@ static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
 // Saves the FPU state to a 512-byte region of memory using FXSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_fxsave(void* mem) {
-    asm volatile("fxsave %0" : "+m"(*(uint8_t*)mem)::"memory");
+    asm volatile("fxsave [%0]" ::"r"(mem) : "memory");
 }
 
 // Restores the FPU state from a 512-byte region of memory using FXRSTOR.
 // Pointer must be 16-byte aligned.
-static inline void asm_fxrstor(void* mem) {
+static inline void asm_fxrstor(const void* mem) {
-    asm volatile("fxrstor %0" ::"m"(*(uint8_t*)mem) : "memory");
+    asm volatile("fxrstor [%0]" ::"r"(mem) : "memory");
 }
 
 // Saves the FPU state to a region of memory using XSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_xsave(void* mem) {
-    asm volatile("xsave %0" : "+m"(*(uint8_t*)mem)::"memory");
+    asm volatile("xsave [%0]" ::"r"(mem) : "memory");
 }
 
 // Restores the FPU state from a region of memory using XRSTOR.
 // Pointer must be 16-byte aligned.
-static inline void asm_xrstor(void* mem) {
+static inline void asm_xrstor(const void* mem) {
-    asm volatile("xrstor %0" ::"m"(*(uint8_t*)mem) : "memory");
+    asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
 }
 
 static inline uint8_t asm_inb(uint16_t port) {
@@ -76,3 +93,43 @@ static inline void asm_outw(uint16_t port, uint16_t value) {
 static inline void asm_outl(uint16_t port, uint32_t value) {
     asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
 }
+
+static inline uint16_t asm_read_ds() {
+    uint16_t value = 0;
+    asm volatile("mov %0, ds" : "=r"(value));
+    return value;
+}
+
+static inline uint16_t asm_read_es() {
+    uint16_t value = 0;
+    asm volatile("mov %0, es" : "=r"(value));
+    return value;
+}
+
+static inline uint16_t asm_read_fs() {
+    uint16_t value = 0;
+    asm volatile("mov %0, fs" : "=r"(value));
+    return value;
+}
+
+static inline uint16_t asm_read_gs() {
+    uint16_t value = 0;
+    asm volatile("mov %0, gs" : "=r"(value));
+    return value;
+}
+
+static inline void asm_write_ds(uint16_t value) {
+    asm volatile("mov ds, %0" ::"r"(value));
+}
+
+static inline void asm_write_es(uint16_t value) {
+    asm volatile("mov es, %0" ::"r"(value));
+}
+
+static inline void asm_write_fs(uint16_t value) {
+    asm volatile("mov fs, %0" ::"r"(value));
+}
+
+static inline void asm_write_gs(uint16_t value) {
+    asm volatile("mov gs, %0" ::"r"(value));
+}

kernel/arch/x86_64/include/x86_64/defs.h

@@ -134,6 +134,13 @@
 #define RFLAGS_VIP  (1u << 20)
 #define RFLAGS_ID   (1u << 21)
 
+#define XCR0_X87              (1 << 0)
+#define XCR0_SSE              (1 << 1)
+#define XCR0_AVX              (1 << 2)
+#define XCR0_AXV512_OPMASK    (1 << 5)
+#define XCR0_AVX512_ZMM_HI256 (1 << 6)
+#define XCR0_AVX512_HI16_ZMM  (1 << 7)
+
 #define CR0_PE (1 << 0)
 #define CR0_MP (1 << 1)
 #define CR0_EM (1 << 2)
@@ -189,3 +196,6 @@
 
 #define CPL_USER   (0b11)
 #define CPL_KERNEL (0b00)
+
+#define IDT_PF          (0xE)
+#define IDT_IPI_RESCHED (0xFF)

kernel/arch/x86_64/include/x86_64/gdt.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <common/compiler.h>
+#include <stdint.h>
+
+union gdt_desc {
+    uint64_t val;
+    struct {
+        uint16_t limit0;
+        uint16_t base0;
+        uint8_t base1;
+        uint8_t access;
+        uint8_t limit1_flags;
+        uint8_t base2;
+    };
+};
+
+static_assert(sizeof(union gdt_desc) == 0x8);
+
+union gdt_long_desc {
+    uint64_t val[2];
+    struct {
+        uint16_t limit0;
+        uint16_t base0;
+        uint8_t base1;
+        uint8_t access;
+        uint8_t limit1_flags;
+        uint8_t base2;
+        uint32_t base3;
+        uint32_t reserved;
+    };
+};
+
+static_assert(sizeof(union gdt_long_desc) == 0x10);
+
+struct [[__packed]] gdt {
+    union gdt_desc null;
+    union gdt_desc kernel_code32;
+    union gdt_desc kernel_data32;
+    union gdt_desc kernel_code64;
+    union gdt_desc kernel_data64;
+    union gdt_desc user_code32;
+    union gdt_desc user_data;
+    union gdt_desc user_code64;
+    union gdt_long_desc tss;
+};
+
+struct [[__packed]] gdtr {
+    uint16_t limit;
+    struct gdt* base;
+};
+
+struct percpu;
+
+// Initializes a GDT on the local CPU.
+void gdt_load();

kernel/arch/x86_64/include/x86_64/hpet.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void hpet_init();

kernel/arch/x86_64/include/x86_64/idt.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <kernel/init.h>
+
+[[__init]]
+void idt_init();
+
+// Loads the IDT on this CPU.
+void idt_load();
+
+void interrupt_return();

kernel/arch/x86_64/include/x86_64/smp.h

@@ -13,11 +13,11 @@
 
 #ifndef __ASSEMBLER__
 
-#include <gdt.h>
+#include <common/compiler.h>
-#include <kernel/common.h>
+#include <common/utils.h>
-#include <kernel/compiler.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <x86_64/gdt.h>
 
 struct [[__packed]] smp_info {
     struct gdt gdt;

kernel/arch/x86_64/include/x86_64/syscall.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void arch_syscall_stub();

kernel/arch/x86_64/include/x86_64/tsc.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void tsc_init();

kernel/arch/x86_64/include/x86_64/tss.h

@@ -1,9 +1,9 @@
 #pragma once
 
-#include <kernel/compiler.h>
+#include <common/compiler.h>
 #include <stdint.h>
 
-struct [[__packed, __aligned(0x10)]] tss {
+struct [[__packed]] tss {
     uint32_t reserved0;
     uint64_t rsp0;
     uint64_t rsp1;
@@ -22,3 +22,5 @@ struct [[__packed, __aligned(0x10)]] tss {
     uint16_t reserved5;
     uint16_t iopb;
 };
+
+static_assert(sizeof(struct tss) == 0x68);

kernel/arch/x86_64/irq.c

@@ -1,17 +1,19 @@
-#pragma once
-
-#include <defs.h>
 #include <stdint.h>
+#include <x86_64/defs.h>
 
-static inline bool arch_irq_get_state() {
+bool arch_irq_get_state() {
     uint64_t flags;
     asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
     return flags & RFLAGS_IF;
 }
 
-static inline void arch_irq_set_state(bool state) {
+void arch_irq_set_state(bool state) {
     if (state)
         asm volatile("sti");
     else
         asm volatile("cli");
 }
+
+void arch_irq_wait() {
+    asm volatile("hlt");
+}

kernel/arch/x86_64/mem.c

@@ -1,56 +1,31 @@
-#include <kernel/mem.h>
 #include <kernel/types.h>
+#include <stddef.h>
+#include <stdint.h>
 
-enum : pte_t {
+size_t arch_mem_page_size() {
-    ARCH_FLAG_PRESENT = 1 << 0,
+    return 1 << 12;
-    ARCH_FLAG_READ_WRITE = 1 << 1,
-    ARCH_FLAG_USER_MODE = 1 << 2,
-    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
-    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
-    ARCH_FLAG_ACCESSED = 1 << 5,
-    ARCH_FLAG_DIRTY = 1 << 6,
-    ARCH_FLAG_SIZE = 1 << 7,
-    ARCH_FLAG_GLOBAL = 1 << 8,
-    ARCH_FLAG_AVAILABLE = 1 << 9,
-    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
-    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
-
-    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
-};
-
-void arch_mem_pte_clear(pte_t* pte) {
-    *pte = 0;
 }
 
-pte_t arch_mem_pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
+size_t arch_mem_page_bits() {
-    pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
+    return 12;
-
-    if (flags & PTE_USER)
-        result |= ARCH_FLAG_USER_MODE;
-
-    if (flags & PTE_DIR)
-        result |= ARCH_FLAG_READ_WRITE;
-    else {
-        if (flags & PTE_WRITE)
-            result |= ARCH_FLAG_READ_WRITE;
-        if (!(flags & PTE_EXEC))
-            result |= ARCH_FLAG_EXECUTE_DISABLE;
-    }
-    return result;
 }
 
-bool arch_mem_pte_is_present(pte_t* pte) {
+size_t arch_mem_level_bits() {
-    return *pte & ARCH_FLAG_PRESENT;
+    return 9;
 }
 
-bool arch_mem_pte_is_dir(pte_t* pte) {
+size_t arch_mem_num_levels() {
-    return *pte & ARCH_FLAG_SIZE;
+    return 4;
 }
 
-phys_t arch_mem_pte_address(pte_t* pte) {
+uintptr_t arch_mem_hhdm_addr() {
-    return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
+    return 0xFFFF'8000'0000'0000;
 }
 
-void arch_mem_pt_set(struct page_table* pt) {
+uintptr_t arch_mem_pfndb_addr() {
-    asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
+    return 0xFFFF'A000'0000'0000;
+}
+
+uintptr_t arch_mem_mapping_addr() {
+    return 0xFFFF'C000'0000'0000;
 }

kernel/arch/x86_64/meson.build

@@ -1,17 +1,24 @@
 kernel_c_args += [
-    '-masm=intel',
     '-mgeneral-regs-only',
     '-mno-red-zone',
     '-mcmodel=kernel',
 ]
 
 kernel_sources += files(
+    'apic.c',
     'gdt.c',
+    'hpet.c',
     'idt.c',
+    'irq.c',
     'mem.c',
+    'percpu.c',
+    'pmap.c',
+    'sched.c',
     'serial.c',
     'smp.c',
     'sys.c',
+    'syscall.c',
     'trampoline.S',
+    'tsc.c',
     'usercopy.c',
 )

kernel/arch/x86_64/percpu.c

@@ -0,0 +1,126 @@
+#include <kernel/percpu.h>
+#include <kernel/print.h>
+#include <stddef.h>
+#include <x86_64/apic.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+#include <x86_64/hpet.h>
+#include <x86_64/idt.h>
+#include <x86_64/syscall.h>
+#include <x86_64/tsc.h>
+
+struct percpu* arch_percpu_get() {
+    struct percpu* result;
+    asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
+    return result;
+}
+
+void arch_percpu_bsp_init() {
+    asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
+    asm_wrmsr(MSR_FS_BASE, 0);
+    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
+
+    pic_disable();
+    gdt_load();
+    idt_init();
+    idt_load();
+}
+
+union ia32_star {
+    uint64_t value;
+    struct {
+        uint32_t rsvd;
+        uint16_t kernel_cs_ss;
+        uint16_t user_cs_ss;
+    };
+};
+
+static_assert(sizeof(union ia32_star) == 8);
+
+static void setup_cpu(struct percpu* cpu) {
+    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
+    asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
+    asm_wrmsr(MSR_FS_BASE, 0);
+
+    gdt_load();
+    idt_load();
+
+    // Syscall extension.
+    asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
+    union ia32_star star = {0};
+    star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
+    star.user_cs_ss = offsetof(struct gdt, user_code32);
+    asm_wrmsr(MSR_STAR, star.value);
+    asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
+    asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
+
+    // TODO: FRED
+
+    uint64_t cr0, cr4;
+    asm volatile("mov %0, cr0" : "=r"(cr0));
+    asm volatile("mov %0, cr4" : "=r"(cr4));
+
+    struct cpuid cpuid1 = asm_cpuid(1, 0);
+    struct cpuid cpuid7 = asm_cpuid(7, 0);
+    struct cpuid cpuid13 = asm_cpuid(13, 0);
+
+    // Enable SSE.
+    cr0 &= ~CR0_EM;
+    cr0 |= CR0_MP;
+    cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
+
+    if (cpuid1.ecx & CPUID_1C_XSAVE) {
+        cr4 |= CR4_OSXSAVE;
+        asm volatile("mov cr4, %0" ::"r"(cr4));
+
+        uint64_t xcr0 = XCR0_X87 | XCR0_SSE;
+        // AVX
+        if (cpuid1.ecx & CPUID_1C_AVX)
+            xcr0 |= XCR0_AVX;
+
+        // AVX-512
+        if (cpuid7.ebx & CPUID_7B_AVX512F)
+            xcr0 |= XCR0_AXV512_OPMASK | XCR0_AVX512_ZMM_HI256 | XCR0_AVX512_HI16_ZMM;
+
+        asm_wrxcr(0, xcr0);
+
+        cpu->arch.fpu_size = cpuid13.ecx;
+        cpu->arch.fpu_save = asm_xsave;
+        cpu->arch.fpu_restore = asm_xrstor;
+    } else {
+        cpu->arch.fpu_size = 512;
+        cpu->arch.fpu_save = asm_fxsave;
+        cpu->arch.fpu_restore = asm_fxrstor;
+    }
+
+    if (cpuid7.ecx & CPUID_7C_UMIP)
+        cr4 |= CR4_UMIP;
+
+    if (cpuid7.ebx & CPUID_7B_SMEP)
+        cr4 |= CR4_SMEP;
+
+    if (cpuid7.ebx & CPUID_7B_SMAP) {
+        cr4 |= CR4_SMAP;
+        cpu->arch.can_smap = true;
+    }
+
+    if (cpuid7.ebx & CPUID_7B_FSGSBASE)
+        cr4 |= CR4_FSGSBASE;
+
+    asm volatile("mov %0, cr0" : "=r"(cr0));
+    asm volatile("mov %0, cr4" : "=r"(cr4));
+
+    lapic_init(&cpu->arch.lapic);
+    cpu->online = true;
+}
+
+void arch_percpu_init() {
+    hpet_init();
+    tsc_init();
+
+    // Init the BSP.
+    setup_cpu(&percpu_bsp);
+
+    // Init any APs.
+}

kernel/arch/x86_64/pmap.c

@@ -0,0 +1,56 @@
+#include <kernel/pmap.h>
+#include <kernel/types.h>
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};
+
+void pte_clear(pte_t* pte) {
+    *pte = 0;
+}
+
+pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
+    pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
+
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+
+    if (flags & PTE_DIR)
+        result |= ARCH_FLAG_READ_WRITE;
+    else {
+        if (flags & PTE_WRITE)
+            result |= ARCH_FLAG_READ_WRITE;
+        if (!(flags & PTE_EXEC))
+            result |= ARCH_FLAG_EXECUTE_DISABLE;
+    }
+    return result;
+}
+
+bool pte_is_present(pte_t* pte) {
+    return *pte & ARCH_FLAG_PRESENT;
+}
+
+bool pte_is_dir(pte_t* pte) {
+    return *pte & ARCH_FLAG_SIZE;
+}
+
+phys_t pte_address(pte_t* pte) {
+    return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
+}
+
+void pmap_set(struct pmap* pt) {
+    asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
+}

kernel/arch/x86_64/sched.c

@@ -0,0 +1,206 @@
+#include <zinnia/status.h>
+#include <common/compiler.h>
+#include <kernel/alloc.h>
+#include <kernel/arch.h>
+#include <kernel/percpu.h>
+#include <kernel/sched.h>
+#include <kernel/virt.h>
+#include <bits/sched.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+#include <x86_64/idt.h>
+
+// The task frame consists of registers that the C ABI marks as callee-saved.
+// If we don't save them, these registers are lost during a context switch.
+struct task_frame {
+    uint64_t rbx, rbp, r12, r13, r14, r15, rip;
+};
+
+[[__naked]]
+static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) {
+    asm volatile(
+        "sub rsp, 0x30\n" // Make room for all regs (except RIP).
+        "mov [rsp + %c0], rbx\n"
+        "mov [rsp + %c1], rbp\n"
+        "mov [rsp + %c2], r12\n"
+        "mov [rsp + %c3], r13\n"
+        "mov [rsp + %c4], r14\n"
+        "mov [rsp + %c5], r15\n"
+        "mov [rdi], rsp\n" // rdi = old_rsp
+        "mov rsp, rsi\n"   // rsi = new_rsp
+        "mov rbx, [rsp + %c0]\n"
+        "mov rbp, [rsp + %c1]\n"
+        "mov r12, [rsp + %c2]\n"
+        "mov r13, [rsp + %c3]\n"
+        "mov r14, [rsp + %c4]\n"
+        "mov r15, [rsp + %c5]\n"
+        "add rsp, 0x30\n"
+        "call %c6\n"
+        "ret" // This will conveniently move us to the RIP we put at this stack entry.
+        :
+        : "i"(offsetof(struct task_frame, rbx)),
+          "i"(offsetof(struct task_frame, rbp)),
+          "i"(offsetof(struct task_frame, r12)),
+          "i"(offsetof(struct task_frame, r13)),
+          "i"(offsetof(struct task_frame, r14)),
+          "i"(offsetof(struct task_frame, r15)),
+          "i"(irq_unlock)
+        : "memory"
+    );
+}
+
+void arch_sched_switch(struct task* from, struct task* to) {
+    struct arch_percpu* cpu = &percpu_get()->arch;
+    struct arch_task_context* from_ctx = &from->task_ctx;
+    struct arch_task_context* to_ctx = &to->task_ctx;
+
+    cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
+
+    if (from->user_stack) {
+        cpu->fpu_save(from_ctx->fpu_region);
+        from_ctx->ds = asm_read_ds();
+        from_ctx->es = asm_read_es();
+        from_ctx->fs = asm_read_fs();
+        from_ctx->gs = asm_read_gs();
+        from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE);
+        from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
+    }
+
+    if (to->user_stack) {
+        cpu->fpu_restore(to_ctx->fpu_region);
+        asm_write_ds(to_ctx->ds);
+        asm_write_es(to_ctx->es);
+        asm_write_fs(to_ctx->fs);
+
+        // If we have to change the GS segment we need to reload the MSR, otherwise we lose its value.
+        if (to_ctx->gs != asm_read_gs()) {
+            uint64_t percpu = asm_rdmsr(MSR_GS_BASE);
+            asm_write_gs(to_ctx->gs);
+            asm_wrmsr(MSR_GS_BASE, percpu);
+        }
+
+        asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base);
+        asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base);
+    }
+
+    perform_switch(&from_ctx->rsp, to_ctx->rsp);
+}
+
+[[__naked]]
+void task_entry_thunk() {
+    asm volatile(
+        "mov rdi, rbx\n"
+        "mov rsi, r12\n"
+        "jmp %c0"
+        :
+        : "i"(task_entry)
+        : "memory"
+    );
+}
+
+zn_status_t arch_task_init(
+    struct arch_task_context* context,
+    void* entry,
+    uintptr_t arg0,
+    uintptr_t stack_start,
+    bool is_user
+) {
+    struct arch_percpu* cpu = &percpu_get()->arch;
+
+    // Prepare a dummy stack with an entry point function to return to.
+    struct task_frame* frame = (struct task_frame*)stack_start - 1;
+    frame->rbx = (uint64_t)entry;
+    frame->r12 = arg0;
+    frame->rip = (uint64_t)task_entry_thunk;
+    context->rsp = (uint64_t)frame;
+
+    if (is_user) {
+        // Allocate FPU space.
+        const size_t fpu_size = cpu->fpu_size;
+        uintptr_t fpu_base = vm_alloc(fpu_size);
+        if (!fpu_base)
+            return ZN_ERR_NO_MEMORY;
+        for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
+            phys_t page;
+            zn_status_t s = mem_phys_alloc(1, 0, &page);
+            if (s) {
+                vm_free(fpu_base, fpu_size);
+                return s;
+            }
+
+            pmap_map(&kernel_vas.pmap, fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
+        }
+        context->fpu_region = (void*)fpu_base;
+
+        context->ds = asm_read_ds();
+        context->es = asm_read_es();
+        context->fs = asm_read_fs();
+        context->gs = asm_read_gs();
+        context->fs_base = asm_rdmsr(MSR_FS_BASE);
+        context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
+    }
+
+    return ZN_OK;
+}
+
+[[__naked]]
+void arch_sched_preempt_disable() {
+    asm volatile(
+        "inc qword ptr gs:%c0\n"
+        "ret"
+        :
+        : "i"(offsetof(struct percpu, sched.preempt_level))
+    );
+}
+
+[[__naked]]
+bool arch_sched_preempt_enable() {
+    asm volatile(
+        "dec qword ptr gs:%c0\n"
+        "setz al\n"
+        "movzx eax, al\n"
+        "ret\n"
+        :
+        : "i"(offsetof(struct percpu, sched.preempt_level))
+        : "cc"
+    );
+}
+
+[[__naked]]
+void arch_sched_jump_to_context(struct arch_context* context) {
+    asm volatile(
+        "mov rsp, rdi\n"
+        "jmp %c0\n" ::"i"(interrupt_return)
+    );
+}
+
+void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
+    struct arch_context context = {
+        .rip = ip,
+        .rsp = sp,
+        .rdi = arg,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+
+    uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
+    const uint16_t zero = 0;
+    asm volatile(
+        "mov ds, %0\n"
+        "mov es, %0\n"
+        "mov fs, %0\n"
+        "mov gs, %0\n"
+        :
+        : "r"(zero)
+    );
+
+    asm_wrmsr(MSR_FS_BASE, 0);
+    asm_wrmsr(MSR_GS_BASE, percpu);
+    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
+
+    arch_sched_jump_to_context(&context);
+}

kernel/arch/x86_64/serial.c

@@ -1,6 +1,6 @@
-#include <asm.h>
 #include <kernel/console.h>
 #include <stdint.h>
+#include <x86_64/asm.h>
 
 constexpr uint16_t COM1_BASE = 0x3F8;
 

kernel/arch/x86_64/smp.c

@@ -1,6 +1,6 @@
-#include <kernel/compiler.h>
+#include <common/compiler.h>
 #include <kernel/types.h>
-#include <smp.h>
+#include <x86_64/smp.h>
 
 extern uint8_t smp_trampoline_start[];
 extern uint8_t smp_trampoline_end[];

kernel/arch/x86_64/sys.c

@@ -1,14 +1,16 @@
 #include <zinnia/archctl.h>
 #include <zinnia/status.h>
-#include <kernel/compiler.h>
+#include <common/compiler.h>
+#include <kernel/arch.h>
 #include <kernel/init.h>
 #include <kernel/percpu.h>
-#include <gdt.h>
+#include <x86_64/apic.h>
-#include "asm.h"
+#include <x86_64/asm.h>
-#include "defs.h"
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
 
-[[__init, __naked]]
+[[__init, __naked, __used]]
-void _start() {
+void arch_start() {
     asm volatile(
         "lea rsp, [rip + %0]\n"
         "jmp %1"
@@ -17,26 +19,18 @@ void _start() {
     );
 }
 
-void percpu_bsp_early_init() {
+void arch_panic() {}
-    asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
-    asm_wrmsr(MSR_FS_BASE, 0);
-    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
-
-    gdt_init();
-}
-
-[[noreturn]]
-void arch_panic() {
-    asm volatile("cli; hlt");
-    __unreachable();
-}
 
 zn_status_t arch_archctl(zn_archctl_t op, uintptr_t arg) {
     switch (op) {
     case ZN_ARCHCTL_SET_FSBASE:
         asm_wrmsr(MSR_FS_BASE, arg);
         return 0;
+    case ZN_ARCHCTL_SET_GSBASE:
+        // Slightly misleading, but KERNEL_GS_BASE is the currently inactive register.
+        asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
+        return 0;
     default:
-        return ZN_ERR_BAD_ARG;
+        return ZN_ERR_INVALID;
     }
 }

kernel/arch/x86_64/syscall.c

@@ -0,0 +1,68 @@
+#include <common/compiler.h>
+#include <kernel/percpu.h>
+#include <kernel/syscalls.h>
+#include <bits/sched.h>
+#include <stddef.h>
+#include <x86_64/defs.h>
+
+[[__naked]]
+void arch_syscall_stub() {
+    asm volatile(
+        "swapgs\n"
+        "mov gs:%c0, rsp\n"
+        "mov rsp, gs:%c1\n"
+        "cld\n"
+        // We're pretending to be an interrupt, so fill the bottom fields of `Context`.
+        "push %c3\n" // SS and CS are not changed during SYSCALL. Use `gdt::user_data | CPL_USER`.
+        "push gs:%c0\n"
+        "push r11\n"  // RFLAGS is moved into r11 by the CPU.
+        "push %c2\n"  // Same as SS. Use `gdt::user_code64 | CPL_USER`
+        "push rcx\n"  // RIP is moved into rcx by the CPU.
+        "push 0x00\n" // arch_context::error field
+        "push 0x00\n" // arch_context::isr field
+        "push rax\n"
+        "push rbx\n"
+        "push rcx\n"
+        "push rdx\n"
+        "push rbp\n"
+        "push rdi\n"
+        "push rsi\n"
+        "push r8\n"
+        "push r9\n"
+        "push r10\n"
+        "push r11\n"
+        "push r12\n"
+        "push r13\n"
+        "push r14\n"
+        "push r15\n"
+        "xor rbp, rbp\n"
+        "mov rdi, rsp\n" // Put the trap frame struct as first argument.
+        "call %c4\n"     // Call syscall handler
+        "cli\n"
+        "pop r15\n"
+        "pop r14\n"
+        "pop r13\n"
+        "pop r12\n"
+        "pop r11\n"
+        "pop r10\n"
+        "pop r9\n"
+        "pop r8\n"
+        "pop rsi\n"
+        "pop rdi\n"
+        "pop rbp\n"
+        "pop rdx\n"
+        "pop rcx\n"
+        "pop rbx\n"
+        "pop rax\n"
+        "add rsp, 0x10\n"   // Skip .error and .isr fields (2 * sizeof(uint64_t))
+        "mov rsp, gs:%c0\n" // Load user stack from `percpu::user_stack`.
+        "swapgs\n"
+        "sysretq\n" // Return to user mode.
+        :
+        : "i"(offsetof(struct percpu, user_stack)),
+          "i"(offsetof(struct percpu, kernel_stack)),
+          "i"(offsetof(struct gdt, user_code64) | CPL_USER),
+          "i"(offsetof(struct gdt, user_data) | CPL_USER),
+          "i"(syscall_dispatch)
+    );
+}

kernel/arch/x86_64/trampoline.S

@@ -1,5 +1,5 @@
-#include <smp.h>
+#include <x86_64/smp.h>
-#include <defs.h>
+#include <x86_64/defs.h>
 
 .intel_syntax noprefix
 

kernel/arch/x86_64/tsc.c

@@ -0,0 +1,54 @@
+#include <kernel/clock.h>
+#include <kernel/cmdline.h>
+#include <kernel/print.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+
+static bool use_tsc = true;
+
+void tsc_init() {
+    if (!use_tsc)
+        return;
+
+    if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
+        kprintf("No invariant TSC detected!\n");
+        return;
+    }
+
+    // Check for the TSC info leaf.
+    bool has_tsc_leaf = false;
+    struct cpuid tsc_leaf = {0};
+    if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) {
+        tsc_leaf = asm_cpuid(0x15, 0);
+    }
+
+    uint64_t freq = 0;
+    if (clock_available()) {
+        // Calibrate using existing clock.
+        uint64_t t1 = asm_rdtsc();
+        clock_spin_ns(10'000'000);
+        uint64_t t2 = asm_rdtsc();
+        freq = (t2 - t1) * 100;
+    } else if (has_tsc_leaf) {
+        // If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency.
+        // On a normal system, this should usually never be called and is a last resort
+        // since at this point we have at least the HPET timer.
+        if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
+            kprintf("Calibrating TSC using CPUID 0x15");
+            freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
+        } else {
+            kprintf("Unable to calibrate TSC using CPUID frequency information");
+            return;
+        }
+    } else {
+        kprintf("Unable to calibrate the TSC without a clock or static frequency info!\n");
+        return;
+    }
+
+    kprintf("Timer frequency is %lu MHz\n", freq / 1'000'000);
+}
+
+static void tsc_option(bool opt) {
+    use_tsc = opt;
+}
+CMDLINE_OPTION("tsc", tsc_option);

kernel/arch/x86_64/usercopy.c

@@ -1,4 +1,4 @@
-#include <kernel/compiler.h>
+#include <common/compiler.h>
 #include <kernel/usercopy.h>
 
 extern void x86_64_read_start();
@@ -12,12 +12,13 @@ static const struct usercopy_region read_region = {
 };
 
 [[__naked]]
-bool arch_usercopy_read(uint8_t* dst, const __user uint8_t* src, size_t len) {
+bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) {
     asm volatile(
+        "stac\n"
         // Setup regs for `rep movsb`.
         "xchg rcx, rdx\n"
         "mov rax, [rip + %0]\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
 
         ".global x86_64_read_start\n"
         "x86_64_read_start:\n"
@@ -28,12 +29,14 @@ bool arch_usercopy_read(uint8_t* dst, const __user uint8_t* src, size_t len) {
         "xor rax, rax\n"
         "mov [rdx], rax\n"
         "mov rax, 1\n"
+        "clac\n"
         "ret\n"
 
         ".global x86_64_read_fault\n"
         "x86_64_read_fault:\n"
         "xor rax, rax\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
+        "clac\n"
         "ret\n"
         :
         : "i"(&read_region)
@@ -52,12 +55,13 @@ static const struct usercopy_region write_region = {
 };
 
 [[__naked]]
-bool arch_usercopy_write(__user uint8_t* dst, const uint8_t* src, size_t len) {
+bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) {
     asm volatile(
+        "stac\n"
         // Setup regs for `rep movsb`.
         "xchg rcx, rdx\n"
         "mov rax, [rip + %0]\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
 
         ".global x86_64_write_start\n"
         "x86_64_write_start:\n"
@@ -68,61 +72,17 @@ bool arch_usercopy_write(__user uint8_t* dst, const uint8_t* src, size_t len) {
         "xor rax, rax\n"
         "mov [rdx], rax\n"
         "mov rax, 1\n"
+        "clac\n"
         "ret\n"
 
         ".global x86_64_write_fault\n"
         "x86_64_write_fault:\n"
         "xor rax, rax\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
+        "clac\n"
         "ret\n"
         :
-        : "i"(&read_region)
+        : "i"(&write_region)
-        : "memory"
-    );
-}
-
-extern void x86_64_strlen_start();
-extern void x86_64_strlen_end();
-extern void x86_64_strlen_fault();
-
-static const struct usercopy_region strlen_region = {
-    .start_ip = x86_64_strlen_start,
-    .end_ip = x86_64_strlen_end,
-    .fault_ip = x86_64_strlen_fault,
-};
-
-[[__naked]]
-bool arch_usercopy_strlen(const __user uint8_t* str, size_t max, size_t* len) {
-    asm volatile(
-        "mov rax, [rip + %0]\n"
-        "mov [rcx], rax\n"
-
-        ".global x86_64_strlen_start\n"
-        "x86_64_strlen_start:\n"
-        "xor r8, r8\n"
-        ".Lloop:\n"
-        "cmp byte ptr [rdi + r8], 0\n"
-        "je .Lleave\n"
-        "inc r8\n"
-        "cmp rsi, r8\n"
-        "jne .Lloop\n"
-        ".Lleave:\n"
-        "mov [rdx], r8\n"
-
-        ".global x86_64_strlen_end\n"
-        "x86_64_strlen_end:\n"
-        "xor rax, rax\n"
-        "mov [rcx], rax\n"
-        "mov rax, 1\n"
-        "ret\n"
-
-        ".global x86_64_strlen_fault\n"
-        "x86_64_strlen_fault:\n"
-        "xor rax, rax\n"
-        "mov [rcx], rax\n"
-        "ret\n"
-        :
-        : "i"(&read_region)
         : "memory"
     );
 }

kernel/boot/cmdline.c

@@ -1,22 +1,23 @@
 #include <kernel/cmdline.h>
 #include <kernel/init.h>
 #include <stddef.h>
+#include <stdlib.h>
 #include <string.h>
 
 // Defined in linker script.
-extern struct cmdline_option __ld_cmdline_start[];
+extern const struct cmdline_option* __ld_cmdline_start[];
-extern struct cmdline_option __ld_cmdline_end[];
+extern const struct cmdline_option* __ld_cmdline_end[];
 
-static char cmdline_buf[CMDLINE_MAX];
+static char cmdline_buf[0x1000];
 
 [[__init]]
 void cmdline_parse(const char* cmdline) {
-    struct cmdline_option* opt = __ld_cmdline_start;
+    const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
-    const size_t len = strnlen(cmdline, CMDLINE_MAX - 1);
     size_t idx = 0;
     memcpy(cmdline_buf, cmdline, len);
 
     while (1) {
+        const struct cmdline_option** opt = __ld_cmdline_start;
         char* name = nullptr;
         char* value = nullptr;
 
@@ -59,8 +60,39 @@ void cmdline_parse(const char* cmdline) {
 
         // Find the corresponding option.
         while (opt < __ld_cmdline_end) {
-            if (!strcmp(opt->name, name))
+            const struct cmdline_option* current = *opt;
-                opt->func(value);
+            if (!strcmp(current->name, name)) {
+                switch (current->option_type) {
+                case CMDLINE_STRING:
+                    current->func_str(value);
+                    break;
+                case CMDLINE_BOOL:
+                    if (!strcmp(value, "on") || !strcmp(value, "yes") || !strcmp(value, "true") || !strcmp(value, "1"))
+                        current->func_bool(true);
+                    else
+                        current->func_bool(false);
+                    break;
+                case CMDLINE_INTPTR: {
+                    int base = 10;
+                    if (!strncmp(value, "0x", 2)) {
+                        base = 0x10;
+                        value += 2;
+                    }
+                    current->func_intptr(atol(value, base));
+                    break;
+                }
+                case CMDLINE_UINTPTR: {
+                    int base = 10;
+                    if (!strncmp(value, "0x", 2)) {
+                        base = 0x10;
+                        value += 2;
+                    }
+                    current->func_uintptr(atolu(value, base));
+                    break;
+                }
+                }
+                break;
+            }
             opt++;
         }
 

kernel/boot/limine/entry.c

@@ -1,9 +1,9 @@
-#include <kernel/assert.h>
+#include <common/utils.h>
+#include <kernel/alloc.h>
 #include <kernel/cmdline.h>
-#include <kernel/common.h>
 #include <kernel/init.h>
-#include <kernel/mem.h>
 #include <kernel/print.h>
+#include <string.h>
 #include "limine.h"
 
 [[__initdata_sorted("limine.0")]]
@@ -61,6 +61,9 @@ void kernel_entry() {
         case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
             mem[i].usage = PHYS_RECLAIMABLE;
             break;
+        case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
+            mem[i].usage = PHYS_STATIC;
+            break;
         default:
             mem[i].usage = PHYS_RESERVED;
             break;
@@ -68,18 +71,23 @@ void kernel_entry() {
     }
 
     if (module_res) {
+        info.files = files;
         info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
         for (size_t i = 0; i < info.num_files; i++) {
-            files[i].data = module_res->modules[i]->address;
+            files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset;
             files[i].length = module_res->modules[i]->size;
-            files[i].path = module_res->modules[i]->path;
+            strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path));
         }
     }
 
     info.mem_map = mem;
     info.phys_base = (phys_t)exec_res->physical_base;
-    info.virt_base = (virt_t)exec_res->virtual_base;
+    info.virt_base = (uintptr_t)exec_res->virtual_base;
-    info.hhdm_base = (virt_t)hhdm_request.response->offset;
+    info.hhdm_base = (uintptr_t)hhdm_request.response->offset;
+
+    info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base;
 
     kernel_main(&info);
 }
+
+phys_t rsdp_addr;

kernel/boot/main.c

@@ -1,42 +1,145 @@
-#include <kernel/assert.h>
+#include <common/compiler.h>
+#include <kernel/alloc.h>
+#include <kernel/arch.h>
+#include <kernel/channel.h>
+#include <kernel/clock.h>
 #include <kernel/cmdline.h>
-#include <kernel/compiler.h>
 #include <kernel/console.h>
+#include <kernel/elf.h>
+#include <kernel/futex.h>
 #include <kernel/init.h>
 #include <kernel/irq.h>
-#include <kernel/mem.h>
+#include <kernel/namespace.h>
 #include <kernel/percpu.h>
 #include <kernel/print.h>
+#include <kernel/sched.h>
+#include <kernel/vas.h>
+#include <kernel/vmo.h>
+#include <assert.h>
 #include <config.h>
+#include <stdint.h>
+
+static zn_status_t load_server(
+    struct boot_file* file,
+    const char** argv,
+    const char** envp,
+    struct namespace_desc* init_handle,
+    struct task** out
+) {
+    struct vas* space;
+    zn_status_t status = vas_new(&space);
+    if (status != ZN_OK)
+        return status;
+
+    struct paged_vmo* file_vmo;
+    status = vmo_new_phys(&file_vmo);
+    if (status != ZN_OK)
+        return status;
+    vmo_write(&file_vmo->object, 0, HHDM_PTR(file->data), file->length, nullptr);
+
+    struct namespace* ns;
+    status = namespace_new(&ns);
+    if (status != ZN_OK)
+        return status;
+
+    struct exec_info info = {
+        .file_obj = &file_vmo->object,
+        .ns = ns,
+        .space = space,
+        .argv = argv,
+        .envp = envp,
+        .init_handle = init_handle,
+    };
+
+    kprintf("Loading executable \"%s\"\n", file->path);
+    return elf_load(&info, out);
+}
 
 static const char zinnia_banner[] =
-    "zinnia " ZINNIA_VERSION " (" ZINNIA_ARCH ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
+    "Zinnia " ZINNIA_VERSION " (" ZINNIA_ARCH ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
 
-[[__init]]
+static void kernel_main_task(uintptr_t arg0) {
-void kernel_early_init() {
+    struct boot_info* info = (struct boot_info*)arg0;
-    percpu_bsp_early_init();
+    kprintf("%s\n", zinnia_banner); // Say hello!
-    percpu_get()->online = true;
+
-    irq_lock();
+    assert(info->num_files >= 1, "No init executable provided\n");
+
+    struct channel* init_ep;
+    struct channel* other_ep;
+    assert(channel_create(&init_ep, &other_ep) == ZN_OK, "Failed to create init channel\n");
+
+    struct vas* init_space;
+    assert(vas_new(&init_space) == ZN_OK, "");
+
+    struct boot_file* file_data = &info->files[0];
+    struct paged_vmo* init_file;
+    assert(vmo_new_phys(&init_file) == ZN_OK, "");
+    vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
+
+    struct namespace* ns;
+    assert(namespace_new(&ns) == ZN_OK, "");
+
+    struct namespace_desc init_ch_desc = {
+        .type = NAMESPACE_DESC_CHANNEL,
+        .rights = ZN_RIGHTS_ALL,
+        .value.channel = init_ep,
+    };
+
+    // Load init.
+    const char* init_argv[] = {"init", nullptr};
+    const char* init_envp[] = {nullptr};
+
+    struct task* init_task;
+    zn_status_t status = load_server(&info->files[0], init_argv, init_envp, &init_ch_desc, &init_task);
+    assert(status == ZN_OK, "Failed to load init process (%i)\n", status);
+    sched_add_task(&percpu_get()->sched, init_task);
+
+    // Load other daemons.
+    for (size_t i = 1; i < info->num_files; i++) {
+        struct namespace_desc ch_desc = {
+            .type = NAMESPACE_DESC_CHANNEL,
+            .rights = ZN_RIGHTS_ALL,
+            .value.channel = other_ep,
+        };
+
+        const char* argv[] = {info->files[i].path, nullptr};
+        const char* envp[] = {nullptr};
+
+        struct task* task;
+        status = load_server(&info->files[i], argv, envp, &ch_desc, &task);
+        assert(status == ZN_OK, "Failed to load server \"%s\": (%i)\n", info->files[i].path, status);
+        sched_add_task(&percpu_get()->sched, task);
+    }
+
+    while (true) {
+        arch_irq_wait();
+    }
 }
 
 [[noreturn]]
 void kernel_main(struct boot_info* info) {
     cmdline_parse(info->cmdline);
+
     console_init();
-    kprintf("%s\n", zinnia_banner); // Say hello!
-    kprintf("Command line: \"%s\"\n", info->cmdline);
-
     mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
+    rsdp_addr = info->rsdp; // TODO
+    percpu_init();
+    sched_init(&percpu_get()->sched);
+    futex_init();
+
+    struct task* main_task;
+    task_create("main", &kernel_vas, nullptr, kernel_main_task, (uintptr_t)info, &main_task);
+    sched_add_task(&percpu_get()->sched, main_task);
 
-    sched_init();
     irq_unlock();
+    sched_yield(&percpu_get()->sched);
 
-    while (1) {}
+    unreachable();
 }
 
-[[noreturn]]
+[[__init]]
-void kernel_main_task() {
+void kernel_early_init() {
-    while (1) {}
+    percpu_bsp_init();
-
+    percpu_get()->online = true;
-    ASSERT(false, "Nothing to do");
+    irq_lock();
 }

kernel/core/channel.c

@@ -0,0 +1,112 @@
+#include <zinnia/status.h>
+#include <kernel/alloc.h>
+#include <kernel/channel.h>
+#include <kernel/namespace.h>
+#include <kernel/percpu.h>
+#include <kernel/sched.h>
+#include <kernel/tailq.h>
+#include <kernel/wait.h>
+#include <stdatomic.h>
+
+zn_status_t channel_create(struct channel** ep0, struct channel** ep1) {
+    struct channel* a = mem_alloc(sizeof(struct channel), 0);
+    if (!a)
+        return ZN_ERR_NO_MEMORY;
+
+    struct channel* b = mem_alloc(sizeof(struct channel), 0);
+    if (!b) {
+        mem_free(a);
+        return ZN_ERR_NO_MEMORY;
+    }
+
+    a->peer = b;
+    a->lock = (struct spinlock){0};
+    tailq_init(&a->messages);
+    wait_queue_init(&a->waiters);
+
+    b->peer = a;
+    b->lock = (struct spinlock){0};
+    tailq_init(&b->messages);
+    wait_queue_init(&b->waiters);
+
+    *ep0 = a;
+    *ep1 = b;
+    return ZN_OK;
+}
+
+void channel_msg_free(struct channel_msg* msg) {
+    mem_free(msg->data);
+    mem_free(msg->descs);
+    mem_free(msg);
+}
+
+zn_status_t channel_write(
+    struct channel* ch,
+    void* data,
+    size_t num_bytes,
+    struct namespace_desc* descs,
+    size_t num_handles
+) {
+    // Check that the peer is still alive.
+    struct channel* peer = atomic_load_explicit(&ch->peer, memory_order_acquire);
+    if (!peer)
+        return ZN_ERR_BAD_HANDLE;
+
+    // Allocate the message.
+    struct channel_msg* msg = mem_alloc(sizeof(struct channel_msg), 0);
+    if (!msg)
+        return ZN_ERR_NO_MEMORY;
+
+    msg->data = data;
+    msg->num_bytes = num_bytes;
+    msg->descs = descs;
+    msg->num_handles = num_handles;
+
+    // Enqueue on the peer's message queue and wake a waiter.
+    spin_lock(&peer->lock);
+    tailq_insert_tail(&peer->messages, msg, next);
+    spin_unlock(&peer->lock);
+
+    struct sched_percpu* sched = &percpu_get()->sched;
+    sched_wake(sched, &peer->waiters);
+
+    return ZN_OK;
+}
+
+zn_status_t channel_read(struct channel* ch, struct channel_msg** msg_out) {
+    spin_lock(&ch->lock);
+    struct channel_msg* msg = tailq_first(&ch->messages);
+
+    if (msg != nullptr) {
+        tailq_remove(&ch->messages, msg, next);
+        spin_unlock(&ch->lock);
+        *msg_out = msg;
+        return ZN_OK;
+    }
+
+    // No message and peer is gone.
+    if (!ch->peer) {
+        spin_unlock(&ch->lock);
+        return ZN_ERR_BAD_HANDLE;
+    }
+
+    spin_unlock(&ch->lock);
+    return ZN_ERR_INVALID;
+}
+
+zn_status_t channel_poll(struct channel* ch) {
+    spin_lock(&ch->lock);
+
+    if (!tailq_empty(&ch->messages)) {
+        spin_unlock(&ch->lock);
+        return ZN_OK;
+    }
+
+    if (!ch->peer) {
+        spin_unlock(&ch->lock);
+        return ZN_ERR_BAD_HANDLE;
+    }
+
+    spin_unlock(&ch->lock);
+    return ZN_ERR_INVALID;
+}

kernel/core/clock.c

@@ -1 +1,54 @@
+#include <common/compiler.h>
+#include <kernel/clock.h>
+#include <kernel/print.h>
+#include <kernel/spin.h>
+#include <stdatomic.h>
+#include <stdint.h>
 
+static struct clock* active_clock = nullptr;
+static uint64_t counter_base = 0;
+static struct spinlock lock = {};
+
+bool clock_available() {
+    return active_clock != nullptr;
+}
+
+uint64_t clock_get_elapsed_ns() {
+    struct clock* active = atomic_load(&active_clock);
+    if (__unlikely(!active))
+        return 0;
+
+    return atomic_load(&counter_base) + active->get_elapsed_ns(active);
+}
+
+bool clock_switch(struct clock* clock) {
+    spin_lock(&lock);
+
+    if (active_clock) {
+        if (clock->priority <= active_clock->priority) {
+            spin_unlock(&lock);
+            return false;
+        }
+    }
+
+    kprintf("Switching to clock source \"%s\"\n", clock->name);
+
+    uint64_t elapsed = clock_get_elapsed_ns();
+    counter_base += elapsed;
+
+    active_clock = clock;
+    spin_unlock(&lock);
+    return true;
+}
+
+void clock_spin_ns(uint64_t ns) {
+    if (active_clock == nullptr) {
+        kprintf("Unable to sleep for %lu ns, no clock available!\n", ns);
+        return;
+    }
+
+    uint64_t target = clock_get_elapsed_ns() + ns;
+    while (clock_get_elapsed_ns() < target) {
+        // TODO: Maybe pause?
+    }
+}

kernel/core/elf.c

@@ -1 +1,202 @@
+#include <zinnia/mem.h>
+#include <zinnia/status.h>
+#include <common/elf.h>
+#include <common/utils.h>
+#include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/elf.h>
+#include <kernel/percpu.h>
+#include <kernel/pmap.h>
+#include <kernel/print.h>
+#include <kernel/sched.h>
+#include <kernel/vas.h>
+#include <kernel/vdso.h>
+#include <kernel/vmo.h>
+#include <assert.h>
+#include <string.h>
+
+zn_status_t elf_load(struct exec_info* info, struct task** out) {
+    zn_status_t status;
+
+    size_t read;
+    struct elf_ehdr ehdr = {};
+    vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
+
+    // Check if the file is an ELF.
+    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
+        return ZN_ERR_INVALID;
+    if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
+        return ZN_ERR_INVALID;
+    if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS)
+        return ZN_ERR_INVALID;
+    if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
+        return ZN_ERR_INVALID;
+    if (ehdr.e_machine != ELF_ARCH_MACHINE)
+        return ZN_ERR_INVALID;
+    if (ehdr.e_type != ET_EXEC)
+        return ZN_ERR_INVALID;
+
+    const size_t page_size = arch_mem_page_size();
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        struct elf_phdr phdr = {};
+        vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
+        if (phdr.p_type == PT_LOAD) {
+            enum zn_vm_flags prot = 0;
+            if (phdr.p_flags & PF_R)
+                prot |= ZN_VM_MAP_READ;
+            if (phdr.p_flags & PF_W)
+                prot |= ZN_VM_MAP_WRITE;
+            if (phdr.p_flags & PF_X)
+                prot |= ZN_VM_MAP_EXEC;
+
+            assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
+
+            const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
+            const uintptr_t map_address = phdr.p_vaddr - misalign;
+            const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
+            const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
+
+            // Copy the file data into its own mapping.
+            struct paged_vmo* phdr_obj;
+            status = vmo_new_phys(&phdr_obj);
+            if (status)
+                return status;
+
+            status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
+            if (status)
+                return status;
+
+            // We map more than we copied so the rest is filled with zeroed pages.
+            status = vas_map_vmo(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
+            if (status)
+                return status;
+        }
+    }
+
+    const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
+    const size_t stack_size = 2 * 1024 * 1024; // 2MiB
+    const uintptr_t stack_start = highest - stack_size;
+
+    // Allocate stack.
+    struct paged_vmo* stack;
+    status = vmo_new_phys(&stack);
+    if (status)
+        return status;
+
+    // Fill in stack info.
+    uintptr_t stack_off = stack_size;
+
+    // TODO: Instead of allocating, calculate it on the fly.
+    size_t num_envp = 0;
+    for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
+        num_envp++;
+    }
+    size_t num_argv = 0;
+    for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
+        num_argv++;
+    }
+    uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
+    uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
+
+    for (size_t env = 0; env < num_envp; env++) {
+        const char nul = 0;
+        stack_off -= 1;
+        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
+
+        const size_t len = strlen(info->envp[env]);
+        stack_off -= len;
+        vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr);
+
+        envp_offsets[env] = stack_start + stack_off;
+    }
+
+    for (size_t arg = 0; arg < num_argv; arg++) {
+        const char nul = 0;
+        stack_off -= 1;
+        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
+
+        const size_t len = strlen(info->argv[arg]);
+        stack_off -= len;
+        vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr);
+
+        argv_offsets[arg] = stack_start + stack_off;
+    }
+
+    stack_off = ALIGN_DOWN(stack_off, 16);
+
+    // Align the stack if argc + argv + envp does not add up to 16 byte alignment.
+    if ((1 + num_argv + num_envp) & 1) {
+        stack_off -= sizeof(uintptr_t);
+        uintptr_t zero = 0;
+        vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
+    }
+
+    // Add the init handle to the process namespace.
+    zn_handle_t init_handle = ZN_HANDLE_INVALID;
+    if (info->init_handle) {
+        status = namespace_add_desc(info->ns, *info->init_handle, &init_handle);
+        if (status)
+            return status;
+    }
+
+#define WRITE_AUXV(auxv, value) \
+    do { \
+        stack_off -= sizeof(uintptr_t); \
+        uintptr_t auxv_val = value; \
+        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
+        stack_off -= sizeof(uintptr_t); \
+        auxv_val = auxv; \
+        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
+    } while (0)
+
+    // Write auxiliary values.
+    WRITE_AUXV(AT_NULL, 0);
+    WRITE_AUXV(AT_SECURE, 0);
+    WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
+    WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
+    WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
+    WRITE_AUXV(AT_INIT_HANDLE, init_handle);
+
+    // envp pointers.
+    stack_off -= sizeof(uintptr_t);
+    const uintptr_t zero = 0;
+    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
+    for (size_t env = 0; env < num_envp; env++) {
+        stack_off -= sizeof(uintptr_t);
+        vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr);
+    }
+
+    // argv pointers.
+    stack_off -= sizeof(uintptr_t);
+    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
+    for (size_t arg = 0; arg < num_argv; arg++) {
+        stack_off -= sizeof(uintptr_t);
+        vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr);
+    }
+
+    mem_free(envp_offsets);
+    mem_free(argv_offsets);
+
+    // argc
+    stack_off -= sizeof(uintptr_t);
+    uintptr_t argc = num_argv;
+    vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
+
+    status = vas_map_vmo(info->space, &stack->object, stack_start, stack_size, ZN_VM_MAP_READ | ZN_VM_MAP_WRITE, 0);
+    if (status)
+        return status;
+
+    struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
+    start_info->ip = ehdr.e_entry;
+    start_info->sp = highest - stack_size + stack_off;
+    start_info->arg = 0;
+
+    struct task* result;
+    status = task_create(info->argv[0], info->space, info->ns, sched_to_user, (uintptr_t)start_info, &result);
+    if (status)
+        return status;
+
+    *out = result;
+    return ZN_OK;
+}

kernel/core/futex.c

@@ -0,0 +1,120 @@
+#include <zinnia/status.h>
+#include <kernel/futex.h>
+#include <kernel/percpu.h>
+#include <kernel/print.h>
+#include <kernel/sched.h>
+#include <kernel/spin.h>
+#include <kernel/usercopy.h>
+
+struct futex_bucket {
+    struct wait_queue wq;
+    uintptr_t key; // Virtual address used as the futex key.
+    bool in_use;
+};
+
+#define FUTEX_BUCKET_COUNT 64
+
+static struct futex_bucket futex_table[FUTEX_BUCKET_COUNT];
+static struct spinlock futex_table_lock;
+
+void futex_init(void) {
+    futex_table_lock = (struct spinlock){0};
+    for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) {
+        wait_queue_init(&futex_table[i].wq);
+        futex_table[i].key = 0;
+        futex_table[i].in_use = false;
+    }
+}
+
+static size_t futex_hash(uintptr_t addr) {
+    // Simple hash: mix the address bits and mod by bucket count.
+    addr = (addr >> 2) ^ (addr >> 12);
+    return addr % FUTEX_BUCKET_COUNT;
+}
+
+static struct futex_bucket* futex_bucket_get(uintptr_t addr) {
+    size_t idx = futex_hash(addr);
+    // Linear probing to find a matching or free bucket.
+    for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) {
+        size_t slot = (idx + i) % FUTEX_BUCKET_COUNT;
+        struct futex_bucket* b = &futex_table[slot];
+        if (b->in_use && b->key == addr)
+            return b;
+        if (!b->in_use) {
+            b->key = addr;
+            b->in_use = true;
+            return b;
+        }
+    }
+    return nullptr;
+}
+
+zn_status_t futex_wait(__user int* addr, int expected) {
+    uintptr_t key = (uintptr_t)addr;
+
+    spin_lock(&futex_table_lock);
+
+    // Read the current value at the userspace address.
+    int current;
+    if (!usercopy_read(&current, addr, sizeof(*addr))) {
+        spin_unlock(&futex_table_lock);
+        return ZN_ERR_BAD_BUFFER;
+    }
+
+    // If the value has already changed, don't block.
+    if (current != expected) {
+        spin_unlock(&futex_table_lock);
+        return ZN_OK;
+    }
+
+    struct futex_bucket* bucket = futex_bucket_get(key);
+    if (!bucket) {
+        spin_unlock(&futex_table_lock);
+        return ZN_ERR_NO_MEMORY;
+    }
+
+    spin_unlock(&futex_table_lock);
+
+    // Block on the bucket's wait queue.
+    struct sched_percpu* sched = &percpu_get()->sched;
+    sched_block(sched, &bucket->wq);
+
+    return ZN_OK;
+}
+
+zn_status_t futex_wake(__user int* addr, int count) {
+    uintptr_t key = (uintptr_t)addr;
+
+    spin_lock(&futex_table_lock);
+
+    size_t idx = futex_hash(key);
+    struct futex_bucket* bucket = nullptr;
+    for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) {
+        size_t slot = (idx + i) % FUTEX_BUCKET_COUNT;
+        struct futex_bucket* b = &futex_table[slot];
+        if (b->in_use && b->key == key) {
+            bucket = b;
+            break;
+        }
+        if (!b->in_use)
+            break;
+    }
+
+    if (!bucket) {
+        spin_unlock(&futex_table_lock);
+        return ZN_OK; // No waiters, nothing to do.
+    }
+
+    spin_unlock(&futex_table_lock);
+
+    struct sched_percpu* sched = &percpu_get()->sched;
+    if (count == INT32_MAX) {
+        sched_wake_all(sched, &bucket->wq);
+    } else {
+        for (int i = 0; i < count; i++) {
+            sched_wake(sched, &bucket->wq);
+        }
+    }
+
+    return ZN_OK;
+}

kernel/core/irq.c

@@ -1,16 +1,29 @@
+#include <kernel/arch.h>
 #include <kernel/irq.h>
 #include <kernel/percpu.h>
 #include <stdatomic.h>
 
 void irq_lock() {
-    irq_set_state(false);
+    struct irq_percpu* cpu = &percpu_get()->irq;
+    if (atomic_load_explicit(&cpu->in_interrupt, memory_order_acquire))
+        return;
+
+    arch_irq_set_state(false);
     atomic_fetch_add_explicit(&percpu_get()->irq.level, 1, memory_order_acq_rel);
 }
 
 void irq_unlock() {
-    uint32_t old_level = atomic_fetch_sub_explicit(&percpu_get()->irq.level, 1, memory_order_acq_rel);
+    struct irq_percpu* cpu = &percpu_get()->irq;
+    if (atomic_load_explicit(&cpu->in_interrupt, memory_order_acquire))
+        return;
+
+    uint32_t old_level = atomic_fetch_sub_explicit(&cpu->level, 1, memory_order_acq_rel);
     // If it was 1, the new IRQ level is now 0.
     if (old_level == 1) {
-        irq_set_state(true);
+        arch_irq_set_state(true);
     }
 }
+
+bool irq_set_interrupted(bool is_interrupted) {
+    return atomic_exchange_explicit(&percpu_get()->irq.in_interrupt, is_interrupted, memory_order_release);
+}

kernel/core/meson.build

@@ -1,11 +1,16 @@
 kernel_sources += files(
+    'channel.c',
     'clock.c',
     'console.c',
     'elf.c',
+    'futex.c',
     'irq.c',
+    'namespace.c',
     'panic.c',
     'percpu.c',
     'print.c',
     'sched.c',
     'spin.c',
+    'uacpi_glue.c',
+    'vdso.c',
 )

kernel/core/namespace.c

@@ -0,0 +1,121 @@
+#include <zinnia/handle.h>
+#include <zinnia/rights.h>
+#include <zinnia/status.h>
+#include <kernel/alloc.h>
+#include <kernel/mutex.h>
+#include <kernel/namespace.h>
+#include <stdatomic.h>
+
+zn_status_t namespace_new(struct namespace** out) {
+    if (!out)
+        return ZN_ERR_INTERNAL;
+
+    struct namespace* namespace = mem_alloc(sizeof(struct namespace), 0);
+    if (!namespace)
+        return ZN_ERR_NO_MEMORY;
+
+    namespace->mutex = (struct mutex){0};
+    namespace->next_handle = 1;
+    hashmap_init(&namespace->descriptors);
+
+    *out = namespace;
+    return ZN_OK;
+}
+
+zn_status_t namespace_add_desc(struct namespace* namespace, struct namespace_desc desc, zn_handle_t* out) {
+    if (!namespace)
+        return ZN_ERR_INTERNAL;
+
+    // Skip over handles that are already in use (e.g. hard-coded handles).
+    // mutex_lock(&namespace->mutex);
+    zn_handle_t handle;
+    do {
+        handle = atomic_fetch_add_explicit(&namespace->next_handle, 1, memory_order_acq_rel);
+        if (handle <= ZN_HANDLE_INVALID)
+            continue;
+    } while (hashmap_get(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default) != nullptr);
+
+    zn_status_t status =
+        hashmap_insert(&namespace->descriptors, handle, desc, hashmap_hash_default, hashmap_eq_default);
+    // mutex_unlock(&namespace->mutex);
+
+    if (status != ZN_OK)
+        return status;
+
+    *out = handle;
+    return ZN_OK;
+}
+
+zn_status_t namespace_add_desc_at(struct namespace* namespace, struct namespace_desc desc, zn_handle_t handle) {
+    if (!namespace)
+        return ZN_ERR_INTERNAL;
+
+    // Skip Invalid or reserved handles.
+    if (handle <= ZN_HANDLE_INVALID)
+        return ZN_ERR_BAD_HANDLE;
+
+    // Fail if a descriptor already exists at this handle.
+    if (hashmap_get(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default) != nullptr)
+        return ZN_ERR_ALREADY_EXISTS;
+
+    return hashmap_insert(&namespace->descriptors, handle, desc, hashmap_hash_default, hashmap_eq_default);
+}
+
+zn_status_t namespace_get(struct namespace* namespace, zn_handle_t handle, struct namespace_desc* out) {
+    if (!namespace)
+        return ZN_ERR_INTERNAL;
+    if (!out)
+        return ZN_ERR_INTERNAL;
+
+    if (handle <= ZN_HANDLE_INVALID)
+        return ZN_ERR_BAD_HANDLE;
+
+    struct namespace_desc* desc =
+        hashmap_get(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default);
+    if (!desc)
+        return ZN_ERR_BAD_HANDLE;
+
+    *out = *desc;
+    return ZN_OK;
+}
+
+zn_status_t namespace_del_desc(struct namespace* namespace, zn_handle_t handle) {
+    if (!namespace)
+        return ZN_ERR_INTERNAL;
+
+    if (handle <= ZN_HANDLE_INVALID)
+        return ZN_ERR_BAD_HANDLE;
+
+    if (!hashmap_remove(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default))
+        return ZN_ERR_BAD_HANDLE;
+
+    return ZN_OK;
+}
+
+zn_status_t namespace_clone_desc(
+    struct namespace* ns,
+    struct namespace_desc* desc,
+    zn_rights_t cloned_rights,
+    zn_handle_t* out
+) {
+    // The source handle must have the clone right.
+    if (!(desc->rights & ZN_RIGHT_CLONE))
+        return ZN_ERR_BAD_PERMS;
+
+    // Determine the rights for the cloned handle.
+    zn_rights_t new_rights;
+    if (cloned_rights & ZN_RIGHTS_IDENTICAL) {
+        new_rights = desc->rights;
+    } else {
+        // The requested rights must be a subset of the original rights.
+        if (cloned_rights & ~desc->rights)
+            return ZN_ERR_BAD_PERMS;
+        new_rights = cloned_rights;
+    }
+
+    // Create the cloned descriptor.
+    struct namespace_desc new_desc = *desc;
+    new_desc.rights = new_rights;
+
+    return namespace_add_desc(ns, new_desc, out);
+}

kernel/core/panic.c

@@ -1,7 +1,33 @@
+#include <kernel/arch.h>
+#include <kernel/irq.h>
 #include <kernel/panic.h>
+#include <kernel/percpu.h>
+#include <kernel/print.h>
+#include <stdarg.h>
 
 [[noreturn]]
-void panic() {
+void panic(const char* msg, ...) {
-    // TODO
+    // TODO: Stop all other CPUs with IPI.
-    while (1) {}
+    arch_irq_set_state(false);
+
+    struct percpu* cpu = percpu_get();
+    struct task* current = cpu->sched.current;
+    kprintf(
+        "----[ Kernel panic ]----\n"
+        "In task \"%s\" (TID %zu) on CPU %zu!\n",
+        current ? current->name : nullptr,
+        current ? current->id : 0,
+        cpu->id
+    );
+
+    va_list args;
+    va_start(args, msg);
+    kvprintf(msg, args);
+    va_end(args);
+
+    kprintf("----[ End of panic ]----\n");
+
+    while (1) {
+        arch_irq_wait();
+    }
 }

kernel/core/percpu.c

@@ -1,4 +1,5 @@
-#include <kernel/compiler.h>
+#include <common/compiler.h>
+#include <kernel/arch.h>
 #include <kernel/percpu.h>
 
 [[__used, __section(".percpu")]]
@@ -12,3 +13,15 @@ struct percpu* percpu_new() {
     // TODO
     return nullptr;
 }
+
+struct percpu* percpu_get() {
+    return arch_percpu_get();
+}
+
+void percpu_bsp_init() {
+    arch_percpu_bsp_init();
+}
+
+void percpu_init() {
+    arch_percpu_init();
+}

kernel/core/print.c

@@ -1,3 +1,4 @@
+#include <kernel/clock.h>
 #include <kernel/console.h>
 #include <kernel/panic.h>
 #include <kernel/print.h>
@@ -17,73 +18,97 @@ static void reverse(char* s) {
     }
 }
 
-static int pow(int x, unsigned int y) {
+static int char_to_digit(char c) {
-    if (y == 0)
+    if (c >= '0' && c <= '9')
-        return 1;
+        return c - '0';
-    else if ((y % 2) == 0)
+    if (c >= 'a' && c <= 'z')
-        return pow(x, y / 2) * pow(x, y / 2);
+        return c - 'a' + 10;
-    else
+    if (c >= 'A' && c <= 'Z')
-        return x * pow(x, y / 2) * pow(x, y / 2);
+        return c - 'A' + 10;
+    return -1;
+}
+
+unsigned long atolu(const char* str, int base) {
+    unsigned long result = 0;
+
+    if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
+        str += 2;
+    }
+
+    while (*str) {
+        int digit = char_to_digit(*str++);
+        if (digit < 0 || digit >= base)
+            break;
+
+        result = result * base + digit;
+    }
+
+    return result;
 }
 
 long atol(const char* str, int base) {
-    size_t len = strlen(str);
+    int neg = 0;
-    int64_t result = 0;
-    size_t i = 0;
-    if (str[0] == '-')
-        i++;
-    for (; i < len; i++)
-        result += (str[i] - '0') * pow(base, len - i - 1);
-    if (str[0] == '-')
-        result *= -1;
-    return result;
-};
 
-unsigned long atolu(const char* str, int base) {
+    if (*str == '-') {
-    size_t len = strlen(str);
+        neg = 1;
-    uint64_t result = 0;
+        str++;
-    size_t i = 0;
+    }
-    for (; i < len; i++)
-        result += (str[i] - '0') * pow(base, len - i - 1);
-    return result;
-};
 
-char* ltoa(int64_t value, char* str, int base) {
+    unsigned long val = atolu(str, base);
-    size_t i;
+    return neg ? -(long)val : (long)val;
-    int64_t sign;
+}
-    sign = value;
-    if (base == 10 && sign < 0)
-        value = -value;
-    i = 0;
-    do {
-        char c = value % base;
-        if (c > 9)
-            c += 7;
-        str[i++] = c + '0';
-    } while ((value /= base) > 0);
-    if (base == 10 && sign < 0)
-        str[i++] = '-';
-    str[i] = '\0';
-    reverse(str);
-    return str;
-};
 
 char* lutoa(uint64_t value, char* str, int base) {
+    if (base < 2 || base > 36)
+        return nullptr;
+
     size_t i = 0;
+
     do {
-        char c = value % base;
+        uint64_t digit = value % base;
-        if (c > 9)
+        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
-            c += 7;
+        value /= base;
-        str[i++] = c + '0';
+    } while (value);
-    } while ((value /= base) > 0);
+
     str[i] = '\0';
     reverse(str);
     return str;
-};
+}
+
+char* ltoa(int64_t value, char* str, int base) {
+    if (base < 2 || base > 36)
+        return nullptr;
+
+    uint64_t magnitude;
+    int negative = 0;
+
+    if (value < 0) {
+        negative = 1;
+        magnitude = (uint64_t)(-(value + 1)) + 1;
+        // avoids INT64_MIN overflow
+    } else {
+        magnitude = (uint64_t)value;
+    }
+
+    size_t i = 0;
+
+    do {
+        uint64_t digit = magnitude % base;
+        str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
+        magnitude /= base;
+    } while (magnitude);
+
+    if (negative)
+        str[i++] = '-';
+
+    str[i] = '\0';
+    reverse(str);
+    return str;
+}
 
 typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
 
-static void kvprintf(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
+static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
     // Amount of bytes written.
     int32_t written = 0;
 
@@ -219,8 +244,8 @@ check_fmt:
             if (!str)
                 str = "(null)";
             size_t len = 0;
-            if (has_width)
+            if (has_precision)
-                len = strnlen(str, width);
+                len = strnlen(str, precision);
             else
                 len = strlen(str);
 
@@ -365,8 +390,10 @@ static void printf_console(void*, const char* msg, size_t len) {
 void kprintf(const char* message, ...) {
     va_list args;
     va_start(args, message);
-
+    do_print(printf_console, nullptr, message, args);
-    kvprintf(printf_console, nullptr, message, args);
-
     va_end(args);
 }
+
+void kvprintf(const char* message, va_list args) {
+    do_print(printf_console, nullptr, message, args);
+}

kernel/core/sched.c

@@ -1,6 +1,185 @@
+#include <zinnia/status.h>
+#include <kernel/alloc.h>
+#include <kernel/arch.h>
+#include <kernel/irq.h>
+#include <kernel/list.h>
+#include <kernel/percpu.h>
 #include <kernel/print.h>
 #include <kernel/sched.h>
+#include <kernel/spin.h>
+#include <kernel/tailq.h>
+#include <assert.h>
+#include <stdatomic.h>
+#include <stdint.h>
+#include <string.h>
+
+static void idle_func(uintptr_t) {
+    while (1) {
+        arch_irq_set_state(true);
+        arch_irq_wait();
+    }
+}
+
+void sched_to_user(uintptr_t info_ptr) {
+    struct task_start_info* info = (struct task_start_info*)info_ptr;
+    uintptr_t ip = info->ip;
+    uintptr_t sp = info->sp;
+    uintptr_t arg = info->arg;
+    mem_free(info);
+    arch_sched_jump_to_user(ip, sp, arg);
+}
+
+void sched_to_user_context(uintptr_t ctx) {
+    arch_sched_jump_to_context((struct arch_context*)ctx);
+}
+
+void sched_init(struct sched_percpu* sched) {
+    struct task* idle_task;
+    assert(
+        task_create("idle", &kernel_vas, nullptr, idle_func, 0, &idle_task) == ZN_OK,
+        "Unable to create idle task!\n"
+    );
+    sched->idle_task = idle_task;
+    sched->current = idle_task;
+
+    tailq_init(&sched->run_queue);
 
-void sched_init() {
     kprintf("Scheduler initialized\n");
 }
+
+void sched_add_task(struct sched_percpu* sched, struct task* task) {
+    assert(task != nullptr, "Tried to add NULL to the run queue\n");
+    tailq_insert_tail(&sched->run_queue, task, next);
+}
+
+static struct task* next_task(struct sched_percpu* sched) {
+    struct task* t = tailq_first(&sched->run_queue);
+
+    if (t == nullptr)
+        return sched->idle_task;
+
+    tailq_remove(&sched->run_queue, t, next);
+
+    return t;
+}
+
+static void do_reschedule(struct sched_percpu* sched) {
+    struct task* from = atomic_load_explicit(&sched->current, memory_order_acquire);
+    struct task* to = next_task(sched);
+
+    if (from == to) {
+        irq_unlock();
+        return;
+    }
+
+    atomic_store_explicit(&sched->current, to, memory_order_relaxed);
+
+    pmap_set(&to->space->pmap);
+
+    struct percpu* cpu = percpu_get();
+    from->kernel_stack = cpu->kernel_stack;
+    from->user_stack = cpu->user_stack;
+    cpu->kernel_stack = to->kernel_stack;
+    cpu->user_stack = to->user_stack;
+
+    arch_sched_switch(from, to);
+}
+
+void sched_reschedule(struct sched_percpu* sched) {
+    irq_lock();
+    if (sched->current->state != TASK_STATE_DEAD)
+        sched_add_task(sched, sched->current);
+    do_reschedule(sched);
+}
+
+void sched_yield(struct sched_percpu* sched) {
+    irq_lock();
+    do_reschedule(sched);
+}
+
+zn_status_t task_create(
+    const char* name,
+    struct vas* space,
+    struct namespace* ns,
+    task_fn_t entry,
+    uintptr_t arg0,
+    struct task** out
+) {
+    if (!out)
+        return ZN_ERR_INVALID;
+
+    struct task* new_task = mem_alloc(sizeof(struct task), 0);
+    if (new_task == nullptr)
+        return ZN_ERR_NO_MEMORY;
+
+    static size_t last_tid = 0;
+    new_task->id = atomic_fetch_add(&last_tid, 1);
+    new_task->space = space;
+    new_task->namespace = ns;
+    strncpy(new_task->name, name, sizeof(new_task->name));
+
+    // Allocate a kernel stack.
+    new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0);
+    if (!new_task->kernel_stack)
+        return ZN_ERR_NO_MEMORY;
+    new_task->user_stack = 0;
+
+    // Initialize the arch-dependent parts.
+    zn_status_t s = arch_task_init(&new_task->task_ctx, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
+    if (s != ZN_OK)
+        return s;
+
+    *out = new_task;
+    return ZN_OK;
+}
+
+void task_entry(task_fn_t entry, uintptr_t arg0) {
+    entry(arg0);
+
+    struct sched_percpu* sched = &percpu_get()->sched;
+    sched->current->state = TASK_STATE_DEAD;
+    sched_yield(sched);
+    panic("unreachable");
+}
+
+void wait_queue_init(struct wait_queue* wq) {
+    tailq_init(&wq->waiters);
+    wq->lock = (struct spinlock){0};
+}
+
+void sched_block(struct sched_percpu* sched, struct wait_queue* wq) {
+    irq_lock();
+    spin_lock(&wq->lock);
+
+    sched->current->state = TASK_STATE_BLOCKED;
+    tailq_insert_tail(&wq->waiters, sched->current, next);
+
+    spin_unlock(&wq->lock);
+    do_reschedule(sched);
+}
+
+void sched_wake(struct sched_percpu* sched, struct wait_queue* wq) {
+    spin_lock(&wq->lock);
+
+    struct task* t = tailq_first(&wq->waiters);
+    if (t != nullptr) {
+        tailq_remove(&wq->waiters, t, next);
+        t->state = TASK_STATE_READY;
+        sched_add_task(sched, t);
+    }
+
+    spin_unlock(&wq->lock);
+}
+
+void sched_wake_all(struct sched_percpu* sched, struct wait_queue* wq) {
+    spin_lock(&wq->lock);
+
+    struct task* t;
+    while ((t = tailq_first(&wq->waiters)) != nullptr) {
+        tailq_remove(&wq->waiters, t, next);
+        t->state = TASK_STATE_READY;
+        sched_add_task(sched, t);
+    }
+
+    spin_unlock(&wq->lock);
+}

kernel/core/spin.c

@@ -1,16 +1,15 @@
 #include <kernel/spin.h>
 #include <stdatomic.h>
 
-// TODO: Ticketing system
-
 void spin_lock(struct spinlock* spin) {
-    while (atomic_exchange_explicit(&spin->locked, true, memory_order_acquire)) {
+    uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed);
+
+    while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) {
         asm volatile("pause");
     }
-
-    atomic_store_explicit(&spin->locked, true, memory_order_release);
 }
 
 void spin_unlock(struct spinlock* spin) {
-    atomic_store_explicit(&spin->locked, false, memory_order_release);
+    uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed);
+    atomic_store_explicit(&spin->owner, val + 1, memory_order_release);
 }

kernel/core/uacpi_glue.c

@@ -0,0 +1,42 @@
+#include <common/utils.h>
+#include <kernel/arch.h>
+#include <kernel/init.h>
+#include <kernel/mmio.h>
+#include <kernel/pmap.h>
+#include <kernel/print.h>
+#include <kernel/vas.h>
+#include <kernel/virt.h>
+#include <stdarg.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <uacpi/types.h>
+
+uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) {
+    *out_rsdp_address = (uacpi_phys_addr)rsdp_addr;
+    return UACPI_STATUS_OK;
+}
+
+void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
+    uintptr_t vaddr = vm_alloc(len);
+    for (size_t i = 0; i < len; i += arch_mem_page_size()) {
+        pmap_map(&kernel_vas.pmap, vaddr, addr, PTE_READ | PTE_WRITE, CACHE_MMIO);
+    }
+    return (void*)vaddr;
+}
+
+void uacpi_kernel_unmap(void* addr, uacpi_size len) {
+    // TODO
+}
+
+void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
+    va_list args;
+    va_start(args, msg);
+
+    kvprintf(msg, args);
+
+    va_end(args);
+}
+
+void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
+    kvprintf(msg, args);
+}

kernel/core/vdso.c

@@ -0,0 +1,10 @@
+#include <common/compiler.h>
+#include <kernel/vdso.h>
+#include <stdint.h>
+
+[[__used, __section(".vdso")]]
+const uint8_t vdso_bytes[] = {
+#embed VDSO_PATH
+};
+
+const size_t vdso_length = sizeof(vdso_bytes);

kernel/include/assert.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <common/compiler.h>
+#include <kernel/panic.h>
+
+// Like the standard C assert, but also active in debug builds.
+#define assert(expr, msg, ...) \
+    do { \
+        if (__unlikely(!(expr))) { \
+            panic( \
+                "In function \"%s\" (%s:%u):\n" \
+                "Assertion \"%s\" failed!\n" msg, \
+                __FUNCTION__, \
+                __FILE__, \
+                __LINE__, \
+                #expr, \
+                ##__VA_ARGS__ \
+            ); \
+        } \
+    } while (0)
+
+#ifndef NDEBUG
+#define debug_assert(...) assert(__VA_ARGS__)
+#else
+#define debug_assert(...) ((void)0)
+#endif

kernel/include/kernel/alloc.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <zinnia/status.h>
+#include <kernel/types.h>
+#include <stddef.h>
+
+enum alloc_flags {
+    ALLOC_NOZERO = 1 << 0, // Don't zero out the allocated memory.
+    ALLOC_MEM32 = 1 << 1,  // Allocated memory needs to fit inside 32 bits.
+    ALLOC_MEM20 = 1 << 2,  // Allocated memory needs to fit inside 20 bits.
+};
+
+enum phys_mem_usage {
+    PHYS_RESERVED,
+    PHYS_USABLE,
+    PHYS_STATIC,
+    PHYS_RECLAIMABLE,
+};
+
+struct phys_mem {
+    phys_t address;
+    size_t length;
+    enum phys_mem_usage usage;
+};
+
+// Base address of the HHDM.
+extern uintptr_t mem_hhdm_base;
+#define HHDM_PTR(paddr) (void*)(mem_hhdm_base + (paddr))
+
+// Allocates a region of memory which can be smaller than the page size.
+// Returns `nullptr` if the allocator cannot provide an allocation for the given `length` + `flags`.
+// Always returns `nullptr` if `length` is 0.
+void* mem_alloc(size_t length, enum alloc_flags flags);
+
+// Frees an allocation created by `mem_alloc`.
+// Passing `nullptr` is a no-op.
+void mem_free(void* mem);
+
+zn_status_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out);
+zn_status_t mem_phys_free(phys_t start, size_t num_pages);
+
+void mem_init(struct phys_mem* map, size_t length, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm);
+void slab_init();
+void mem_phys_bootstrap(struct phys_mem* mem);
+void mem_phys_init(struct phys_mem* map, size_t length);
+
+size_t mem_high_shift();

kernel/include/kernel/arch.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+void arch_start();
+void arch_panic();
+
+size_t arch_mem_page_size();
+size_t arch_mem_page_bits();
+size_t arch_mem_level_bits();
+size_t arch_mem_num_levels();
+uintptr_t arch_mem_hhdm_addr();
+uintptr_t arch_mem_pfndb_addr();
+uintptr_t arch_mem_mapping_addr();
+
+#include <kernel/usercopy.h>
+
+bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region);
+bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region);
+
+#include <kernel/irq.h>
+
+void arch_irq_set_state(bool state);
+bool arch_irq_get_state();
+void arch_irq_wait();
+
+#include <kernel/percpu.h>
+
+struct percpu* arch_percpu_get();
+void arch_percpu_bsp_init();
+void arch_percpu_init();

kernel/include/kernel/assert.h

@@ -1,22 +0,0 @@
-#pragma once
-
-#include <kernel/compiler.h>
-#include <kernel/panic.h>
-#include <kernel/print.h>
-
-#define ASSERT(expr, msg, ...) \
-    ({ \
-        if (__unlikely(!(expr))) { \
-            kprintf( \
-                "\e[31mKernel panic - Environment is unsound!\n" \
-                "In function \"%s\" (%s:%u):\n" \
-                "Assertion \"%s\" failed! " msg "\n", \
-                __FUNCTION__, \
-                __FILE__, \
-                __LINE__, \
-                #expr, \
-                ##__VA_ARGS__ \
-            ); \
-            panic(); \
-        } \
-    })

kernel/include/kernel/channel.h

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <zinnia/channel.h>
+#include <zinnia/status.h>
+#include <kernel/spin.h>
+#include <kernel/tailq.h>
+#include <kernel/wait.h>
+#include <stddef.h>
+
+struct namespace_desc;
+
+struct channel_msg {
+    void* data;
+    size_t num_bytes;
+    struct namespace_desc* descs;
+    size_t num_handles;
+    tailq_link(struct channel_msg) next;
+};
+
+struct channel {
+    struct channel* peer;
+    struct spinlock lock;
+    enum zn_channel_flags flags;
+    tailq_head(struct channel_msg) messages;
+    struct wait_queue waiters;
+};
+
+// Creates a linked pair of channel endpoints.
+zn_status_t channel_create(struct channel** ep0, struct channel** ep1);
+
+// Writes a message to the peer channel.
+// On success, ownership of `data` and `descs` transfers to the message.
+// On failure, the caller retains ownership and must free them.
+zn_status_t channel_write(
+    struct channel* ch,
+    void* data,
+    size_t num_bytes,
+    struct namespace_desc* descs,
+    size_t num_handles
+);
+
+// Non-blocking dequeue of the next message.
+zn_status_t channel_read(struct channel* ch, struct channel_msg** msg);
+
+// Non-blocking check for message availability.
+zn_status_t channel_poll(struct channel* ch);
+
+// Frees a channel message and its associated data and descriptors.
+void channel_msg_free(struct channel_msg* msg);

kernel/include/kernel/clock.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <stdint.h>
+
+struct clock {
+    const char* name;
+    uint8_t priority;
+
+    void (*reset)(struct clock* c);
+    uint64_t (*get_elapsed_ns)(struct clock* c);
+};
+
+// Attempts to set a new clock. Returns true if the clock was changed.
+bool clock_switch(struct clock* clock);
+
+// Returns true, if a clock is available.
+bool clock_available();
+
+// Gets the current timestamp in nanoseconds since the last reset.
+uint64_t clock_get_elapsed_ns();
+
+// Spin for `ns` nanoseconds.
+void clock_spin_ns(uint64_t ns);

kernel/include/kernel/cmdline.h

@@ -1,24 +1,45 @@
 #pragma once
 
-#include <kernel/common.h>
+#include <common/compiler.h>
-#include <kernel/compiler.h>
+#include <common/utils.h>
+#include <stdint.h>
 
-#define CMDLINE_MAX 0x1000
+#define _CMDLINE_OPTION(x, opt_name, opt_func) \
-
+    static const struct cmdline_option x = { \
-#define CMDLINE_OPTION(opt_name, opt_func) \
-    [[__used, __section(".cmdline")]] \
-    static const struct cmdline_option UNIQUE_IDENT(__cmline_option_) = { \
         .name = opt_name, \
-        .func = opt_func, \
+        .option_type = _Generic( \
-    }
+            (opt_func), \
+            void (*)(const char*): CMDLINE_STRING, \
+            void (*)(bool): CMDLINE_BOOL, \
+            void (*)(intptr_t): CMDLINE_INTPTR, \
+            void (*)(uintptr_t): CMDLINE_UINTPTR \
+        ), \
+        .func = (void*)(opt_func), \
+    }; \
+    [[__used, __section(".cmdline")]] \
+    static const struct cmdline_option* CONCAT(x, _ptr) = &x;
+
+#define CMDLINE_OPTION(opt_name, opt_func) _CMDLINE_OPTION(UNIQUE_IDENT(cmdline_option), opt_name, opt_func)
+
+enum cmdline_type {
+    CMDLINE_STRING,
+    CMDLINE_BOOL,
+    CMDLINE_INTPTR,
+    CMDLINE_UINTPTR,
+};
 
 struct cmdline_option {
     // The name of this option.
     const char* name;
     // Gets called if this option is present on the command line.
-    // If an option is specified as `name=value`, then the `value` is passed as well.
+    union {
-    // Otherwise, it's NULL.
+        void (*func_str)(const char* value);
-    void (*func)(const char* value);
+        void (*func_bool)(bool value);
+        void (*func_intptr)(intptr_t value);
+        void (*func_uintptr)(uintptr_t value);
+        void* func;
+    };
+    enum cmdline_type option_type;
 };
 
 // Parses the command line and invokes all options.

kernel/include/kernel/console.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <kernel/common.h>
+#include <common/compiler.h>
-#include <kernel/compiler.h>
+#include <common/utils.h>
 #include <stddef.h>
 
 struct console {

kernel/include/kernel/elf.h

@@ -1,227 +1,18 @@
 #pragma once
 
-#include <bits/elf.h>
+#include <zinnia/status.h>
-#include <stdint.h>
+#include <kernel/namespace.h>
+#include <kernel/sched.h>
+#include <kernel/vas.h>
+#include <kernel/vmo.h>
 
-constexpr char ELF_MAG[4] = {0x7F, 'E', 'L', 'F'};
+struct exec_info {
-
+    struct vas* space;
-typedef uint64_t elf64_addr_t;
+    struct vmo* file_obj;
-typedef uint64_t elf64_off_t;
+    struct namespace* ns;
-typedef uint32_t elf32_addr_t;
+    const char** argv;
-typedef uint32_t elf32_off_t;
+    const char** envp;
-
+    struct namespace_desc* init_handle;
-enum : uint8_t {
-    EI_MAG0 = 0,
-    EI_MAG1 = 1,
-    EI_MAG2 = 2,
-    EI_MAG3 = 3,
-    EI_CLASS = 4,
-    EI_DATA = 5,
-    EI_VERSION = 6,
-    EI_OSABI = 7,
-    EI_ABIVERSION = 8,
-    EI_PAD = 9,
-    EI_NIDENT = 16,
 };
 
-enum : uint16_t {
+zn_status_t elf_load(struct exec_info* info, struct task** out);
-    EM_X86_64 = 62,
-    EM_AARCH64 = 183,
-    EM_RISCV = 243,
-    EM_LOONGARCH = 258,
-};
-
-enum : uint32_t {
-    ELFCLASS32 = 1,
-    ELFCLASS64 = 2,
-};
-
-enum : uint32_t {
-    ELFDATA2LSB = 1,
-    ELFDATA2MSB = 2,
-};
-
-enum : uint32_t {
-    EV_NONE = 0,
-    EV_CURRENT = 1,
-    EV_NUM = 2,
-};
-
-enum : uint32_t {
-    ELFOSABI_SYSV = 0,
-    ELFOSABI_HPUX = 1,
-    ELFOSABI_STANDALONE = 255,
-};
-
-enum : uint32_t {
-    ET_NONE = 0,
-    ET_REL = 1,
-    ET_EXEC = 2,
-    ET_DYN = 3,
-    ET_CORE = 4,
-};
-
-enum : uint32_t {
-    PT_NULL = 0x00000000,
-    PT_LOAD = 0x00000001,
-    PT_DYNAMIC = 0x00000002,
-    PT_INTERP = 0x00000003,
-    PT_NOTE = 0x00000004,
-    PT_SHLIB = 0x00000005,
-    PT_PHDR = 0x00000006,
-    PT_TLS = 0x00000007,
-};
-
-enum : uint32_t {
-    PF_X = 0x00000001,
-    PF_W = 0x00000002,
-    PF_R = 0x00000004,
-};
-
-enum : uint32_t {
-    DT_NULL = 0,
-    DT_NEEDED = 1,
-    DT_PLTRELSZ = 2,
-    DT_PLTGOT = 3,
-    DT_HASH = 4,
-    DT_STRTAB = 5,
-    DT_SYMTAB = 6,
-    DT_RELA = 7,
-    DT_RELASZ = 8,
-    DT_RELAENT = 9,
-    DT_STRSZ = 10,
-    DT_SYMENT = 11,
-    DT_INIT = 12,
-    DT_FINI = 13,
-    DT_SONAME = 14,
-    DT_RPATH = 15,
-    DT_SYMBOLIC = 16,
-    DT_REL = 17,
-    DT_RELSZ = 18,
-    DT_RELENT = 19,
-    DT_PLTREL = 20,
-    DT_DEBUG = 21,
-    DT_TEXTREL = 22,
-    DT_JMPREL = 23,
-    DT_BIND_NOW = 24,
-    DT_INIT_ARRAY = 25,
-    DT_FINI_ARRAY = 26,
-    DT_INIT_ARRAYSZ = 27,
-    DT_FINI_ARRAYSZ = 28,
-    DT_LOOS = 0x60000000,
-    DT_HIOS = 0x6FFFFFFF,
-    DT_LOPROC = 0x70000000,
-    DT_HIPROC = 0x7FFFFFFF,
-};
-
-enum : uint32_t {
-    AT_NULL = 0,
-    AT_IGNORE = 1,
-    AT_EXECFD = 2,
-    AT_PHDR = 3,
-    AT_PHENT = 4,
-    AT_PHNUM = 5,
-    AT_PAGESZ = 6,
-    AT_BASE = 7,
-    AT_FLAGS = 8,
-    AT_ENTRY = 9,
-    AT_NOTELF = 10,
-    AT_UID = 11,
-    AT_EUID = 12,
-    AT_GID = 13,
-    AT_EGID = 14,
-};
-
-struct elf64_ehdr {
-    uint8_t e_ident[EI_NIDENT];
-    uint16_t e_type;
-    uint16_t e_machine;
-    uint32_t e_version;
-    elf64_addr_t e_entry;
-    elf64_off_t e_phoff;
-    elf64_off_t e_shoff;
-    uint32_t e_flags;
-    uint16_t e_ehsize;
-    uint16_t e_phentsize;
-    uint16_t e_phnum;
-    uint16_t e_shentsize;
-    uint16_t e_shnum;
-    uint16_t e_shstrndx;
-};
-
-struct elf32_ehdr {
-    uint8_t e_ident[EI_NIDENT];
-    uint16_t e_type;
-    uint16_t e_machine;
-    uint32_t e_version;
-    elf32_addr_t e_entry;
-    elf32_off_t e_phoff;
-    elf32_off_t e_shoff;
-    uint32_t e_flags;
-    uint16_t e_ehsize;
-    uint16_t e_phentsize;
-    uint16_t e_phnum;
-    uint16_t e_shentsize;
-    uint16_t e_shnum;
-    uint16_t e_shstrndx;
-};
-
-struct elf64_phdr {
-    uint32_t p_type;
-    uint32_t p_flags;
-    elf64_off_t p_offset;
-    elf64_addr_t p_vaddr;
-    elf64_addr_t p_paddr;
-    uint64_t p_filesz;
-    uint64_t p_memsz;
-    uint64_t p_align;
-};
-
-struct elf32_phdr {
-    uint32_t p_type;
-    elf32_off_t p_offset;
-    elf32_addr_t p_vaddr;
-    elf32_addr_t p_paddr;
-    uint32_t p_filesz;
-    uint32_t p_memsz;
-    uint32_t p_flags;
-    uint32_t p_align;
-};
-
-struct elf64_dyn {
-    int64_t d_tag;
-    union {
-        uint64_t d_val;
-        elf64_addr_t d_ptr;
-    } d_un;
-};
-
-struct elf32_dyn {
-    int32_t r_offset;
-    union {
-        uint32_t d_val;
-        elf32_addr_t d_ptr;
-    } d_un;
-};
-
-struct elf64_nhdr {
-    uint64_t n_namesz;
-    uint64_t n_descsz;
-    uint64_t n_type;
-};
-
-struct elf32_nhdr {
-    uint32_t n_namesz;
-    uint32_t n_descsz;
-    uint32_t n_type;
-};
-
-struct elf64_auxv {
-    uint64_t atype;
-    uint64_t avalue;
-};
-
-struct elf32_auxv {
-    uint32_t atype;
-    uint32_t avalue;
-};

kernel/include/kernel/futex.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <zinnia/status.h>
+#include <common/compiler.h>
+#include <kernel/wait.h>
+
+// Initializes the global futex table.
+void futex_init(void);
+
+// Blocks the calling task if *addr == expected.
+zn_status_t futex_wait(__user int* addr, int expected);
+
+// Wakes up to `count` tasks waiting on the given address.
+// Returns the number of tasks woken.
+zn_status_t futex_wake(__user int* addr, int count);

kernel/include/kernel/hashmap.h

@@ -0,0 +1,223 @@
+#pragma once
+
+#include <zinnia/status.h>
+#include <kernel/alloc.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+// FNV-1a hash over raw bytes.
+static inline uint64_t hashmap_hash_default(const void* key, size_t len) {
+    const uint8_t* data = (const uint8_t*)key;
+    uint64_t hash = 0xcbf29ce484222325;
+    for (size_t i = 0; i < len; i++) {
+        hash ^= data[i];
+        hash *= 0x100000001b3;
+    }
+    return hash;
+}
+
+// Byte-wise equality.
+static inline bool hashmap_eq_default(const void* a, const void* b, size_t len) {
+    return memcmp(a, b, len) == 0;
+}
+
+// Ensure a hash is never zero (zero is the empty sentinel).
+#define _HASHMAP_FIXHASH(h) ((h) == 0 ? (uint64_t)1 : (h))
+
+#define hashmap(K, V) \
+    struct { \
+        K* keys; \
+        V* values; \
+        uint64_t* hashes; \
+        size_t capacity; \
+        size_t count; \
+    }
+
+#define hashmap_init(map) \
+    do { \
+        (map)->keys = nullptr; \
+        (map)->values = nullptr; \
+        (map)->hashes = nullptr; \
+        (map)->capacity = 0; \
+        (map)->count = 0; \
+    } while (0)
+
+#define hashmap_destroy(map) \
+    do { \
+        mem_free((map)->keys); \
+        mem_free((map)->values); \
+        mem_free((map)->hashes); \
+        (map)->keys = nullptr; \
+        (map)->values = nullptr; \
+        (map)->hashes = nullptr; \
+        (map)->capacity = 0; \
+        (map)->count = 0; \
+    } while (0)
+
+#define hashmap_count(map) ((map)->count)
+
+// Internal: probe distance from ideal position.
+#define _HASHMAP_PROBE_DIST(hash, pos, cap) (((pos) - ((hash) & ((cap) - 1))) & ((cap) - 1))
+
+// Internal: grow and rehash.
+#define _HASHMAP_GROW(map, hash_fn, eq_fn) \
+    do { \
+        size_t _old_cap = (map)->capacity; \
+        size_t _new_cap = _old_cap == 0 ? 16 : _old_cap * 2; \
+        typeof((map)->keys) _old_keys = (map)->keys; \
+        typeof((map)->values) _old_values = (map)->values; \
+        uint64_t* _old_hashes = (map)->hashes; \
+        (map)->keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \
+        (map)->values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \
+        (map)->hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \
+        (map)->capacity = _new_cap; \
+        (map)->count = 0; \
+        for (size_t _i = 0; _i < _old_cap; _i++) { \
+            if (_old_hashes[_i] != 0) { \
+                _HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \
+            } \
+        } \
+        mem_free(_old_keys); \
+        mem_free(_old_values); \
+        mem_free(_old_hashes); \
+    } while (0)
+
+// Internal: insert with a precomputed hash. Used by grow and the public INSERT macro.
+#define _HASHMAP_INSERT_INNER(map, k, v, h) \
+    do { \
+        size_t _cap = (map)->capacity; \
+        size_t _pos = (h) & (_cap - 1); \
+        typeof(*(map)->keys) _ik = (k); \
+        typeof(*(map)->values) _iv = (v); \
+        uint64_t _ih = (h); \
+        for (;;) { \
+            if ((map)->hashes[_pos] == 0) { \
+                (map)->keys[_pos] = _ik; \
+                (map)->values[_pos] = _iv; \
+                (map)->hashes[_pos] = _ih; \
+                (map)->count++; \
+                break; \
+            } \
+            size_t _existing_dist = _HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap); \
+            size_t _insert_dist = _HASHMAP_PROBE_DIST(_ih, _pos, _cap); \
+            if (_insert_dist > _existing_dist) { \
+                typeof(*(map)->keys) _tk = (map)->keys[_pos]; \
+                typeof(*(map)->values) _tv = (map)->values[_pos]; \
+                uint64_t _th = (map)->hashes[_pos]; \
+                (map)->keys[_pos] = _ik; \
+                (map)->values[_pos] = _iv; \
+                (map)->hashes[_pos] = _ih; \
+                _ik = _tk; \
+                _iv = _tv; \
+                _ih = _th; \
+            } \
+            _pos = (_pos + 1) & (_cap - 1); \
+        } \
+    } while (0)
+
+// Insert or update a key-value pair. Returns 0 or ZN_ERR_NO_MEMORY.
+#define hashmap_insert(map, key, val, hash_fn, eq_fn) \
+    ({ \
+        zn_status_t _status = 0; \
+        typeof(*(map)->keys) _k = (key); \
+        uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \
+        /* Check for existing key first. */ \
+        bool _found = false; \
+        if ((map)->capacity > 0) { \
+            size_t _cap = (map)->capacity; \
+            size_t _pos = _h & (_cap - 1); \
+            for (size_t _d = 0; _d < _cap; _d++) { \
+                if ((map)->hashes[_pos] == 0) \
+                    break; \
+                if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \
+                    break; \
+                if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \
+                    (map)->values[_pos] = (val); \
+                    _found = true; \
+                    break; \
+                } \
+                _pos = (_pos + 1) & (_cap - 1); \
+            } \
+        } \
+        if (!_found) { \
+            if ((map)->count * 4 >= (map)->capacity * 3 || (map)->capacity == 0) { \
+                _HASHMAP_GROW(map, hash_fn, eq_fn); \
+                if ((map)->keys == nullptr || (map)->values == nullptr || (map)->hashes == nullptr) { \
+                    _status = ZN_ERR_NO_MEMORY; \
+                } \
+            } \
+            if (_status == 0) { \
+                _HASHMAP_INSERT_INNER(map, _k, (val), _h); \
+            } \
+        } \
+        _status; \
+    })
+
+// Look up a key. Returns a pointer to the value, or nullptr if not found.
+#define hashmap_get(map, key, hash_fn, eq_fn) \
+    ({ \
+        typeof(*(map)->values)* _result = nullptr; \
+        if ((map)->capacity > 0) { \
+            typeof(*(map)->keys) _k = (key); \
+            uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \
+            size_t _cap = (map)->capacity; \
+            size_t _pos = _h & (_cap - 1); \
+            for (size_t _d = 0; _d < _cap; _d++) { \
+                if ((map)->hashes[_pos] == 0) \
+                    break; \
+                if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \
+                    break; \
+                if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \
+                    _result = &(map)->values[_pos]; \
+                    break; \
+                } \
+                _pos = (_pos + 1) & (_cap - 1); \
+            } \
+        } \
+        _result; \
+    })
+
+// Remove a key. Returns true if the key was found and removed.
+// Uses backward-shift deletion to avoid tombstones.
+#define hashmap_remove(map, key, hash_fn, eq_fn) \
+    ({ \
+        bool _removed = false; \
+        if ((map)->capacity > 0) { \
+            typeof(*(map)->keys) _k = (key); \
+            uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \
+            size_t _cap = (map)->capacity; \
+            size_t _pos = _h & (_cap - 1); \
+            for (size_t _d = 0; _d < _cap; _d++) { \
+                if ((map)->hashes[_pos] == 0) \
+                    break; \
+                if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \
+                    break; \
+                if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \
+                    (map)->hashes[_pos] = 0; \
+                    (map)->count--; \
+                    /* Backward-shift: move subsequent displaced entries back. */ \
+                    size_t _empty = _pos; \
+                    size_t _next = (_pos + 1) & (_cap - 1); \
+                    while ((map)->hashes[_next] != 0 && _HASHMAP_PROBE_DIST((map)->hashes[_next], _next, _cap) > 0) { \
+                        (map)->keys[_empty] = (map)->keys[_next]; \
+                        (map)->values[_empty] = (map)->values[_next]; \
+                        (map)->hashes[_empty] = (map)->hashes[_next]; \
+                        (map)->hashes[_next] = 0; \
+                        _empty = _next; \
+                        _next = (_next + 1) & (_cap - 1); \
+                    } \
+                    _removed = true; \
+                    break; \
+                } \
+                _pos = (_pos + 1) & (_cap - 1); \
+            } \
+        } \
+        _removed; \
+    })
+
+// Iterate over all occupied entries. `idx` is a size_t loop variable.
+// Access keys and values via (map)->keys[idx] and (map)->values[idx].
+#define hashmap_foreach(map, idx) \
+    for (size_t idx = 0; idx < (map)->capacity; idx++) \
+        if ((map)->hashes[idx] != 0)

kernel/include/kernel/init.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <kernel/common.h>
+#include <common/compiler.h>
-#include <kernel/compiler.h>
+#include <common/utils.h>
 #include <kernel/types.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -11,9 +11,9 @@
 #define __initdata_sorted(name) __used, __section(".init.data." name)
 
 struct boot_file {
-    uint8_t* data;
+    phys_t data;
     size_t length;
-    const char* path;
+    char path[128];
 };
 
 struct boot_info {
@@ -21,15 +21,18 @@ struct boot_info {
     struct phys_mem* mem_map;
     size_t num_mem_maps;
     phys_t phys_base;
-    virt_t virt_base;
+    uintptr_t virt_base;
-    virt_t hhdm_base;
+    uintptr_t hhdm_base;
     struct boot_file* files;
     size_t num_files;
+    phys_t rsdp;
 };
 
 extern uint8_t __ld_stack_top[];
 extern uint8_t __ld_stack_bottom[];
 
+extern phys_t rsdp_addr;
+
 // Entry point for the kernel after arch-specific setup has finished.
 void kernel_entry();
 
@@ -38,6 +41,3 @@ void kernel_early_init();
 
 [[noreturn]]
 void kernel_main(struct boot_info* info);
-
-[[noreturn]]
-void kernel_main_task();

kernel/include/kernel/irq.h

@@ -1,9 +1,7 @@
 #pragma once
 
-#include <bits/irq.h>
+#include <common/compiler.h>
-#include <kernel/compiler.h>
 #include <kernel/list.h>
-#include <kernel/vector.h>
 #include <stdint.h>
 
 enum irq_status : uint8_t {
@@ -23,7 +21,7 @@ enum irq_trigger_mode : uint8_t {
 
 struct irq_handler {
     enum irq_status (*handler)(void* self);
-    SLIST_LINK(struct irq_handler*) next;
+    list_link(struct irq_handler*) next;
 };
 
 struct irq_line {
@@ -32,7 +30,7 @@ struct irq_line {
     void (*unmask)(struct irq_line* self);
     void (*eoi)(struct irq_line* self);
 
-    SLIST_HEAD(struct irq_handler*) handlers;
+    list_head(struct irq_handler*) handlers;
     bool is_busy;
 };
 
@@ -40,20 +38,9 @@ void irq_line_attach(struct irq_line* self, struct irq_handler* handler);
 
 struct irq_percpu {
     uint32_t level;
+    bool in_interrupt;
 };
 
 void irq_lock();
-
 void irq_unlock();
-
+bool irq_set_interrupted(bool is_interrupted);
-// Enables or disables interrupts on this CPU.
-// Don't use this function directly.
-static inline void irq_set_state(bool state) {
-    return arch_irq_set_state(state);
-}
-
-// Returns true if interrupts are enabled on this CPU.
-// Don't use this function directly.
-static inline bool irq_get_state() {
-    return arch_irq_get_state();
-}

kernel/include/kernel/list.h

@@ -1,29 +1,29 @@
 #pragma once
 
-#define SLIST_HEAD(type) \
+#define list_head(type) \
     struct { \
-        type sl_head; \
+        type _sl_head; \
     }
 
-#define SLIST_LINK(type) \
+#define list_link(type) \
     struct { \
-        type sl_link; \
+        type _sl_link; \
     }
 
-#define SLIST_EMPTY(head)      ((head)->sl_head == NULL)
+#define list_is_empty(head)   ((head)->_sl_head == NULL)
-#define SLIST_FIRST(head)      ((head)->sl_head)
+#define list_first(head)      ((head)->_sl_head)
-#define SLIST_NEXT(elm, field) ((elm)->field.sl_link)
+#define list_next(elm, field) ((elm)->field._sl_link)
 
-#define SLIST_INSERT_END(slistelm, elm, field) \
+#define list_insert_end(slistelm, elm, field) \
     do { \
-        SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \
+        list_next((elm), field) = list_next((slistelm), field); \
-        SLIST_NEXT((slistelm), field) = (elm); \
+        list_next((slistelm), field) = (elm); \
     } while (0)
 
-#define SLIST_INSERT_HEAD(head, elm, field) \
+#define list_insert_head(head, elm, field) \
     do { \
-        SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \
+        list_next((elm), field) = list_first((head)); \
-        SLIST_FIRST((head)) = (elm); \
+        list_first((head)) = (elm); \
     } while (0)
 
-#define SLIST_FOREACH(var, head, field) for ((var) = SLIST_FIRST((head)); (var); (var) = SLIST_NEXT((var), field))
+#define list_foreach(var, head, field) for ((var) = list_first((head)); (var); (var) = list_next((var), field))

kernel/include/kernel/mem.h

@@ -1,180 +0,0 @@
-#pragma once
-
-#include <zinnia/status.h>
-#include <kernel/compiler.h>
-#include <kernel/spin.h>
-#include <kernel/types.h>
-#include <bits/mem.h>
-#include <stddef.h>
-
-ASSERT_TYPE(pte_t);
-
-enum pte_flags {
-    PTE_READ = 1 << 0,  // Can read from this page.
-    PTE_WRITE = 1 << 1, // Can write to this page.
-    PTE_EXEC = 1 << 2,  // Can execute code on this page.
-    PTE_USER = 1 << 3,  // Can be accessed by the user.
-    PTE_DIR = 1 << 4,   // Is a non-leaf page.
-};
-
-enum cache_mode {
-    CACHE_NONE,
-    CACHE_WRITE_COMBINE,
-    CACHE_WRITE_THROUGH,
-    CACHE_WRITE_BACK,
-    CACHE_MMIO,
-};
-
-struct page_table {
-    phys_t root;
-    struct spinlock lock;
-};
-
-enum alloc_flags {
-    ALLOC_NOZERO = 1 << 0, // Don't zero out the allocated memory.
-    ALLOC_MEM32 = 1 << 1,  // Allocated memory needs to fit inside 32 bits.
-    ALLOC_MEM20 = 1 << 2,  // Allocated memory needs to fit inside 20 bits.
-};
-
-enum phys_mem_usage {
-    PHYS_RESERVED,
-    PHYS_USABLE,
-    PHYS_RECLAIMABLE,
-};
-
-struct phys_mem {
-    phys_t address;
-    size_t length;
-    enum phys_mem_usage usage;
-};
-
-enum page_type {
-    PAGE_PHYS = 0, // Regular physical memory.
-};
-
-struct page {
-    enum page_type type;
-    int32_t flags;
-    size_t refcount;
-    union {
-        struct {
-            struct page* next; // Pointer to the next chunk.
-            size_t count;      // Amount of free pages.
-        } freelist;
-    };
-};
-static_assert(0x1000 % sizeof(struct page) == 0, "struct must be a multiple of the page size!");
-static_assert(sizeof(struct page) <= 64, "struct must be smaller than 64 bytes!");
-
-struct address_space {
-    // TODO
-};
-
-extern struct page_table mem_kernel_table;
-
-// Base address of the `struct page` array.
-extern struct page* mem_pfndb;
-
-// Base address of the HHDM.
-extern virt_t mem_hhdm_base;
-#define HHDM_PTR(paddr) (void*)((paddr) + mem_hhdm_base)
-
-void mem_init(struct phys_mem* map, size_t length, virt_t kernel_virt, phys_t kernel_phys, virt_t tmp_hhdm);
-
-void mem_phys_bootstrap(struct phys_mem* mem);
-void mem_phys_init(struct phys_mem* map, size_t length);
-
-// Allocates a region of memory which can be smaller than the page size.
-// Returns `nullptr` if the allocator cannot provide an allocation for the
-// given `length` + `flags`. Always returns `nullptr` if `length` is 0.
-zn_status_t mem_alloc(size_t length, enum alloc_flags flags, void** out);
-
-// Frees an allocation created by `mem_alloc`.
-// Passing `nullptr` is a no-op.
-zn_status_t mem_free(void* mem);
-
-// Allocates an amount of contiguous physical pages.
-zn_status_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out);
-
-// Frees an allocated region of physical pages.
-zn_status_t mem_phys_free(phys_t start, size_t num_pages);
-
-// Creates a new page table for the kernel.
-zn_status_t mem_pt_new_kernel(struct page_table* pt, enum alloc_flags flags);
-
-// Creates a new page table for a user process.
-zn_status_t mem_pt_new_user(struct page_table* pt, enum alloc_flags flags);
-
-// Sets a page table on the current processor.
-static inline void mem_pt_set(struct page_table* pt) {
-    arch_mem_pt_set(pt);
-}
-
-// Maps a single page to a virtual address in the given page table.
-zn_status_t mem_pt_map(struct page_table* pt, virt_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache);
-
-// Changes the protection of a page.
-zn_status_t mem_pt_protect(struct page_table* pt, virt_t vaddr, enum pte_flags flags);
-
-// Unmaps a page.
-zn_status_t mem_pt_unmap(struct page_table* pt, virt_t vaddr);
-
-// Base address used to access physical memory.
-static inline virt_t mem_hhdm_addr() {
-    return arch_mem_hhdm_addr();
-}
-
-// Base address used to access the page array.
-static inline virt_t mem_pfndb_addr() {
-    return arch_mem_pfndb_addr();
-}
-
-// Base address used to access kernel memory mappings.
-static inline virt_t mem_mapping_addr() {
-    return arch_mem_mapping_addr();
-}
-
-// Bit shift in a page.
-static inline size_t mem_page_bits() {
-    return arch_mem_page_bits();
-}
-
-// Gets the page size in bytes.
-static inline size_t mem_page_size() {
-    return 1 << mem_page_bits();
-}
-
-// The amount of bits in a level.
-static inline size_t mem_level_bits() {
-    return arch_mem_level_bits();
-}
-
-// The amount of levels in an address.
-static inline size_t mem_num_levels() {
-    return arch_mem_num_levels();
-}
-
-// Clears out a page table entry.
-static inline void mem_pte_clear(pte_t* pte) {
-    return arch_mem_pte_clear(pte);
-}
-
-// Builds a page table entry from the given info.
-static inline pte_t mem_pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
-    return arch_mem_pte_build(addr, flags, cache);
-}
-
-// Returns true if the given PTE is present and valid.
-static inline bool mem_pte_is_present(pte_t* pte) {
-    return arch_mem_pte_is_present(pte);
-}
-
-// Returns true if the given PTE contains another level.
-static inline bool mem_pte_is_dir(pte_t* pte) {
-    return arch_mem_pte_is_dir(pte);
-}
-
-// Returns the address component of the page table entry.
-static inline phys_t mem_pte_address(pte_t* pte) {
-    return arch_mem_pte_address(pte);
-}

kernel/include/kernel/mmio.h

@@ -0,0 +1,34 @@
+#pragma once
+#include <stdint.h>
+
+static inline uint8_t mmio_read8(void* ptr) {
+    return *((volatile uint8_t*)ptr);
+}
+
+static inline uint16_t mmio_read16(void* ptr) {
+    return *((volatile uint16_t*)ptr);
+}
+
+static inline uint32_t mmio_read32(void* ptr) {
+    return *((volatile uint32_t*)ptr);
+}
+
+static inline uint64_t mmio_read64(void* ptr) {
+    return *((volatile uint64_t*)ptr);
+}
+
+static inline void mmio_write8(void* ptr, uint8_t value) {
+    *((volatile uint8_t*)ptr) = value;
+}
+
+static inline void mmio_write16(void* ptr, uint16_t value) {
+    *((volatile uint16_t*)ptr) = value;
+}
+
+static inline void mmio_write32(void* ptr, uint32_t value) {
+    *((volatile uint32_t*)ptr) = value;
+}
+
+static inline void mmio_write64(void* ptr, uint64_t value) {
+    *((volatile uint64_t*)ptr) = value;
+}

kernel/include/kernel/mutex.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <kernel/sched.h>
+struct task;
 
 struct mutex {
     struct task* owner;

kernel/include/kernel/namespace.h

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <zinnia/handle.h>
+#include <kernel/channel.h>
+#include <kernel/hashmap.h>
+#include <kernel/mutex.h>
+#include <kernel/spin.h>
+#include <kernel/vas.h>
+#include <kernel/vmo.h>
+
+enum namespace_desc_type {
+    NAMESPACE_DESC_NS,
+    NAMESPACE_DESC_TASK,
+    NAMESPACE_DESC_VAS,
+    NAMESPACE_DESC_VMO,
+    NAMESPACE_DESC_CHANNEL,
+};
+
+struct namespace_desc {
+    enum namespace_desc_type type;
+    zn_rights_t rights;
+    union {
+        struct namespace* namespace;
+        struct task* task;
+        struct channel* channel;
+        struct vas* vas;
+        struct vmo* vmo;
+    } value;
+};
+
+// Translates between handles and objects.
+struct namespace {
+    zn_handle_t next_handle;
+    struct mutex mutex;
+    hashmap(zn_handle_t, struct namespace_desc) descriptors;
+};
+
+// Creates a new namespace with no contents.
+zn_status_t namespace_new(struct namespace** out);
+
+// Adds a descriptor to this namespace with an auto-allocated handle.
+zn_status_t namespace_add_desc(struct namespace* namespace, struct namespace_desc desc, zn_handle_t* out);
+
+// Adds a descriptor to this namespace at a specific handle value.
+zn_status_t namespace_add_desc_at(struct namespace* namespace, struct namespace_desc desc, zn_handle_t handle);
+
+// Gets the descriptor referenced by the given handle.
+zn_status_t namespace_get(struct namespace* namespace, zn_handle_t handle, struct namespace_desc* out);
+
+// Deletes a descriptor from this namespace.
+zn_status_t namespace_del_desc(struct namespace* namespace, zn_handle_t handle);
+
+// Clones a descriptor with optionally restricted rights.
+// If `cloned_rights` contains ZN_RIGHTS_IDENTICAL, the clone keeps the original rights.
+// Otherwise, `cloned_rights` must be a subset of the original descriptor's rights.
+zn_status_t namespace_clone_desc(
+    struct namespace* ns,
+    struct namespace_desc* desc,
+    zn_rights_t cloned_rights,
+    zn_handle_t* out
+);

kernel/include/kernel/pager.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <zinnia/status.h>
+#include <stdint.h>
+
+struct pager {
+    zn_status_t (*try_get_page)(struct pager* pager, uintptr_t offset);
+};

kernel/include/kernel/panic.h

@@ -1,5 +1,6 @@
 #pragma once
+#include <common/compiler.h>
 
 // Stop all execution upon panic.
-[[noreturn]]
+[[noreturn, __format(printf, 1, 2)]]
-void panic();
+void panic(const char* msg, ...);

kernel/include/kernel/percpu.h

@@ -1,9 +1,10 @@
 #pragma once
 
-#include <bits/percpu.h>
+#include <common/compiler.h>
-#include <kernel/compiler.h>
 #include <kernel/irq.h>
 #include <kernel/sched.h>
+#include <kernel/usercopy.h>
+#include <bits/percpu.h>
 #include <stddef.h>
 
 ASSERT_TYPE(struct arch_percpu);
@@ -13,8 +14,8 @@ struct percpu {
     struct percpu* self;       // A pointer to this structure.
     size_t id;                 // The virtual ID of this CPU.
     bool online;               // Whether this CPU is initialized and active.
-    virt_t kernel_stack;       // The kernel mode stack.
+    uintptr_t kernel_stack;    // The kernel mode stack.
-    virt_t user_stack;         // The user mode stack.
+    uintptr_t user_stack;      // The user mode stack.
     struct arch_percpu arch;   // Architecture-specific fields.
     struct irq_percpu irq;     // IRQ information.
     struct sched_percpu sched; // Scheduler information.
@@ -23,13 +24,14 @@ struct percpu {
 // Per-CPU data for the bootstrap processor.
 extern struct percpu percpu_bsp;
 
-// Gets the per-CPU data on the current CPU.
-static inline struct percpu* percpu_get() {
-    return arch_percpu_get();
-}
-
 // Allocates a block of memory for a new CPU.
 struct percpu* percpu_new();
 
+// Gets the per-CPU data on the current CPU.
+struct percpu* percpu_get();
+
 // Initializes the bootstrap processor.
-void percpu_bsp_early_init();
+void percpu_bsp_init();
+
+// Initializes all processors.
+void percpu_init();