refactor

kernel/arch/x86_64/apic.c

@@ -2,6 +2,9 @@
 #include <kernel/clock.h>
 #include <kernel/mmio.h>
 #include <kernel/percpu.h>
+#include <kernel/pmap.h>
+#include <kernel/vas.h>
+#include <kernel/virt.h>
 #include <assert.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
@@ -12,11 +15,11 @@ static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
     if (lapic->xapic_regs) {
         if (reg == APIC_REG_ICR) {
             uint64_t lo, hi;
-            lo = lapic->xapic_regs[APIC_REG_ICR / sizeof(uint32_t)];
-            hi = lapic->xapic_regs[APIC_REG_ICR_HI / sizeof(uint32_t)];
+            lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR);
+            hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI);
             return (hi << 32) | lo;
         } else {
-            return lapic->xapic_regs[reg / sizeof(uint32_t)];
+            return mmio_read32(lapic->xapic_regs + reg);
         }
     } else {
         return asm_rdmsr(0x800 + (reg >> 4));
@@ -26,10 +29,10 @@ static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
 static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
     if (lapic->xapic_regs) {
         if (reg == APIC_REG_ICR) {
-            lapic->xapic_regs[APIC_REG_ICR_HI / sizeof(uint32_t)] = value >> 32;
-            lapic->xapic_regs[APIC_REG_ICR / sizeof(uint32_t)] = value & 0xFFFF'FFFF;
+            mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32);
+            mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value);
         } else {
-            lapic->xapic_regs[reg / sizeof(uint32_t)] = value & 0xFFFF'FFFF;
+            mmio_write32(lapic->xapic_regs + reg, (uint32_t)value);
         }
     } else {
         asm_wrmsr(0x800 + (reg >> 4), value);
@@ -44,7 +47,9 @@ void lapic_init(struct local_apic* lapic) {
     if (cpuid.ecx & CPUID_1C_X2APIC)
         apic_msr |= 1 << 10;
     else {
-        lapic->xapic_regs = mmio_new(apic_msr & 0xFFFF'F000, 0x1000);
+        uintptr_t vaddr = vm_alloc(0x1000);
+        pmap_map(&kernel_vas.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        lapic->xapic_regs = (void*)vaddr;
         assert(lapic->xapic_regs, "Failed to allocate virtual memory!\n");
     }
 

kernel/arch/x86_64/hpet.c

@@ -1,8 +1,10 @@
 #include <common/utils.h>
 #include <kernel/clock.h>
 #include <kernel/cmdline.h>
-#include <kernel/mmio.h>
+#include <kernel/pmap.h>
 #include <kernel/print.h>
+#include <kernel/vas.h>
+#include <kernel/virt.h>
 #include <assert.h>
 #include <uacpi/acpi.h>
 #include <uacpi/status.h>
@@ -56,7 +58,11 @@ void hpet_init() {
     assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
 
     struct acpi_hpet* hpet = table.ptr;
-    hpet_clock.regs = mmio_new(hpet->address.address, 0x1000);
+
+    uintptr_t vaddr = vm_alloc(0x1000);
+    pmap_map(&kernel_vas.pmap, vaddr, hpet->address.address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+    hpet_clock.regs = (void*)vaddr;
+
     uacpi_table_unref(&table);
     kprintf("HPET regs at %p\n", hpet_clock.regs);
 

kernel/arch/x86_64/include/bits/mem.h

@@ -1,33 +0,0 @@
-#pragma once
-
-#include <kernel/types.h>
-#include <stddef.h>
-#include <stdint.h>
-
-static inline size_t arch_mem_page_size() {
-    return 1 << 12;
-}
-
-static inline size_t arch_mem_page_bits() {
-    return 12;
-}
-
-static inline size_t arch_mem_level_bits() {
-    return 9;
-}
-
-static inline size_t arch_mem_num_levels() {
-    return 4;
-}
-
-static inline uintptr_t arch_mem_hhdm_addr() {
-    return 0xFFFF'8000'0000'0000;
-}
-
-static inline uintptr_t arch_vm_pfndb_addr() {
-    return 0xFFFF'A000'0000'0000;
-}
-
-static inline uintptr_t arch_mem_mapping_addr() {
-    return 0xFFFF'C000'0000'0000;
-}

kernel/arch/x86_64/include/bits/percpu.h

@@ -6,12 +6,6 @@
 #include <x86_64/gdt.h>
 #include <x86_64/tss.h>
 
-static inline struct percpu* arch_percpu_get() {
-    struct percpu* result;
-    asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
-    return result;
-}
-
 struct arch_percpu {
     struct gdt gdt;
     struct tss tss;
@@ -22,6 +16,3 @@ struct arch_percpu {
     void (*fpu_restore)(const void*);
     bool can_smap;
 };
-
-void arch_percpu_bsp_init();
-void arch_percpu_init();

kernel/arch/x86_64/include/x86_64/apic.h

@@ -5,7 +5,7 @@
 struct local_apic {
     uint32_t id;
     uint32_t ticks_per_10ms;
-    volatile uint32_t* xapic_regs;
+    void* xapic_regs;
 };
 
 enum apic_delivery_mode {

kernel/arch/x86_64/irq.c

@@ -1,21 +1,19 @@
-#pragma once
-
 #include <stdint.h>
 #include <x86_64/defs.h>
 
-static inline bool arch_irq_get_state() {
+bool arch_irq_get_state() {
     uint64_t flags;
     asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
     return flags & RFLAGS_IF;
 }
 
-static inline void arch_irq_set_state(bool state) {
+void arch_irq_set_state(bool state) {
     if (state)
         asm volatile("sti");
     else
         asm volatile("cli");
 }
 
-static inline void arch_irq_wait() {
+void arch_irq_wait() {
     asm volatile("hlt");
 }

kernel/arch/x86_64/mem.c

@@ -0,0 +1,31 @@
+#include <kernel/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+size_t arch_mem_page_size() {
+    return 1 << 12;
+}
+
+size_t arch_mem_page_bits() {
+    return 12;
+}
+
+size_t arch_mem_level_bits() {
+    return 9;
+}
+
+size_t arch_mem_num_levels() {
+    return 4;
+}
+
+uintptr_t arch_mem_hhdm_addr() {
+    return 0xFFFF'8000'0000'0000;
+}
+
+uintptr_t arch_mem_pfndb_addr() {
+    return 0xFFFF'A000'0000'0000;
+}
+
+uintptr_t arch_mem_mapping_addr() {
+    return 0xFFFF'C000'0000'0000;
+}

kernel/arch/x86_64/meson.build

@@ -9,6 +9,8 @@ kernel_sources += files(
     'gdt.c',
     'hpet.c',
     'idt.c',
+    'irq.c',
+    'mem.c',
     'percpu.c',
     'pmap.c',
     'sched.c',

kernel/arch/x86_64/percpu.c

@@ -10,6 +10,12 @@
 #include <x86_64/syscall.h>
 #include <x86_64/tsc.h>
 
+struct percpu* arch_percpu_get() {
+    struct percpu* result;
+    asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
+    return result;
+}
+
 void arch_percpu_bsp_init() {
     asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
     asm_wrmsr(MSR_FS_BASE, 0);
@@ -33,9 +39,9 @@ union ia32_star {
 static_assert(sizeof(union ia32_star) == 8);
 
 static void setup_cpu(struct percpu* cpu) {
+    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
     asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
     asm_wrmsr(MSR_FS_BASE, 0);
-    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
 
     gdt_load();
     idt_load();
@@ -49,6 +55,8 @@ static void setup_cpu(struct percpu* cpu) {
     asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
     asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
 
+    // TODO: FRED
+
     uint64_t cr0, cr4;
     asm volatile("mov %0, cr0" : "=r"(cr0));
     asm volatile("mov %0, cr4" : "=r"(cr4));

kernel/arch/x86_64/sched.c

@@ -1,6 +1,7 @@
 #include <zinnia/status.h>
 #include <common/compiler.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/percpu.h>
 #include <kernel/sched.h>
 #include <kernel/virt.h>
@@ -53,8 +54,8 @@ static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) {
 
 void arch_sched_switch(struct task* from, struct task* to) {
     struct arch_percpu* cpu = &percpu_get()->arch;
-    struct arch_task_context* from_ctx = &from->context;
-    struct arch_task_context* to_ctx = &to->context;
+    struct arch_task_context* from_ctx = &from->task_ctx;
+    struct arch_task_context* to_ctx = &to->task_ctx;
 
     cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
 
@@ -119,7 +120,7 @@ zn_status_t arch_task_init(
     if (is_user) {
         // Allocate FPU space.
         const size_t fpu_size = cpu->fpu_size;
-        void* fpu_base = vm_alloc(fpu_size);
+        uintptr_t fpu_base = vm_alloc(fpu_size);
         if (!fpu_base)
             return ZN_ERR_NO_MEMORY;
         for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
@@ -130,9 +131,9 @@ zn_status_t arch_task_init(
                 return s;
             }
 
-            pmap_map(&kernel_vas.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
+            pmap_map(&kernel_vas.pmap, fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
         }
-        context->fpu_region = fpu_base;
+        context->fpu_region = (void*)fpu_base;
 
         context->ds = asm_read_ds();
         context->es = asm_read_es();

kernel/arch/x86_64/sys.c

@@ -1,6 +1,7 @@
 #include <zinnia/archctl.h>
 #include <zinnia/status.h>
 #include <common/compiler.h>
+#include <kernel/arch.h>
 #include <kernel/init.h>
 #include <kernel/percpu.h>
 #include <x86_64/apic.h>
@@ -9,7 +10,7 @@
 #include <x86_64/gdt.h>
 
 [[__init, __naked, __used]]
-void _start() {
+void arch_start() {
     asm volatile(
         "lea rsp, [rip + %0]\n"
         "jmp %1"
@@ -25,6 +26,10 @@ zn_status_t arch_archctl(zn_archctl_t op, uintptr_t arg) {
     case ZN_ARCHCTL_SET_FSBASE:
         asm_wrmsr(MSR_FS_BASE, arg);
         return 0;
+    case ZN_ARCHCTL_SET_GSBASE:
+        // Slightly misleading, but KERNEL_GS_BASE is the currently inactive register.
+        asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
+        return 0;
     default:
         return ZN_ERR_INVALID;
     }

kernel/arch/x86_64/usercopy.c

@@ -18,7 +18,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
         // Setup regs for `rep movsb`.
         "xchg rcx, rdx\n"
         "mov rax, [rip + %0]\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
 
         ".global x86_64_read_start\n"
         "x86_64_read_start:\n"
@@ -35,7 +35,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
         ".global x86_64_read_fault\n"
         "x86_64_read_fault:\n"
         "xor rax, rax\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
         "clac\n"
         "ret\n"
         :
@@ -61,7 +61,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
         // Setup regs for `rep movsb`.
         "xchg rcx, rdx\n"
         "mov rax, [rip + %0]\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
 
         ".global x86_64_write_start\n"
         "x86_64_write_start:\n"
@@ -78,7 +78,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
         ".global x86_64_write_fault\n"
         "x86_64_write_fault:\n"
         "xor rax, rax\n"
-        "mov [rdx], rax\n"
+        "xchg [rdx], rax\n"
         "clac\n"
         "ret\n"
         :
@@ -86,52 +86,3 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
         : "memory"
     );
 }
-
-extern void x86_64_strlen_start();
-extern void x86_64_strlen_end();
-extern void x86_64_strlen_fault();
-
-static const struct usercopy_region strlen_region = {
-    .start_ip = x86_64_strlen_start,
-    .end_ip = x86_64_strlen_end,
-    .fault_ip = x86_64_strlen_fault,
-};
-
-[[__naked]]
-bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
-    asm volatile(
-        "stac\n"
-        "mov rax, [rip + %0]\n"
-        "mov [rcx], rax\n"
-
-        ".global x86_64_strlen_start\n"
-        "x86_64_strlen_start:\n"
-        "xor r8, r8\n"
-        ".Lloop:\n"
-        "cmp byte ptr [rdi + r8], 0\n"
-        "je .Lleave\n"
-        "inc r8\n"
-        "cmp rsi, r8\n"
-        "jne .Lloop\n"
-        ".Lleave:\n"
-        "mov [rdx], r8\n"
-
-        ".global x86_64_strlen_end\n"
-        "x86_64_strlen_end:\n"
-        "xor rax, rax\n"
-        "mov [rcx], rax\n"
-        "mov rax, 1\n"
-        "clac\n"
-        "ret\n"
-
-        ".global x86_64_strlen_fault\n"
-        "x86_64_strlen_fault:\n"
-        "xor rax, rax\n"
-        "mov [rcx], rax\n"
-        "clac\n"
-        "ret\n"
-        :
-        : "i"(&strlen_region)
-        : "memory"
-    );
-}

kernel/boot/main.c

@@ -1,5 +1,6 @@
 #include <common/compiler.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/channel.h>
 #include <kernel/clock.h>
 #include <kernel/cmdline.h>

kernel/core/elf.c

@@ -3,6 +3,7 @@
 #include <common/elf.h>
 #include <common/utils.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/elf.h>
 #include <kernel/percpu.h>
 #include <kernel/pmap.h>

kernel/core/irq.c

@@ -1,3 +1,4 @@
+#include <kernel/arch.h>
 #include <kernel/irq.h>
 #include <kernel/percpu.h>
 #include <stdatomic.h>

kernel/core/panic.c

@@ -1,3 +1,4 @@
+#include <kernel/arch.h>
 #include <kernel/irq.h>
 #include <kernel/panic.h>
 #include <kernel/percpu.h>

kernel/core/percpu.c

@@ -1,4 +1,5 @@
 #include <common/compiler.h>
+#include <kernel/arch.h>
 #include <kernel/percpu.h>
 
 [[__used, __section(".percpu")]]
@@ -12,3 +13,15 @@ struct percpu* percpu_new() {
     // TODO
     return nullptr;
 }
+
+struct percpu* percpu_get() {
+    return arch_percpu_get();
+}
+
+void percpu_bsp_init() {
+    arch_percpu_bsp_init();
+}
+
+void percpu_init() {
+    arch_percpu_init();
+}

kernel/core/sched.c

@@ -1,5 +1,6 @@
 #include <zinnia/status.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/irq.h>
 #include <kernel/list.h>
 #include <kernel/percpu.h>
@@ -8,7 +9,6 @@
 #include <kernel/spin.h>
 #include <kernel/tailq.h>
 #include <assert.h>
-#include <bits/irq.h>
 #include <stdatomic.h>
 #include <stdint.h>
 #include <string.h>
@@ -125,7 +125,7 @@ zn_status_t task_create(
     new_task->user_stack = 0;
 
     // Initialize the arch-dependent parts.
-    zn_status_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
+    zn_status_t s = arch_task_init(&new_task->task_ctx, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
     if (s != ZN_OK)
         return s;
 

kernel/core/uacpi_glue.c

@@ -1,7 +1,11 @@
 #include <common/utils.h>
+#include <kernel/arch.h>
 #include <kernel/init.h>
 #include <kernel/mmio.h>
+#include <kernel/pmap.h>
 #include <kernel/print.h>
+#include <kernel/vas.h>
+#include <kernel/virt.h>
 #include <stdarg.h>
 #include <uacpi/kernel_api.h>
 #include <uacpi/status.h>
@@ -13,11 +17,15 @@ uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) {
 }
 
 void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
-    return mmio_new(addr, len);
+    uintptr_t vaddr = vm_alloc(len);
+    for (size_t i = 0; i < len; i += arch_mem_page_size()) {
+        pmap_map(&kernel_vas.pmap, vaddr, addr, PTE_READ | PTE_WRITE, CACHE_MMIO);
+    }
+    return (void*)vaddr;
 }
 
 void uacpi_kernel_unmap(void* addr, uacpi_size len) {
-    mmio_free(addr, len);
+    // TODO
 }
 
 void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {

kernel/include/kernel/alloc.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <zinnia/status.h>
-#include <bits/mem.h>
+#include <kernel/types.h>
 #include <stddef.h>
 
 enum alloc_flags {
@@ -44,14 +44,4 @@ void slab_init();
 void mem_phys_bootstrap(struct phys_mem* mem);
 void mem_phys_init(struct phys_mem* map, size_t length);
 
-uintptr_t arch_mem_hhdm_addr();
-uintptr_t arch_vm_pfndb_addr();
-uintptr_t arch_mem_mapping_addr();
-size_t arch_mem_page_bits();
-size_t arch_mem_page_size();
-size_t arch_mem_level_bits();
-size_t arch_mem_num_levels();
-
-static inline size_t mem_high_shift() {
-    return arch_mem_level_bits() * arch_mem_num_levels() + arch_mem_page_bits();
-}
+size_t mem_high_shift();

kernel/include/kernel/arch.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+void arch_start();
+void arch_panic();
+
+size_t arch_mem_page_size();
+size_t arch_mem_page_bits();
+size_t arch_mem_level_bits();
+size_t arch_mem_num_levels();
+uintptr_t arch_mem_hhdm_addr();
+uintptr_t arch_mem_pfndb_addr();
+uintptr_t arch_mem_mapping_addr();
+
+#include <kernel/usercopy.h>
+
+bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region);
+bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region);
+
+#include <kernel/irq.h>
+
+void arch_irq_set_state(bool state);
+bool arch_irq_get_state();
+void arch_irq_wait();
+
+#include <kernel/percpu.h>
+
+struct percpu* arch_percpu_get();
+void arch_percpu_bsp_init();
+void arch_percpu_init();

kernel/include/kernel/bits/usercopy.h

@@ -1,7 +0,0 @@
-#pragma once
-
-#include <kernel/usercopy.h>
-
-bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region);
-bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region);
-bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region);

kernel/include/kernel/irq.h

@@ -2,7 +2,6 @@
 
 #include <common/compiler.h>
 #include <kernel/list.h>
-#include <bits/irq.h>
 #include <stdint.h>
 
 enum irq_status : uint8_t {
@@ -45,7 +44,3 @@ struct irq_percpu {
 void irq_lock();
 void irq_unlock();
 bool irq_set_interrupted(bool is_interrupted);
-
-void arch_irq_set_state(bool state);
-bool arch_irq_get_state();
-void arch_irq_wait();

kernel/include/kernel/mmio.h

@@ -1,7 +1,34 @@
 #pragma once
+#include <stdint.h>
 
-#include <kernel/types.h>
-#include <stddef.h>
+static inline uint8_t mmio_read8(void* ptr) {
+    return *((volatile uint8_t*)ptr);
+}
 
-void* mmio_new(phys_t addr, size_t length);
-void mmio_free(void* ptr, size_t length);
+static inline uint16_t mmio_read16(void* ptr) {
+    return *((volatile uint16_t*)ptr);
+}
+
+static inline uint32_t mmio_read32(void* ptr) {
+    return *((volatile uint32_t*)ptr);
+}
+
+static inline uint64_t mmio_read64(void* ptr) {
+    return *((volatile uint64_t*)ptr);
+}
+
+static inline void mmio_write8(void* ptr, uint8_t value) {
+    *((volatile uint8_t*)ptr) = value;
+}
+
+static inline void mmio_write16(void* ptr, uint16_t value) {
+    *((volatile uint16_t*)ptr) = value;
+}
+
+static inline void mmio_write32(void* ptr, uint32_t value) {
+    *((volatile uint32_t*)ptr) = value;
+}
+
+static inline void mmio_write64(void* ptr, uint64_t value) {
+    *((volatile uint64_t*)ptr) = value;
+}

kernel/include/kernel/percpu.h

@@ -28,16 +28,10 @@ extern struct percpu percpu_bsp;
 struct percpu* percpu_new();
 
 // Gets the per-CPU data on the current CPU.
-static inline struct percpu* percpu_get() {
-    return arch_percpu_get();
-}
+struct percpu* percpu_get();
 
 // Initializes the bootstrap processor.
-static inline void percpu_bsp_init() {
-    arch_percpu_bsp_init();
-}
+void percpu_bsp_init();
 
 // Initializes all processors.
-static inline void percpu_init() {
-    arch_percpu_init();
-}
+void percpu_init();

kernel/include/kernel/sched.h

@@ -11,9 +11,6 @@
 #include <stddef.h>
 #include <stdint.h>
 
-ASSERT_TYPE(struct arch_context);
-ASSERT_TYPE(struct arch_task_context);
-
 enum task_state {
     TASK_STATE_RUNNING, // Task is active and running.
     TASK_STATE_READY,   // Task is active, but not currently scheduled.
@@ -26,19 +23,20 @@ enum task_state {
 
 // A task is the smallest unit of the scheduler.
 struct task {
+    tailq_link(struct task) next;
+
     size_t id;
     char name[TASK_NAME_MAX];
     struct vas* space;
     struct namespace* namespace;
     enum task_state state;
-    struct arch_task_context context;
+    struct arch_task_context task_ctx;
+    struct arch_context ctx;
     struct usercopy_region* usercopy_region;
     uintptr_t kernel_stack;
     uintptr_t user_stack;
     size_t time_slice;
     int8_t priority;
-
-    tailq_link(struct task) next;
 };
 
 // Per-CPU data for scheduling.

kernel/include/kernel/usercopy.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <common/compiler.h>
-#include <kernel/percpu.h>
 #include <stddef.h>
 
 struct usercopy_region {
@@ -15,6 +14,3 @@ bool usercopy_read(void* dst, const __user void* src, size_t len);
 
 // Copies a block of data from kernel to user memory.
 bool usercopy_write(__user void* dst, const void* src, size_t len);
-
-// Performs a strlen() on a user string.
-bool usercopy_strlen(const __user char* str, size_t max, size_t* len);

kernel/include/kernel/vas.h

@@ -8,7 +8,6 @@
 
 // Virtual address space.
 struct vas {
-    // Architecture-specific virtual memory map implementation.
     struct pmap pmap;
 };
 

kernel/include/kernel/vdso.h

@@ -22,8 +22,6 @@ struct vdso_kernel_data {
 
 // Export a function under the current API version.
 #define VDSO_SYMBOL(fn) asm(".symver " #fn ",__vdso_" #fn "@@ZINNIA_0")
-// Export a function as a symbol for an older API version.
-#define VDSO_SYMBOL_LEGACY(fn, symbol, ver) asm(".symver " #fn ",__vdso_" #symbol "@ZINNIA_" ver)
 
 extern const volatile struct vdso_kernel_data vdso_data;
 

kernel/include/kernel/virt.h

@@ -21,8 +21,8 @@ struct page {
 static_assert(0x1000 % sizeof(struct page) == 0, "struct must fit cleanly inside a page!");
 static_assert(sizeof(struct page) <= 64, "struct must be smaller than 64 bytes!");
 
-void* vm_alloc(size_t length);
-void vm_free(void* addr, size_t length);
+uintptr_t vm_alloc(size_t length);
+void vm_free(uintptr_t addr, size_t length);
 
 // Base address of the `struct page` array.
 extern struct page* vm_pfndb;

kernel/kernel.ld

@@ -1,4 +1,4 @@
-ENTRY(_start)
+ENTRY(arch_start)
 
 PHDRS {
     text    PT_LOAD     FLAGS(5);

kernel/mem/init.c

@@ -2,6 +2,7 @@
 
 #include <common/utils.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/init.h>
 #include <kernel/panic.h>
 #include <kernel/print.h>
@@ -141,7 +142,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
 
         const size_t num_pages = (map[i].length + pgsz - 1) / pgsz;
         const size_t metadata_size = num_pages * sizeof(struct page);
-        const uintptr_t metadata_start_vaddr = arch_vm_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
+        const uintptr_t metadata_start_vaddr = arch_mem_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
         const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size;
 
         // Find the page-aligned boundaries that contain this metadata.
@@ -156,7 +157,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
             pmap_map(&kernel_vas.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_NONE);
         }
     }
-    vm_pfndb = (struct page*)arch_vm_pfndb_addr();
+    vm_pfndb = (struct page*)arch_mem_pfndb_addr();
 
     // We don't need the bootstrap allocator from this point on.
     // Initialize the real page allocator.
@@ -164,5 +165,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
 
     slab_init();
 
+    // TODO: Free `__init` memory.
+
     kprintf("Memory initialization complete\n");
 }

kernel/mem/meson.build

@@ -1,6 +1,5 @@
 kernel_sources += files(
     'init.c',
-    'mmio.c',
     'pmap.c',
     'pmm.c',
     'slab.c',

kernel/mem/mmio.c

@@ -1,24 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-#include <kernel/mmio.h>
-#include <kernel/pmap.h>
-#include <kernel/vas.h>
-#include <kernel/virt.h>
-#include <bits/mem.h>
-
-void* mmio_new(phys_t addr, size_t length) {
-    void* ptr = vm_alloc(length);
-    for (size_t i = 0; i < length; i += arch_mem_page_size()) {
-
-        if (pmap_map(&kernel_vas.pmap, (uintptr_t)ptr + i, addr + i, PTE_READ | PTE_WRITE, CACHE_MMIO)) {
-            vm_free(ptr, length);
-            return nullptr;
-        }
-    }
-
-    return ptr;
-}
-
-void mmio_free(void* ptr, size_t length) {
-    // TODO
-}

kernel/mem/pmap.c

@@ -2,6 +2,7 @@
 
 #include <zinnia/status.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/pmap.h>
 #include <kernel/spin.h>
 #include <kernel/vas.h>

kernel/mem/pmm.c

@@ -2,6 +2,7 @@
 
 #include <common/compiler.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/panic.h>
 #include <kernel/print.h>
 #include <kernel/spin.h>

kernel/mem/slab.c

@@ -3,6 +3,7 @@
 #include <zinnia/status.h>
 #include <common/compiler.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/spin.h>
 #include <string.h>
 

kernel/mem/usercopy.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 
-#include <kernel/bits/usercopy.h>
+#include <kernel/arch.h>
 #include <kernel/percpu.h>
 #include <kernel/usercopy.h>
 
@@ -11,7 +11,3 @@ bool usercopy_read(void* dst, const __user void* src, size_t len) {
 bool usercopy_write(__user void* dst, const void* src, size_t len) {
     return arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
 }
-
-bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
-    return arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
-}

kernel/mem/vas.c

@@ -3,6 +3,7 @@
 #include <zinnia/status.h>
 #include <common/utils.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/vas.h>
 #include <assert.h>
 

kernel/mem/virt.c

@@ -2,6 +2,7 @@
 
 #include <common/utils.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/virt.h>
 #include <stdatomic.h>
 #include <stddef.h>
@@ -12,17 +13,21 @@ struct page* vm_pfndb = nullptr;
 
 static size_t mapping_offset = 0;
 
-void* vm_alloc(size_t length) {
+uintptr_t vm_alloc(size_t length) {
     uintptr_t base = arch_mem_mapping_addr();
     size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()));
 
-    return (void*)(base + offset);
+    return (base + offset);
 }
 
-void vm_free(void* ptr, size_t length) {
+void vm_free(uintptr_t ptr, size_t length) {
     // TODO
 }
 
 uintptr_t vm_page_addr(struct page* page) {
     return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
 }
+
+size_t mem_high_shift() {
+    return arch_mem_level_bits() * arch_mem_num_levels() + arch_mem_page_bits();
+}

kernel/mem/vmo.c

@@ -3,6 +3,7 @@
 #include <zinnia/status.h>
 #include <common/utils.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/list.h>
 #include <kernel/virt.h>
 #include <kernel/vmo.h>

kernel/syscalls/system.c

@@ -2,6 +2,7 @@
 
 #include <zinnia/status.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/clock.h>
 #include <kernel/console.h>
 #include <kernel/syscalls.h>

kernel/syscalls/thread.c

@@ -101,7 +101,7 @@ zn_status_t syscall_thread_start(struct arch_context* ctx) {
 
     // Re-initialize the task's arch context to enter user mode via sched_to_user_arg.
     status =
-        arch_task_init(&task->context, sched_to_user, (uintptr_t)info, task->kernel_stack + KERNEL_STACK_SIZE, true);
+        arch_task_init(&task->task_ctx, sched_to_user, (uintptr_t)info, task->kernel_stack + KERNEL_STACK_SIZE, true);
     if (status != ZN_OK) {
         mem_free(info);
         return status;

kernel/syscalls/vmo.c

@@ -4,6 +4,7 @@
 #include <zinnia/status.h>
 #include <common/utils.h>
 #include <kernel/alloc.h>
+#include <kernel/arch.h>
 #include <kernel/namespace.h>
 #include <kernel/percpu.h>
 #include <kernel/print.h>