Initial commit

kernel/arch/x86_64/gdt.c

@@ -0,0 +1,51 @@
+#include <kernel/percpu.h>
+#include <kernel/syscalls.h>
+#include <asm.h>
+#include <defs.h>
+#include <gdt.h>
+#include <stddef.h>
+#include <string.h>
+
+struct gdt main_gdt = {
+    .null = 0,
+    .kernel_code32 = 0x00cf9b000000ffff,
+    .kernel_data32 = 0x00cf93000000ffff,
+    .kernel_code64 = 0x00a09b0000000000,
+    .kernel_data64 = 0x0000930000000000,
+    .user_code = 0x00cffb000000ffff,
+    .user_data = 0x0000f30000000000,
+    .user_code64 = 0x0000fb0000000000,
+    .tss = {0x0000890000000000, 0},
+};
+
+void gdt_init() {
+    // Set initial values.
+    memcpy(&percpu_get()->arch.gdt, &main_gdt, sizeof(struct gdt));
+
+    struct gdtr gdtr = {
+        .limit = sizeof(struct gdt) - 1,
+        .base = &percpu_get()->arch.gdt,
+    };
+
+    asm volatile("lgdt [%0]" ::"r"(&gdtr));
+
+    // Save the contents of MSR_GS_BASE, as they get cleared by a write to `gs`.
+    virt_t gs = asm_rdmsr(MSR_GS_BASE);
+    asm volatile(
+        "push %0\n"
+        "lea rax, [rip + 1f]\n"
+        "push rax\n"
+        "retfq\n"
+        "1:\n"
+        "mov ax, %1\n"
+        "mov ds, ax\n"
+        "mov es, ax\n"
+        "mov fs, ax\n"
+        "mov gs, ax\n"
+        "mov ss, ax\n"
+        :
+        : "i"(offsetof(struct gdt, kernel_code64)), "i"(offsetof(struct gdt, kernel_data64))
+        : "rax", "memory"
+    );
+    asm_wrmsr(MSR_GS_BASE, gs);
+}

kernel/arch/x86_64/idt.c

@@ -0,0 +1,7 @@
+#include <kernel/compiler.h>
+#include <stdint.h>
+
+struct [[__packed]] idtr {
+    uint16_t limit;
+    struct idt* base;
+};

kernel/arch/x86_64/include/asm.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <stdint.h>
+
+static inline uint64_t asm_rdmsr(uint32_t msr) {
+    uint32_t eax;
+    uint32_t edx;
+    asm volatile("rdmsr" : "=a"(eax), "=d"(edx) : "c"(msr) : "memory");
+    return ((uint64_t)edx << 32) | eax;
+}
+
+// Writes a 64-bit value to a given MSR.
+static inline void asm_wrmsr(uint32_t msr, uint64_t val) {
+    uint32_t eax = (uint32_t)val;
+    uint32_t edx = val >> 32;
+    asm volatile("wrmsr" : : "a"(eax), "d"(edx), "c"(msr) : "memory");
+}
+
+// Writes a 64-bit value to a control register using XSETBV.
+static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
+    uint32_t eax = val;
+    uint32_t edx = val >> 32;
+    asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory");
+}
+
+// Saves the FPU state to a 512-byte region of memory using FXSAVE.
+// Pointer must be 16-byte aligned.
+static inline void asm_fxsave(void* mem) {
+    asm volatile("fxsave %0" : "+m"(*(uint8_t*)mem)::"memory");
+}
+
+// Restores the FPU state from a 512-byte region of memory using FXRSTOR.
+// Pointer must be 16-byte aligned.
+static inline void asm_fxrstor(void* mem) {
+    asm volatile("fxrstor %0" ::"m"(*(uint8_t*)mem) : "memory");
+}
+
+// Saves the FPU state to a region of memory using XSAVE.
+// Pointer must be 16-byte aligned.
+static inline void asm_xsave(void* mem) {
+    asm volatile("xsave %0" : "+m"(*(uint8_t*)mem)::"memory");
+}
+
+// Restores the FPU state from a region of memory using XRSTOR.
+// Pointer must be 16-byte aligned.
+static inline void asm_xrstor(void* mem) {
+    asm volatile("xrstor %0" ::"m"(*(uint8_t*)mem) : "memory");
+}
+
+static inline uint8_t asm_inb(uint16_t port) {
+    uint8_t result;
+    asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
+    return result;
+}
+
+static inline uint16_t asm_inw(uint16_t port) {
+    uint16_t result;
+    asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
+    return result;
+}
+
+static inline uint32_t asm_inl(uint16_t port) {
+    uint32_t result;
+    asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
+    return result;
+}
+
+static inline void asm_outb(uint16_t port, uint8_t value) {
+    asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
+}
+
+static inline void asm_outw(uint16_t port, uint16_t value) {
+    asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
+}
+
+static inline void asm_outl(uint16_t port, uint32_t value) {
+    asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
+}

kernel/arch/x86_64/include/bits/elf.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#define ELF_ARCH_CLASS   ELFCLASS64
+#define ELF_ARCH_DATA    ELFDATA2LSB
+#define ELF_ARCH_MACHINE EM_X86_64
+
+#define elf_hdr  elf64_hdr
+#define elf_phdr elf64_phdr
+#define elf_dyn  elf64_dyn
+#define elf_addr elf64_addr
+#define elf_off  elf64_off
+#define elf_nhdr elf64_nhdr
+#define elf_auxv elf64_auxv

kernel/arch/x86_64/include/bits/irq.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <defs.h>
+#include <stdint.h>
+
+static inline bool arch_irq_get_state() {
+    uint64_t flags;
+    asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
+    return flags & RFLAGS_IF;
+}
+
+static inline void arch_irq_set_state(bool state) {
+    if (state)
+        asm volatile("sti");
+    else
+        asm volatile("cli");
+}

kernel/arch/x86_64/include/bits/mem.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <kernel/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+static inline size_t arch_mem_page_bits() {
+    return 12;
+}
+
+static inline size_t arch_mem_level_bits() {
+    return 9;
+}
+
+static inline size_t arch_mem_num_levels() {
+    return 4;
+}
+
+static inline virt_t arch_mem_hhdm_addr() {
+    return 0xFFFF'8000'0000'0000;
+}
+
+static inline virt_t arch_mem_pfndb_addr() {
+    return 0xFFFF'A000'0000'0000;
+}
+
+static inline virt_t arch_mem_mapping_addr() {
+    return 0xFFFF'C000'0000'0000;
+}
+
+typedef uint64_t pte_t;
+enum pte_flags;
+enum cache_mode;
+struct page_table;
+
+void arch_mem_pte_clear(pte_t* pte);
+pte_t arch_mem_pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
+bool arch_mem_pte_is_present(pte_t* pte);
+bool arch_mem_pte_is_dir(pte_t* pte);
+phys_t arch_mem_pte_address(pte_t* pte);
+void arch_mem_pt_set(struct page_table* pt);

kernel/arch/x86_64/include/bits/percpu.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <gdt.h>
+#include <stdint.h>
+
+static inline struct percpu* arch_percpu_get() {
+    struct percpu* result;
+    asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
+    return result;
+}
+
+struct arch_percpu {
+    uint32_t lapic_id;
+    struct gdt gdt;
+    struct tss tss;
+};

kernel/arch/x86_64/include/bits/sched.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <stdint.h>
+
+struct arch_task_context {
+    uint64_t rsp;
+    uint8_t* fpu_region;
+    uint16_t ds, es, fs, gs;
+    uint64_t fs_base, gs_base;
+};
+
+struct arch_context {
+    uint64_t r15;
+    uint64_t r14;
+    uint64_t r13;
+    uint64_t r12;
+    uint64_t r11;
+    uint64_t r10;
+    uint64_t r9;
+    uint64_t r8;
+    uint64_t rsi;
+    uint64_t rdi;
+    uint64_t rbp;
+    uint64_t rdx;
+    uint64_t rcx;
+    uint64_t rbx;
+    uint64_t rax;
+    uint64_t isr;   // Pushed onto the stack by the interrupt handler stubs.
+    uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code.
+    uint64_t rip;   // The rest is pushed onto the stack by the CPU during an interrupt.
+    uint64_t cs;
+    uint64_t rflags;
+    uint64_t rsp;
+    uint64_t ss;
+};

kernel/arch/x86_64/include/bits/types.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stdint.h>
+
+typedef uint64_t phys_t;
+typedef uint64_t virt_t;
+typedef uint64_t reg_t;

kernel/arch/x86_64/include/bits/usercopy.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <kernel/compiler.h>
+#include <stddef.h>
+#include <stdint.h>
+
+bool arch_usercopy_read(uint8_t* dst, const __user uint8_t* src, size_t len);
+bool arch_usercopy_write(__user uint8_t* dst, const uint8_t* src, size_t len);
+bool arch_usercopy_strlen(const __user uint8_t* str, size_t max, size_t* len);

kernel/arch/x86_64/include/defs.h

@@ -0,0 +1,191 @@
+#pragma once
+
+#define CPUID_1C_SSE3       (1 << 0)
+#define CPUID_1C_PCLMUL     (1 << 1)
+#define CPUID_1C_DTES64     (1 << 2)
+#define CPUID_1C_MONITOR    (1 << 3)
+#define CPUID_1C_DS_CPL     (1 << 4)
+#define CPUID_1C_VMX        (1 << 5)
+#define CPUID_1C_SMX        (1 << 6)
+#define CPUID_1C_EST        (1 << 7)
+#define CPUID_1C_TM2        (1 << 8)
+#define CPUID_1C_SSSE3      (1 << 9)
+#define CPUID_1C_CID        (1 << 10)
+#define CPUID_1C_SDBG       (1 << 11)
+#define CPUID_1C_FMA        (1 << 12)
+#define CPUID_1C_CX16       (1 << 13)
+#define CPUID_1C_XTPR       (1 << 14)
+#define CPUID_1C_PDCM       (1 << 15)
+#define CPUID_1C_PCID       (1 << 17)
+#define CPUID_1C_DCA        (1 << 18)
+#define CPUID_1C_SSE4_1     (1 << 19)
+#define CPUID_1C_SSE4_2     (1 << 20)
+#define CPUID_1C_X2APIC     (1 << 21)
+#define CPUID_1C_MOVBE      (1 << 22)
+#define CPUID_1C_POPCNT     (1 << 23)
+#define CPUID_1C_TSC        (1 << 24)
+#define CPUID_1C_AES        (1 << 25)
+#define CPUID_1C_XSAVE      (1 << 26)
+#define CPUID_1C_OSXSAVE    (1 << 27)
+#define CPUID_1C_AVX        (1 << 28)
+#define CPUID_1C_F16C       (1 << 29)
+#define CPUID_1C_RDRAND     (1 << 30)
+#define CPUID_1C_HYPERVISOR (1 << 31)
+
+#define CPUID_1D_FPU     (1 << 0)
+#define CPUID_1D_VME     (1 << 1)
+#define CPUID_1D_DE      (1 << 2)
+#define CPUID_1D_PSE     (1 << 3)
+#define CPUID_1D_TSC     (1 << 4)
+#define CPUID_1D_MSR     (1 << 5)
+#define CPUID_1D_PAE     (1 << 6)
+#define CPUID_1D_MCE     (1 << 7)
+#define CPUID_1D_CX8     (1 << 8)
+#define CPUID_1D_APIC    (1 << 9)
+#define CPUID_1D_SEP     (1 << 11)
+#define CPUID_1D_MTRR    (1 << 12)
+#define CPUID_1D_PGE     (1 << 13)
+#define CPUID_1D_MCA     (1 << 14)
+#define CPUID_1D_CMOV    (1 << 15)
+#define CPUID_1D_PAT     (1 << 16)
+#define CPUID_1D_PSE36   (1 << 17)
+#define CPUID_1D_PSN     (1 << 18)
+#define CPUID_1D_CLFLUSH (1 << 19)
+#define CPUID_1D_DS      (1 << 21)
+#define CPUID_1D_ACPI    (1 << 22)
+#define CPUID_1D_MMX     (1 << 23)
+#define CPUID_1D_FXSR    (1 << 24)
+#define CPUID_1D_SSE     (1 << 25)
+#define CPUID_1D_SSE2    (1 << 26)
+#define CPUID_1D_SS      (1 << 27)
+#define CPUID_1D_HTT     (1 << 28)
+#define CPUID_1D_TM      (1 << 29)
+#define CPUID_1D_IA64    (1 << 30)
+#define CPUID_1D_PBE     (1 << 31)
+
+#define CPUID_7B_FSGSBASE   (1 << 0)
+#define CPUID_7B_SGX        (1 << 2)
+#define CPUID_7B_BMI        (1 << 3)
+#define CPUID_7B_HLE        (1 << 4)
+#define CPUID_7B_AVX2       (1 << 5)
+#define CPUID_7B_SMEP       (1 << 7)
+#define CPUID_7B_BMI2       (1 << 8)
+#define CPUID_7B_RTM        (1 << 11)
+#define CPUID_7B_AVX512F    (1 << 16)
+#define CPUID_7B_AVX512DQ   (1 << 17)
+#define CPUID_7B_RDSEED     (1 << 18)
+#define CPUID_7B_ADX        (1 << 19)
+#define CPUID_7B_SMAP       (1 << 20)
+#define CPUID_7B_AVX512IFMA (1 << 21)
+#define CPUID_7B_CLFLUSHOPT (1 << 23)
+#define CPUID_7B_CLWB       (1 << 24)
+#define CPUID_7B_AVX512CD   (1 << 28)
+#define CPUID_7B_SHA        (1 << 29)
+#define CPUID_7B_AVX512BW   (1 << 30)
+#define CPUID_7B_AVX512VL   (1 << 31)
+
+#define CPUID_7C_AVX512VBMI      (1 << 1)
+#define CPUID_7C_UMIP            (1 << 2)
+#define CPUID_7C_PKU             (1 << 3)
+#define CPUID_7C_OSPKE           (1 << 4)
+#define CPUID_7C_WAITPKG         (1 << 5)
+#define CPUID_7C_AVX512VBMI2     (1 << 6)
+#define CPUID_7C_SHSTK           (1 << 7)
+#define CPUID_7C_GFNI            (1 << 8)
+#define CPUID_7C_VAES            (1 << 9)
+#define CPUID_7C_VPCLMULQDQ      (1 << 10)
+#define CPUID_7C_AVX512VNNI      (1 << 11)
+#define CPUID_7C_AVX512BITALG    (1 << 12)
+#define CPUID_7C_TME_EN          (1 << 13)
+#define CPUID_7C_AVX512VPOPCNTDQ (1 << 14)
+#define CPUID_7C_RDPID           (1 << 22)
+#define CPUID_7C_KL              (1 << 23)
+#define CPUID_7C_CLDEMOTE        (1 << 25)
+#define CPUID_7C_MOVDIRI         (1 << 27)
+#define CPUID_7C_MOVDIR64B       (1 << 28)
+#define CPUID_7C_ENQCMD          (1 << 29)
+
+#define CPUID_7D_UINTR              (1 << 5)
+#define CPUID_7D_AVX512VP2INTERSECT (1 << 8)
+#define CPUID_7D_SERIALIZE          (1 << 14)
+#define CPUID_7D_TSXLDTRK           (1 << 16)
+#define CPUID_7D_PCONFIG            (1 << 18)
+#define CPUID_7D_IBT                (1 << 20)
+#define CPUID_7D_AMX_BF16           (1 << 22)
+#define CPUID_7D_AVX512FP16         (1 << 23)
+#define CPUID_7D_AMX_TILE           (1 << 24)
+#define CPUID_7D_AMX_INT8           (1 << 25)
+
+#define RFLAGS_CF   (1u << 0)
+#define RFLAGS_PF   (1u << 2)
+#define RFLAGS_AF   (1u << 4)
+#define RFLAGS_ZF   (1u << 6)
+#define RFLAGS_SF   (1u << 7)
+#define RFLAGS_TF   (1u << 8)
+#define RFLAGS_IF   (1u << 9)
+#define RFLAGS_DF   (1u << 10)
+#define RFLAGS_OF   (1u << 11)
+#define RFLAGS_IOPL (3u << 12)
+#define RFLAGS_NT   (1u << 14)
+#define RFLAGS_RF   (1u << 16)
+#define RFLAGS_VM   (1u << 17)
+#define RFLAGS_AC   (1u << 18)
+#define RFLAGS_VIF  (1u << 19)
+#define RFLAGS_VIP  (1u << 20)
+#define RFLAGS_ID   (1u << 21)
+
+#define CR0_PE (1 << 0)
+#define CR0_MP (1 << 1)
+#define CR0_EM (1 << 2)
+#define CR0_TS (1 << 3)
+#define CR0_ET (1 << 4)
+#define CR0_NE (1 << 5)
+#define CR0_WP (1 << 16)
+#define CR0_AM (1 << 18)
+#define CR0_NW (1 << 29)
+#define CR0_CD (1 << 30)
+#define CR0_PG (1 << 31)
+
+#define CR4_VME        (1 << 0)
+#define CR4_PVI        (1 << 1)
+#define CR4_TSD        (1 << 2)
+#define CR4_DE         (1 << 3)
+#define CR4_PSE        (1 << 4)
+#define CR4_PAE        (1 << 5)
+#define CR4_MCE        (1 << 6)
+#define CR4_PGE        (1 << 7)
+#define CR4_PCE        (1 << 8)
+#define CR4_OSFXSR     (1 << 9)
+#define CR4_OSXMMEXCPT (1 << 10)
+#define CR4_UMIP       (1 << 11)
+#define CR4_VMXE       (1 << 13)
+#define CR4_SMXE       (1 << 14)
+#define CR4_FSGSBASE   (1 << 16)
+#define CR4_PCIDE      (1 << 17)
+#define CR4_OSXSAVE    (1 << 18)
+#define CR4_SMEP       (1 << 20)
+#define CR4_SMAP       (1 << 21)
+#define CR4_PKE        (1 << 22)
+#define CR4_CET        (1 << 23)
+#define CR4_PKS        (1 << 24)
+
+#define MSR_EFER_SCE   (1 << 0)
+#define MSR_EFER_LME   (1 << 8)
+#define MSR_EFER_LMA   (1 << 10)
+#define MSR_EFER_NXE   (1 << 11)
+#define MSR_EFER_SVME  (1 << 12)
+#define MSR_EFER_LMSLE (1 << 13)
+#define MSR_EFER_FFXSR (1 << 14)
+#define MSR_EFER_TCE   (1 << 15)
+
+#define MSR_EFER           (0xC0000080)
+#define MSR_STAR           (0xC0000081)
+#define MSR_LSTAR          (0xC0000082)
+#define MSR_CSTAR          (0xC0000083)
+#define MSR_SFMASK         (0xC0000084)
+#define MSR_FS_BASE        (0xC0000100)
+#define MSR_GS_BASE        (0xC0000101)
+#define MSR_KERNEL_GS_BASE (0xC0000102)
+
+#define CPL_USER   (0b11)
+#define CPL_KERNEL (0b00)

kernel/arch/x86_64/include/gdt.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <kernel/compiler.h>
+#include <stdint.h>
+#include "tss.h"
+
+struct [[__packed]] gdt {
+    uint64_t null;
+    uint64_t kernel_code32;
+    uint64_t kernel_data32;
+    uint64_t kernel_code64;
+    uint64_t kernel_data64;
+    uint64_t user_code;
+    uint64_t user_data;
+    uint64_t user_code64;
+    uint64_t tss[2];
+};
+
+struct [[__packed]] gdtr {
+    uint16_t limit;
+    struct gdt* base;
+};
+
+// Initializes a GDT on the local core.
+void gdt_init();
+
+// Sets the Task State Segment in the given GDT.
+void gdt_set_tss(struct gdt* gdt, struct tss* tss);

kernel/arch/x86_64/include/idt.h

@@ -0,0 +1 @@
+#pragma once

kernel/arch/x86_64/include/smp.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#define SMP_GDTR_OFFSET       80
+#define SMP_FARJMP_OFFSET     90
+#define SMP_TEMP_STACK_OFFSET 98
+#define SMP_TEMP_CR3_OFFSET   102
+#define SMP_ENTRY_OFFSET      106
+#define SMP_HHDM_OFFSET       114
+#define SMP_KERNEL32_DS       16
+#define SMP_KERNEL64_CS       24
+#define SMP_KERNEL64_DS       32
+#define SMP_INFO_SIZE         127
+
+#ifndef __ASSEMBLER__
+
+#include <gdt.h>
+#include <kernel/common.h>
+#include <kernel/compiler.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct [[__packed]] smp_info {
+    struct gdt gdt;
+    struct gdtr gdtr;
+    uint32_t farjmp_offset;
+    uint32_t farjmp_segment;
+    uint32_t temp_stack;
+    uint32_t temp_cr3;
+    uint64_t entry;
+    uint64_t hhdm_offset;
+    uint32_t lapic_id;
+    uint8_t booted;
+};
+
+static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr));
+static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset));
+static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack));
+static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3));
+static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry));
+static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset));
+static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32));
+static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64));
+static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64));
+static_assert(SMP_INFO_SIZE == sizeof(struct smp_info));
+
+// Initializes a LAPIC.
+void x86_64_smp_init(uint32_t id);
+
+#endif

kernel/arch/x86_64/include/tss.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <kernel/compiler.h>
+#include <stdint.h>
+
+struct [[__packed, __aligned(0x10)]] tss {
+    uint32_t reserved0;
+    uint64_t rsp0;
+    uint64_t rsp1;
+    uint64_t rsp2;
+    uint32_t reserved1;
+    uint32_t reserved2;
+    uint64_t ist1;
+    uint64_t ist2;
+    uint64_t ist3;
+    uint64_t ist4;
+    uint64_t ist5;
+    uint64_t ist6;
+    uint64_t ist7;
+    uint32_t reserved3;
+    uint32_t reserved4;
+    uint16_t reserved5;
+    uint16_t iopb;
+};

kernel/arch/x86_64/mem.c

@@ -0,0 +1,56 @@
+#include <kernel/mem.h>
+#include <kernel/types.h>
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};
+
+void arch_mem_pte_clear(pte_t* pte) {
+    *pte = 0;
+}
+
+pte_t arch_mem_pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
+    pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
+
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+
+    if (flags & PTE_DIR)
+        result |= ARCH_FLAG_READ_WRITE;
+    else {
+        if (flags & PTE_WRITE)
+            result |= ARCH_FLAG_READ_WRITE;
+        if (!(flags & PTE_EXEC))
+            result |= ARCH_FLAG_EXECUTE_DISABLE;
+    }
+    return result;
+}
+
+bool arch_mem_pte_is_present(pte_t* pte) {
+    return *pte & ARCH_FLAG_PRESENT;
+}
+
+bool arch_mem_pte_is_dir(pte_t* pte) {
+    return *pte & ARCH_FLAG_SIZE;
+}
+
+phys_t arch_mem_pte_address(pte_t* pte) {
+    return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
+}
+
+void arch_mem_pt_set(struct page_table* pt) {
+    asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
+}

kernel/arch/x86_64/meson.build

@@ -0,0 +1,17 @@
+kernel_c_args += [
+    '-masm=intel',
+    '-mgeneral-regs-only',
+    '-mno-red-zone',
+    '-mcmodel=kernel',
+]
+
+kernel_sources += files(
+    'gdt.c',
+    'idt.c',
+    'mem.c',
+    'serial.c',
+    'smp.c',
+    'sys.c',
+    'trampoline.S',
+    'usercopy.c',
+)

kernel/arch/x86_64/serial.c

@@ -0,0 +1,47 @@
+#include <asm.h>
+#include <kernel/console.h>
+#include <stdint.h>
+
+constexpr uint16_t COM1_BASE = 0x3F8;
+
+static bool is_ready() {
+    return asm_inb(COM1_BASE + 5) & 0x20;
+}
+
+static void serial_write(struct console* con, const char* buf, size_t count) {
+    for (size_t i = 0; i < count; i++) {
+        while (!is_ready())
+            asm volatile("pause");
+
+        asm_outb(COM1_BASE, buf[i]);
+    }
+}
+
+bool serial_setup(struct console* con) {
+    asm_outb(COM1_BASE + 1, 0x00); // Disable interrupts
+    asm_outb(COM1_BASE + 3, 0x80); // Enable DLAB (set baud rate divisor)
+    asm_outb(COM1_BASE, 0x01);     // Set divisor low byte (115200 baud if 1)
+    asm_outb(COM1_BASE + 1, 0x00); // Set divisor high byte
+    asm_outb(COM1_BASE + 3, 0x03); // 8 bits, no parity, one stop bit (8n1)
+    asm_outb(COM1_BASE + 2, 0xC7); // Enable FIFO, clear them, with 14-byte threshold
+    asm_outb(COM1_BASE + 4, 0x0B); // IRQs enabled, RTS/DSR set
+
+    asm_outb(COM1_BASE + 4, 0x1E); // Set to loopback mode.
+    asm_outb(COM1_BASE, 0xAE);     // Send a test byte.
+
+    // If we don't get the same value back, this serial port doesn't work.
+    if (asm_inb(COM1_BASE) != 0xAE) {
+        return false;
+    }
+
+    asm_outb(COM1_BASE + 4, 0x0F); // Disable loopback mode.
+
+    return true;
+}
+
+static struct console serial_con = {
+    .name = "com1",
+    .write = serial_write,
+    .init = serial_setup,
+};
+DEFINE_CONSOLE(&serial_con);

kernel/arch/x86_64/smp.c

@@ -0,0 +1,17 @@
+#include <kernel/compiler.h>
+#include <kernel/types.h>
+#include <smp.h>
+
+extern uint8_t smp_trampoline_start[];
+extern uint8_t smp_trampoline_end[];
+
+[[noreturn]]
+static void ap_entry(phys_t info) {
+    // TODO
+
+    while (1) {
+        asm volatile("hlt");
+    }
+}
+
+void x86_64_smp_init(uint32_t id) {}

kernel/arch/x86_64/sys.c

@@ -0,0 +1,42 @@
+#include <zinnia/archctl.h>
+#include <zinnia/status.h>
+#include <kernel/compiler.h>
+#include <kernel/init.h>
+#include <kernel/percpu.h>
+#include <gdt.h>
+#include "asm.h"
+#include "defs.h"
+
+[[__init, __naked]]
+void _start() {
+    asm volatile(
+        "lea rsp, [rip + %0]\n"
+        "jmp %1"
+        :
+        : "i"(__ld_stack_top), "r"(kernel_entry)
+    );
+}
+
+void percpu_bsp_early_init() {
+    asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
+    asm_wrmsr(MSR_FS_BASE, 0);
+    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
+
+    gdt_init();
+}
+
+[[noreturn]]
+void arch_panic() {
+    asm volatile("cli; hlt");
+    __unreachable();
+}
+
+zn_status_t arch_archctl(zn_archctl_t op, uintptr_t arg) {
+    switch (op) {
+    case ZN_ARCHCTL_SET_FSBASE:
+        asm_wrmsr(MSR_FS_BASE, arg);
+        return 0;
+    default:
+        return ZN_ERR_BAD_ARG;
+    }
+}

kernel/arch/x86_64/trampoline.S

@@ -0,0 +1,107 @@
+#include <smp.h>
+#include <defs.h>
+
+.intel_syntax noprefix
+
+.global smp_trampoline_start
+.global smp_trampoline_end
+
+.section .rodata
+.code16
+
+smp_trampoline_start:
+    cli
+    cld
+    jmp 1f
+
+smp_trampoline_data:
+    .skip SMP_INFO_SIZE
+
+.set data_offset, (smp_trampoline_data - smp_trampoline_start)
+.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET)
+.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET)
+.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET)
+.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET)
+.set entry_offset, (data_offset + SMP_ENTRY_OFFSET)
+.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET)
+
+1:
+    mov bx, cs
+    shl ebx, 4
+
+.set idtr_offset, (invalid_idtr - smp_trampoline_start)
+    lidtd cs:idtr_offset
+    lgdtd cs:gdtr_offset
+
+.set mode32_offset, (mode32 - smp_trampoline_start)
+    lea eax, [ebx + mode32_offset]
+    mov dword ptr cs:farjmp_offset, eax
+
+    mov eax, CR0_PE | CR0_ET
+    mov cr0, eax
+    jmp fword ptr cs:farjmp_offset
+
+.code32
+mode32:
+    mov ax, SMP_KERNEL32_DS
+    mov ds, ax
+    mov es, ax
+    mov fs, ax
+    mov gs, ax
+    mov ss, ax
+
+    xor  eax, eax
+    lldt ax
+
+    mov esp, dword ptr [ebx + temp_stack_offset]
+
+    xor eax, eax
+    mov cr4, eax
+
+    or  eax, CR4_PAE
+    mov cr4, eax
+
+    mov ecx, MSR_EFER
+    mov eax, MSR_EFER_LME
+    xor edx, edx
+    wrmsr
+
+    mov eax, dword ptr [ebx + temp_cr3_offset]
+    mov cr3, eax
+
+    mov eax, cr0
+    or  eax, CR0_PG
+    mov cr0, eax
+
+.set mode64_offset, (mode64 - smp_trampoline_start)
+    lea eax, [ebx + mode64_offset]
+    push SMP_KERNEL64_CS
+    push eax
+    retf
+
+.code64
+mode64:
+    mov ax, SMP_KERNEL64_DS
+    mov ds, ax
+    mov es, ax
+    mov fs, ax
+    mov gs, ax
+    mov ss, ax
+
+    mov ecx, MSR_EFER
+    rdmsr
+    or eax, MSR_EFER_NXE
+    wrmsr
+
+    xor ebp, ebp
+
+    lea rdi, [rbx + data_offset]
+    mov rax, [rbx + entry_offset]
+    add rsp, [rbx + hhdm_offset]
+    jmp rax
+
+invalid_idtr:
+    .word 0
+    .quad 0
+
+smp_trampoline_end:

kernel/arch/x86_64/usercopy.c

@@ -0,0 +1,128 @@
+#include <kernel/compiler.h>
+#include <kernel/usercopy.h>
+
+extern void x86_64_read_start();
+extern void x86_64_read_end();
+extern void x86_64_read_fault();
+
+static const struct usercopy_region read_region = {
+    .start_ip = x86_64_read_start,
+    .end_ip = x86_64_read_end,
+    .fault_ip = x86_64_read_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_read(uint8_t* dst, const __user uint8_t* src, size_t len) {
+    asm volatile(
+        // Setup regs for `rep movsb`.
+        "xchg rcx, rdx\n"
+        "mov rax, [rip + %0]\n"
+        "mov [rdx], rax\n"
+
+        ".global x86_64_read_start\n"
+        "x86_64_read_start:\n"
+        "rep movsb\n"
+
+        ".global x86_64_read_end\n"
+        "x86_64_read_end:\n"
+        "xor rax, rax\n"
+        "mov [rdx], rax\n"
+        "mov rax, 1\n"
+        "ret\n"
+
+        ".global x86_64_read_fault\n"
+        "x86_64_read_fault:\n"
+        "xor rax, rax\n"
+        "mov [rdx], rax\n"
+        "ret\n"
+        :
+        : "i"(&read_region)
+        : "memory"
+    );
+}
+
+extern void x86_64_write_start();
+extern void x86_64_write_end();
+extern void x86_64_write_fault();
+
+static const struct usercopy_region write_region = {
+    .start_ip = x86_64_write_start,
+    .end_ip = x86_64_write_end,
+    .fault_ip = x86_64_write_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_write(__user uint8_t* dst, const uint8_t* src, size_t len) {
+    asm volatile(
+        // Setup regs for `rep movsb`.
+        "xchg rcx, rdx\n"
+        "mov rax, [rip + %0]\n"
+        "mov [rdx], rax\n"
+
+        ".global x86_64_write_start\n"
+        "x86_64_write_start:\n"
+        "rep movsb\n"
+
+        ".global x86_64_write_end\n"
+        "x86_64_write_end:\n"
+        "xor rax, rax\n"
+        "mov [rdx], rax\n"
+        "mov rax, 1\n"
+        "ret\n"
+
+        ".global x86_64_write_fault\n"
+        "x86_64_write_fault:\n"
+        "xor rax, rax\n"
+        "mov [rdx], rax\n"
+        "ret\n"
+        :
+        : "i"(&read_region)
+        : "memory"
+    );
+}
+
+extern void x86_64_strlen_start();
+extern void x86_64_strlen_end();
+extern void x86_64_strlen_fault();
+
+static const struct usercopy_region strlen_region = {
+    .start_ip = x86_64_strlen_start,
+    .end_ip = x86_64_strlen_end,
+    .fault_ip = x86_64_strlen_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_strlen(const __user uint8_t* str, size_t max, size_t* len) {
+    asm volatile(
+        "mov rax, [rip + %0]\n"
+        "mov [rcx], rax\n"
+
+        ".global x86_64_strlen_start\n"
+        "x86_64_strlen_start:\n"
+        "xor r8, r8\n"
+        ".Lloop:\n"
+        "cmp byte ptr [rdi + r8], 0\n"
+        "je .Lleave\n"
+        "inc r8\n"
+        "cmp rsi, r8\n"
+        "jne .Lloop\n"
+        ".Lleave:\n"
+        "mov [rdx], r8\n"
+
+        ".global x86_64_strlen_end\n"
+        "x86_64_strlen_end:\n"
+        "xor rax, rax\n"
+        "mov [rcx], rax\n"
+        "mov rax, 1\n"
+        "ret\n"
+
+        ".global x86_64_strlen_fault\n"
+        "x86_64_strlen_fault:\n"
+        "xor rax, rax\n"
+        "mov [rcx], rax\n"
+        "ret\n"
+        :
+        : "i"(&read_region)
+        : "memory"
+    );
+}