pmap tlb shootdowns

arch/x86_64/apic.c

@@ -4,37 +4,48 @@
 #include <zinnia/mmio.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
+#include <zinnia/utils.h>
 #include <zinnia/vma.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
-static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
+static uint32_t lapic_msr(size_t offset) {
+    return 0x800 + (offset >> 4);
+}
+
+static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
+    if (lapic->xapic_regs)
+        return mmio_reg_read(lapic->xapic_regs, reg);
+    else
+        return asm_rdmsr(lapic_msr(reg.offset));
+}
+
+static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) {
+    if (lapic->xapic_regs)
+        mmio_reg_write(lapic->xapic_regs, reg, value);
+    else
+        asm_wrmsr(lapic_msr(reg.offset), value);
+}
+
+static uint64_t lapic_read_icr(struct local_apic* lapic) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
-            uint64_t lo, hi;
-            lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR);
-            hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI);
-            return (hi << 32) | lo;
-        } else {
-            return mmio_read32(lapic->xapic_regs + reg);
-        }
+        uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO);
+        uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI);
+        return (hi << 32) | lo;
     } else {
-        return asm_rdmsr(0x800 + (reg >> 4));
+        return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset));
     }
 }
 
-static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
+static void lapic_write_icr(struct local_apic* lapic, uint64_t value) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32);
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value);
-        } else {
-            mmio_write32(lapic->xapic_regs + reg, (uint32_t)value);
-        }
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32));
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value);
     } else {
-        asm_wrmsr(0x800 + (reg >> 4), value);
+        asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value);
     }
 }
 
@@ -56,12 +67,15 @@ void lapic_init(struct local_apic* lapic) {
 
     lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
     if (lapic->xapic_regs)
-        lapic->id >>= 24;
+        lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id);
 
     // Reset the TPR.
     lapic_write_reg(lapic, APIC_REG_TPR, 0);
     // Enable APIC bit in the SIVR.
-    lapic_write_reg(lapic, APIC_REG_SIVR, (lapic_read_reg(lapic, APIC_REG_SIVR) & ~0xFF) | 0x100 | IDT_SPURIOUS);
+    uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR);
+    sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS);
+    sivr |= mmio_field(APIC_SIVR_ENABLE, 1);
+    lapic_write_reg(lapic, APIC_REG_SIVR, sivr);
 
     if (lapic->xapic_regs) {
         lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
@@ -88,6 +102,9 @@ void lapic_eoi(struct local_apic* lapic) {
     lapic_write_reg(lapic, APIC_REG_EOI, 0);
 }
 
+constexpr uint64_t the_icr =
+    mmio_field(APIC_ICR_VECTOR, 128) | mmio_field(APIC_ICR_DELIVERY_MODE, APIC_DELIVERY_STARTUP);
+
 void lapic_send_ipi(
     struct local_apic* lapic,
     uint32_t target,
@@ -97,21 +114,21 @@ void lapic_send_ipi(
     enum apic_level level,
     enum apic_trigger_mode trigger_mode
 ) {
-    uint64_t icr = vector;
-    icr |= (uint64_t)delivery_mode << 8;
-    icr |= (uint64_t)dest_mode << 11;
-    icr |= (uint64_t)level << 14;
-    icr |= (uint64_t)trigger_mode << 15;
+    uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector);
+    icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode);
+    icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode);
+    icr |= mmio_field(APIC_ICR_LEVEL, level);
+    icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode);
 
     if (lapic->xapic_regs)
-        icr |= (uint64_t)(uint8_t)target << 56;
+        icr |= mmio_field(APIC_ICR_DEST_XAPIC, target);
     else
-        icr |= (uint64_t)target << 32;
+        icr |= mmio_field(APIC_ICR_DEST_X2APIC, target);
 
-    lapic_write_reg(lapic, APIC_REG_ICR, icr);
+    lapic_write_icr(lapic, icr);
 
     if (lapic->xapic_regs) {
-        while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12))
+        while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING)
             asm volatile("pause");
     }
 }

arch/x86_64/hpet.c

@@ -26,18 +26,20 @@ struct hpet {
     uint32_t period;
 };
 
-constexpr uint32_t HPET_CAP = 0x0;
-constexpr uint32_t HPET_CFG = 0x10;
-constexpr uint32_t HPET_COUNTER = 0xF0;
+constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0};
+constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32};
+constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10};
+constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1};
+constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0};
 
 static void hpet_reset(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    mmio_write64(hpet->regs + HPET_COUNTER, 0);
+    mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0);
 }
 
 static uint64_t hpet_get_elapsed_ns(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
+    uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER);
     return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
 }
 
@@ -69,10 +71,12 @@ static void hpet_init(struct boot_info*) {
     uacpi_table_unref(&table);
     log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
 
-    mmio_write64(hpet_clock.regs + HPET_CFG, mmio_read64(hpet_clock.regs + HPET_CFG) | 1);
+    uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG);
+    cfg |= mmio_field(HPET_CFG_ENABLE, 1);
+    mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg);
 
-    uint64_t caps = mmio_read64(hpet_clock.regs + HPET_CAP);
-    hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
+    uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP);
+    hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap);
 
     assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!");
 }

arch/x86_64/idt.c

@@ -3,6 +3,7 @@
 #include <zinnia/irq.h>
 #include <zinnia/panic.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
 #include <zinnia/usercopy.h>
@@ -93,6 +94,10 @@ void interrupt_handler(struct arch_context* ctx) {
         if (arch_sched_preempt_enable())
             sched_reschedule(&cpu->sched);
         break;
+    case IDT_IPI_SHOOTDOWN:
+        lapic_eoi(&arch_cpu->lapic);
+        pmap_shootdown_handle();
+        break;
     default:
         if (isr < 0x20)
             panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);

arch/x86_64/include/x86_64/apic.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <zinnia/mmio.h>
 #include <stdint.h>
 
 struct local_apic {
@@ -37,21 +38,32 @@ enum apic_trigger_mode {
     APIC_TRIGGER_LEVEL = 1,
 };
 
-enum apic_regs {
-    APIC_REG_ID = 0x20,
-    APIC_REG_TPR = 0x80,
-    APIC_REG_EOI = 0xB0,
-    APIC_REG_LDR = 0xD0,
-    APIC_REG_DFR = 0xE0,
-    APIC_REG_SIVR = 0xF0,
-    APIC_REG_ESR = 0x280,
-    APIC_REG_ICR = 0x300,
-    APIC_REG_ICR_HI = 0x310,
-    APIC_REG_LVT_TR = 0x320,
-    APIC_REG_ICR_TIMER = 0x380,
-    APIC_REG_CCR = 0x390,
-    APIC_REG_DCR = 0x3E0,
-};
+constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
+constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8};
+constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
+constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
+constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
+constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
+constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
+constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8};
+constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1};
+constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
+constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300};
+constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8};
+constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3};
+constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1};
+constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1};
+constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1};
+constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1};
+constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8};
+constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32};
+constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300};
+constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
+constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
+constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8};
+constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
+constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
+constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
 
 void lapic_init(struct local_apic* lapic);
 void lapic_eoi(struct local_apic* lapic);

arch/x86_64/irq.c

@@ -1,6 +1,12 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/percpu.h>
+#include <zinnia/utils.h>
 #include <arch/irq.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
 bool arch_irq_get_state() {
     uint64_t flags;
@@ -20,3 +26,68 @@ bool arch_irq_set_state(bool state) {
 void arch_irq_wait() {
     asm volatile("hlt");
 }
+
+struct apic_msi_line {
+    struct msi_line msi;
+    uint8_t vector;
+    uint32_t lapic;
+};
+
+static phys_t apic_msi_msg_addr(struct msi_line* self) {
+    struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi);
+    return 0xFEE0'0000 | ((phys_t)line->lapic << 12);
+}
+
+static uint32_t apic_msi_msg_data(struct msi_line* self) {
+    return containerof(self, struct apic_msi_line, msi)->vector;
+}
+
+static enum irq_mode apic_msi_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered");
+    return IRQ_MODE_EOI;
+}
+
+static void apic_msi_mask(struct irq_line* self) {}
+
+static void apic_msi_unmask(struct irq_line* self) {}
+
+static void apic_msi_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
+    irq_lock();
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+    size_t slot;
+    for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) {
+        if (!arch_cpu->irq_lines[slot])
+            break;
+    }
+    if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
+        irq_unlock();
+        mem_free(line);
+        return ENOSPC;
+    }
+
+    line->msi.line.set_config = apic_msi_set_config;
+    line->msi.line.mask = apic_msi_mask;
+    line->msi.line.unmask = apic_msi_unmask;
+    line->msi.line.eoi = apic_msi_eoi;
+    line->msi.msg_addr = apic_msi_msg_addr;
+    line->msi.msg_data = apic_msi_msg_data;
+    line->vector = slot + 0x20;
+    line->lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = &line->msi.line;
+    irq_unlock();
+
+    *out = &line->msi;
+    return 0;
+}

arch/x86_64/pmap.c

@@ -1,5 +1,10 @@
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/types.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 #include <x86_64/pmap.h>
 
 void pte_clear(pte_t* pte) {
@@ -10,6 +15,26 @@ void pte_flush(uintptr_t vaddr) {
     asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
 }
 
+void pte_flush_all() {
+    uint64_t cr3;
+    asm volatile("mov %0, cr3" : "=r"(cr3));
+    asm volatile("mov cr3, %0" ::"r"(cr3) : "memory");
+}
+
+void pte_shootdown_kick(struct percpu* cpu) {
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_SHOOTDOWN,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
 

arch/x86_64/sched.c

@@ -2,6 +2,7 @@
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/sched.h>
@@ -10,6 +11,7 @@
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
@@ -160,12 +162,28 @@ void arch_task_fini(struct arch_task_context* context) {
     const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
     for (size_t i = 0; i < fpu_size; i += page_size)
         pmap_unmap(&kernel_vma.pmap, fpu_base + i);
+    pmap_shootdown(&kernel_vma.pmap);
     mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
     vm_free(fpu_base, fpu_size);
 
     context->fpu_region = nullptr;
 }
 
+void arch_task_fork(struct arch_task_context* dst) {
+    if (dst->fpu_region)
+        percpu_ref(arch_data)->fpu_save(dst->fpu_region);
+}
+
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) {
+    *ctx = (struct arch_context){
+        .rip = ip,
+        .rsp = sp,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+}
+
 [[__naked]]
 void arch_sched_preempt_disable() {
     asm volatile(
@@ -189,6 +207,23 @@ bool arch_sched_preempt_enable() {
     );
 }
 
+void arch_sched_kick(struct percpu* cpu) {
+    if (!cpu->online)
+        return;
+    // The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler.
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_RESCHED,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 [[__naked]]
 void arch_sched_jump_to_context(struct arch_context* context) {
     asm volatile(

core/mem/pmap.c

@@ -1,5 +1,8 @@
 #include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/spin.h>
 #include <zinnia/vma.h>
@@ -133,3 +136,103 @@ fail:
     spin_unlock(&pt->lock);
     return status;
 }
+
+errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte))
+        status = ENOENT;
+    else
+        *out = pte_address(pte);
+
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
+    spin_lock(&pt->lock);
+
+    errno_t status = 0;
+    pte_t* pte = get_pte(pt, vaddr, false, true);
+    if (pte == nullptr || !pte_is_present(pte)) {
+        status = ENOENT;
+        goto fail;
+    }
+
+    *pte = pte_build(pte_address(pte), flags, CACHE_ANY);
+    pte_flush(vaddr);
+
+fail:
+    spin_unlock(&pt->lock);
+    return status;
+}
+
+static void free_table(phys_t table, int8_t level) {
+    if (level > 0) {
+        pte_t* entries = HHDM_PTR(table);
+        const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+        for (size_t i = 0; i < num_entries; i++) {
+            if (pte_is_present(&entries[i]))
+                free_table(pte_address(&entries[i]), level - 1);
+        }
+    }
+    mem_phys_free(table, 1);
+}
+
+void pmap_destroy(struct pmap* pt) {
+    if (!pt->is_user)
+        return;
+
+    pte_t* root = HHDM_PTR(pt->root);
+    const size_t num_entries = (size_t)1 << arch_mem_level_bits();
+    for (size_t i = 0; i < num_entries / 2; i++) {
+        if (pte_is_present(&root[i]))
+            free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
+    }
+    mem_phys_free(pt->root, 1);
+    pt->root = 0;
+}
+
+[[__percpu]]
+static atomic_size_t shootdown_req;
+
+[[__percpu]]
+static atomic_size_t shootdown_ack;
+
+void pmap_shootdown_handle() {
+    const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
+    if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
+        return;
+    pte_flush_all();
+    atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
+}
+
+void pmap_shootdown(struct pmap* pt) {
+    const size_t count = percpu_count();
+    if (count < 2)
+        return;
+
+    irq_lock();
+    struct percpu* self = percpu_get();
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
+        pte_shootdown_kick(cpu);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        struct percpu* cpu = percpu_cpu(i);
+        if (cpu == self || !cpu->online)
+            continue;
+        const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
+        while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
+            pmap_shootdown_handle();
+            asm volatile("pause");
+        }
+    }
+    irq_unlock();
+}

core/mem/stack.c

@@ -52,6 +52,7 @@ void stack_destroy(struct stack* ks) {
     const size_t page_size = arch_mem_page_size();
     for (size_t i = 0; i < ks->size; i += page_size)
         pmap_unmap(&kernel_vma.pmap, ks->base + i);
+    pmap_shootdown(&kernel_vma.pmap);
 
     mem_phys_free(ks->phys, ks->size / page_size);
     vm_free(ks->base - page_size, ks->size + page_size);

core/module.c

@@ -204,6 +204,10 @@ static void module_free(struct module* mod) {
         const struct module_segment* seg = &mod->segments[i];
         for (size_t off = 0; off < seg->size; off += page_size)
             pmap_unmap(&kernel_vma.pmap, seg->virt + off);
+    }
+    pmap_shootdown(&kernel_vma.pmap);
+    for (size_t i = 0; i < mod->num_segments; i++) {
+        const struct module_segment* seg = &mod->segments[i];
         mem_phys_free(seg->phys, seg->size / page_size);
     }
     mem_free(mod->segments);

core/percpu.c

@@ -3,12 +3,11 @@
 #include <zinnia/cmdline.h>
 #include <zinnia/compiler.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/utils.h>
+#include <zinnia/vma.h>
 #include <arch/percpu.h>
 
-extern uint8_t __ld_percpu_start[];
-extern uint8_t __ld_percpu_end[];
-
 static _Atomic(size_t) num_cpus = 1;
 
 static size_t max_cpus = SIZE_MAX;
@@ -25,20 +24,33 @@ struct percpu percpu_bsp = {
 };
 
 struct percpu* percpu_new() {
-    const size_t size = __ld_percpu_end - __ld_percpu_start;
-    const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
-
-    // TODO: Map pages past the end of the kernel.
-    // TODO: Document how allocations work.
-
-    phys_t mem;
-    if (mem_phys_alloc(num_pages, 0, &mem))
+    const size_t pgsz = arch_mem_page_size();
+    const size_t stride = percpu_stride();
+    const size_t id = atomic_load(&num_cpus, ATOMIC_RELAXED);
+    if (id >= (UINTPTR_MAX - (uintptr_t)__ld_percpu_start) / stride)
         return nullptr;
 
-    struct percpu* cpu = HHDM_PTR(mem);
+    phys_t mem;
+    if (mem_phys_alloc(stride / pgsz, 0, &mem))
+        return nullptr;
+
+    const uintptr_t base = (uintptr_t)percpu_cpu(id);
+    for (size_t offset = 0; offset < stride; offset += pgsz) {
+        if (pmap_map(&kernel_vma.pmap, base + offset, mem + offset, PTE_READ | PTE_WRITE, CACHE_ANY)) {
+            while (offset) {
+                offset -= pgsz;
+                pmap_unmap(&kernel_vma.pmap, base + offset);
+            }
+            mem_phys_free(mem, stride / pgsz);
+            return nullptr;
+        }
+    }
+
+    struct percpu* cpu = (struct percpu*)base;
     cpu->self = cpu;
-    cpu->id = atomic_fetch_add(&num_cpus, 1, ATOMIC_RELAXED);
+    cpu->id = id;
     cpu->online = false;
+    atomic_store(&num_cpus, id + 1, ATOMIC_RELEASE);
     return cpu;
 }
 
@@ -46,6 +58,10 @@ size_t percpu_max_cpus() {
     return max_cpus;
 }
 
+size_t percpu_count() {
+    return atomic_load(&num_cpus, ATOMIC_RELAXED);
+}
+
 struct percpu* percpu_get() {
     return arch_percpu_get();
 }

include/arch/irq.h

@@ -5,3 +5,4 @@
 bool arch_irq_set_state(bool state);
 bool arch_irq_get_state();
 void arch_irq_wait();
+errno_t arch_irq_alloc_msi(struct msi_line** out);

include/arch/sched.h

@@ -3,8 +3,11 @@
 #include <zinnia/errno.h>
 #include <zinnia/sched.h>
 
+struct percpu;
+
 void arch_sched_preempt_disable();
 bool arch_sched_preempt_enable();
+void arch_sched_kick(struct percpu* cpu);
 struct task* arch_sched_switch(struct task* from, struct task* to);
 errno_t arch_task_init(
     struct arch_task_context* context,
@@ -14,6 +17,8 @@ errno_t arch_task_init(
     bool is_user
 );
 void arch_task_fini(struct arch_task_context* context);
+void arch_task_fork(struct arch_task_context* dst);
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp);
 [[noreturn]]
 void arch_sched_jump_to_context(struct arch_context* context);
 [[noreturn]]

include/zinnia/irq.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <zinnia/compiler.h>
+#include <zinnia/errno.h>
 #include <zinnia/list.h>
 #include <zinnia/types.h>
 #include <stddef.h>

include/zinnia/percpu.h

@@ -5,6 +5,7 @@
 #include <zinnia/sched.h>
 #include <zinnia/usercopy.h>
 #include <stddef.h>
+#include <stdint.h>
 
 // # On using per-CPU data
 //
@@ -37,8 +38,20 @@ struct percpu {
     struct scheduler sched; // Scheduler information.
 };
 
+extern uint8_t __ld_percpu_start[];
+extern uint8_t __ld_percpu_end[];
+
+static inline size_t percpu_stride() {
+    return __ld_percpu_end - __ld_percpu_start;
+}
+
+static inline struct percpu* percpu_cpu(size_t id) {
+    return (struct percpu*)(__ld_percpu_start + id * percpu_stride());
+}
+
 struct percpu* percpu_new();
 size_t percpu_max_cpus();
+size_t percpu_count();
 struct percpu* percpu_get();
 void percpu_bsp_init();
 void percpu_init(struct boot_info* info);

include/zinnia/pmap.h

@@ -6,6 +6,8 @@
 #include <zinnia/utils.h>
 #include <bits/pmap.h>
 
+struct percpu;
+
 enum pte_flags : uint8_t {
     PTE_READ = 1 << 0,  // Can read from this page.
     PTE_WRITE = 1 << 1, // Can write to this page.
@@ -33,10 +35,16 @@ errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags);
 errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache);
 errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags);
 errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr);
+errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out);
+void pmap_destroy(struct pmap* pt);
 void pmap_set(struct pmap* pt);
+void pmap_shootdown(struct pmap* pt);
+void pmap_shootdown_handle();
 
 void pte_clear(pte_t* pte);
 void pte_flush(uintptr_t vaddr);
+void pte_flush_all();
+void pte_shootdown_kick(struct percpu* cpu);
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
 bool pte_is_present(pte_t* pte);
 bool pte_is_dir(pte_t* pte);