port apic and hpet to mmio API, apic_msis

arch/x86_64/apic.c

@@ -4,37 +4,48 @@
 #include <zinnia/mmio.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
+#include <zinnia/utils.h>
 #include <zinnia/vma.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
-static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
+static uint32_t lapic_msr(size_t offset) {
+    return 0x800 + (offset >> 4);
+}
+
+static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
+    if (lapic->xapic_regs)
+        return mmio_reg_read(lapic->xapic_regs, reg);
+    else
+        return asm_rdmsr(lapic_msr(reg.offset));
+}
+
+static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) {
+    if (lapic->xapic_regs)
+        mmio_reg_write(lapic->xapic_regs, reg, value);
+    else
+        asm_wrmsr(lapic_msr(reg.offset), value);
+}
+
+static uint64_t lapic_read_icr(struct local_apic* lapic) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
+        uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO);
-            uint64_t lo, hi;
+        uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI);
-            lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR);
+        return (hi << 32) | lo;
-            hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI);
-            return (hi << 32) | lo;
-        } else {
-            return mmio_read32(lapic->xapic_regs + reg);
-        }
     } else {
-        return asm_rdmsr(0x800 + (reg >> 4));
+        return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset));
     }
 }
 
-static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
+static void lapic_write_icr(struct local_apic* lapic, uint64_t value) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32));
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32);
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value);
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value);
-        } else {
-            mmio_write32(lapic->xapic_regs + reg, (uint32_t)value);
-        }
     } else {
-        asm_wrmsr(0x800 + (reg >> 4), value);
+        asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value);
     }
 }
 
@@ -56,12 +67,15 @@ void lapic_init(struct local_apic* lapic) {
 
     lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
     if (lapic->xapic_regs)
-        lapic->id >>= 24;
+        lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id);
 
     // Reset the TPR.
     lapic_write_reg(lapic, APIC_REG_TPR, 0);
     // Enable APIC bit in the SIVR.
-    lapic_write_reg(lapic, APIC_REG_SIVR, (lapic_read_reg(lapic, APIC_REG_SIVR) & ~0xFF) | 0x100 | IDT_SPURIOUS);
+    uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR);
+    sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS);
+    sivr |= mmio_field(APIC_SIVR_ENABLE, 1);
+    lapic_write_reg(lapic, APIC_REG_SIVR, sivr);
 
     if (lapic->xapic_regs) {
         lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
@@ -88,6 +102,9 @@ void lapic_eoi(struct local_apic* lapic) {
     lapic_write_reg(lapic, APIC_REG_EOI, 0);
 }
 
+constexpr uint64_t the_icr =
+    mmio_field(APIC_ICR_VECTOR, 128) | mmio_field(APIC_ICR_DELIVERY_MODE, APIC_DELIVERY_STARTUP);
+
 void lapic_send_ipi(
     struct local_apic* lapic,
     uint32_t target,
@@ -97,21 +114,21 @@ void lapic_send_ipi(
     enum apic_level level,
     enum apic_trigger_mode trigger_mode
 ) {
-    uint64_t icr = vector;
+    uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector);
-    icr |= (uint64_t)delivery_mode << 8;
+    icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode);
-    icr |= (uint64_t)dest_mode << 11;
+    icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode);
-    icr |= (uint64_t)level << 14;
+    icr |= mmio_field(APIC_ICR_LEVEL, level);
-    icr |= (uint64_t)trigger_mode << 15;
+    icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode);
 
     if (lapic->xapic_regs)
-        icr |= (uint64_t)(uint8_t)target << 56;
+        icr |= mmio_field(APIC_ICR_DEST_XAPIC, target);
     else
-        icr |= (uint64_t)target << 32;
+        icr |= mmio_field(APIC_ICR_DEST_X2APIC, target);
 
-    lapic_write_reg(lapic, APIC_REG_ICR, icr);
+    lapic_write_icr(lapic, icr);
 
     if (lapic->xapic_regs) {
-        while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12))
+        while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING)
             asm volatile("pause");
     }
 }

arch/x86_64/hpet.c

@@ -26,18 +26,20 @@ struct hpet {
     uint32_t period;
 };
 
-constexpr uint32_t HPET_CAP = 0x0;
+constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0};
-constexpr uint32_t HPET_CFG = 0x10;
+constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32};
-constexpr uint32_t HPET_COUNTER = 0xF0;
+constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10};
+constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1};
+constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0};
 
 static void hpet_reset(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    mmio_write64(hpet->regs + HPET_COUNTER, 0);
+    mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0);
 }
 
 static uint64_t hpet_get_elapsed_ns(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
+    uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER);
     return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
 }
 
@@ -69,10 +71,12 @@ static void hpet_init(struct boot_info*) {
     uacpi_table_unref(&table);
     log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
 
-    mmio_write64(hpet_clock.regs + HPET_CFG, mmio_read64(hpet_clock.regs + HPET_CFG) | 1);
+    uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG);
+    cfg |= mmio_field(HPET_CFG_ENABLE, 1);
+    mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg);
 
-    uint64_t caps = mmio_read64(hpet_clock.regs + HPET_CAP);
+    uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP);
-    hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
+    hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap);
 
     assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!");
 }

arch/x86_64/include/x86_64/apic.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <zinnia/mmio.h>
 #include <stdint.h>
 
 struct local_apic {
@@ -37,21 +38,32 @@ enum apic_trigger_mode {
     APIC_TRIGGER_LEVEL = 1,
 };
 
-enum apic_regs {
+constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
-    APIC_REG_ID = 0x20,
+constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8};
-    APIC_REG_TPR = 0x80,
+constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
-    APIC_REG_EOI = 0xB0,
+constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
-    APIC_REG_LDR = 0xD0,
+constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
-    APIC_REG_DFR = 0xE0,
+constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
-    APIC_REG_SIVR = 0xF0,
+constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
-    APIC_REG_ESR = 0x280,
+constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8};
-    APIC_REG_ICR = 0x300,
+constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1};
-    APIC_REG_ICR_HI = 0x310,
+constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
-    APIC_REG_LVT_TR = 0x320,
+constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300};
-    APIC_REG_ICR_TIMER = 0x380,
+constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8};
-    APIC_REG_CCR = 0x390,
+constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3};
-    APIC_REG_DCR = 0x3E0,
+constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1};
-};
+constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1};
+constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1};
+constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1};
+constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8};
+constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32};
+constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300};
+constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
+constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
+constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8};
+constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
+constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
+constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
 
 void lapic_init(struct local_apic* lapic);
 void lapic_eoi(struct local_apic* lapic);

arch/x86_64/irq.c

@@ -1,6 +1,12 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/percpu.h>
+#include <zinnia/utils.h>
 #include <arch/irq.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
 bool arch_irq_get_state() {
     uint64_t flags;
@@ -20,3 +26,68 @@ bool arch_irq_set_state(bool state) {
 void arch_irq_wait() {
     asm volatile("hlt");
 }
+
+struct apic_msi_line {
+    struct msi_line msi;
+    uint8_t vector;
+    uint32_t lapic;
+};
+
+static phys_t apic_msi_msg_addr(struct msi_line* self) {
+    struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi);
+    return 0xFEE0'0000 | ((phys_t)line->lapic << 12);
+}
+
+static uint32_t apic_msi_msg_data(struct msi_line* self) {
+    return containerof(self, struct apic_msi_line, msi)->vector;
+}
+
+static enum irq_mode apic_msi_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered");
+    return IRQ_MODE_EOI;
+}
+
+static void apic_msi_mask(struct irq_line* self) {}
+
+static void apic_msi_unmask(struct irq_line* self) {}
+
+static void apic_msi_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
+    irq_lock();
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+    size_t slot;
+    for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) {
+        if (!arch_cpu->irq_lines[slot])
+            break;
+    }
+    if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
+        irq_unlock();
+        mem_free(line);
+        return ENOSPC;
+    }
+
+    line->msi.line.set_config = apic_msi_set_config;
+    line->msi.line.mask = apic_msi_mask;
+    line->msi.line.unmask = apic_msi_unmask;
+    line->msi.line.eoi = apic_msi_eoi;
+    line->msi.msg_addr = apic_msi_msg_addr;
+    line->msi.msg_data = apic_msi_msg_data;
+    line->vector = slot + 0x20;
+    line->lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = &line->msi.line;
+    irq_unlock();
+
+    *out = &line->msi;
+    return 0;
+}

arch/x86_64/sched.c

@@ -2,6 +2,7 @@
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/sched.h>
@@ -10,6 +11,7 @@
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
@@ -166,6 +168,21 @@ void arch_task_fini(struct arch_task_context* context) {
     context->fpu_region = nullptr;
 }
 
+void arch_task_fork(struct arch_task_context* dst) {
+    if (dst->fpu_region)
+        percpu_ref(arch_data)->fpu_save(dst->fpu_region);
+}
+
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) {
+    *ctx = (struct arch_context){
+        .rip = ip,
+        .rsp = sp,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+}
+
 [[__naked]]
 void arch_sched_preempt_disable() {
     asm volatile(
@@ -189,6 +206,23 @@ bool arch_sched_preempt_enable() {
     );
 }
 
+void arch_sched_kick(struct percpu* cpu) {
+    if (!cpu->online)
+        return;
+    // The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler.
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_RESCHED,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 [[__naked]]
 void arch_sched_jump_to_context(struct arch_context* context) {
     asm volatile(

include/arch/irq.h

@@ -5,3 +5,4 @@
 bool arch_irq_set_state(bool state);
 bool arch_irq_get_state();
 void arch_irq_wait();
+errno_t arch_irq_alloc_msi(struct msi_line** out);

include/arch/sched.h

@@ -3,8 +3,11 @@
 #include <zinnia/errno.h>
 #include <zinnia/sched.h>
 
+struct percpu;
+
 void arch_sched_preempt_disable();
 bool arch_sched_preempt_enable();
+void arch_sched_kick(struct percpu* cpu);
 struct task* arch_sched_switch(struct task* from, struct task* to);
 errno_t arch_task_init(
     struct arch_task_context* context,
@@ -14,6 +17,8 @@ errno_t arch_task_init(
     bool is_user
 );
 void arch_task_fini(struct arch_task_context* context);
+void arch_task_fork(struct arch_task_context* dst);
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp);
 [[noreturn]]
 void arch_sched_jump_to_context(struct arch_context* context);
 [[noreturn]]

include/zinnia/irq.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <zinnia/compiler.h>
+#include <zinnia/errno.h>
 #include <zinnia/list.h>
 #include <zinnia/types.h>
 #include <stddef.h>