From d84232c39424a57e3ef8167a2753e6b10025f8fa Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Wed, 14 Oct 2026 23:57:35 +0200 Subject: [PATCH] port apic and hpet to mmio API, apic_msis --- arch/x86_64/apic.c | 75 +++++++++++++++++++------------ arch/x86_64/hpet.c | 20 +++++---- arch/x86_64/include/x86_64/apic.h | 42 ++++++++++------- arch/x86_64/irq.c | 71 +++++++++++++++++++++++++++++ arch/x86_64/sched.c | 34 ++++++++++++++ include/arch/irq.h | 1 + include/arch/sched.h | 5 +++ include/zinnia/irq.h | 1 + 8 files changed, 197 insertions(+), 52 deletions(-) diff --git a/arch/x86_64/apic.c b/arch/x86_64/apic.c index 67fdc35..e65478a 100644 --- a/arch/x86_64/apic.c +++ b/arch/x86_64/apic.c @@ -4,37 +4,48 @@ #include #include #include +#include #include #include #include #include #include +#include -static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) { +static uint32_t lapic_msr(size_t offset) { + return 0x800 + (offset >> 4); +} + +static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) { + if (lapic->xapic_regs) + return mmio_reg_read(lapic->xapic_regs, reg); + else + return asm_rdmsr(lapic_msr(reg.offset)); +} + +static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) { + if (lapic->xapic_regs) + mmio_reg_write(lapic->xapic_regs, reg, value); + else + asm_wrmsr(lapic_msr(reg.offset), value); +} + +static uint64_t lapic_read_icr(struct local_apic* lapic) { if (lapic->xapic_regs) { - if (reg == APIC_REG_ICR) { - uint64_t lo, hi; - lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR); - hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI); - return (hi << 32) | lo; - } else { - return mmio_read32(lapic->xapic_regs + reg); - } + uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO); + uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI); + return (hi << 32) | lo; } else { - return asm_rdmsr(0x800 + (reg >> 4)); + return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset)); } } -static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) { +static void lapic_write_icr(struct local_apic* lapic, uint64_t value) { if (lapic->xapic_regs) { - if (reg == APIC_REG_ICR) { - mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32); - mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value); - } else { - mmio_write32(lapic->xapic_regs + reg, (uint32_t)value); - } + mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32)); + mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value); } else { - asm_wrmsr(0x800 + (reg >> 4), value); + asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value); } } @@ -56,12 +67,15 @@ void lapic_init(struct local_apic* lapic) { lapic->id = lapic_read_reg(lapic, APIC_REG_ID); if (lapic->xapic_regs) - lapic->id >>= 24; + lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id); // Reset the TPR. lapic_write_reg(lapic, APIC_REG_TPR, 0); // Enable APIC bit in the SIVR. - lapic_write_reg(lapic, APIC_REG_SIVR, (lapic_read_reg(lapic, APIC_REG_SIVR) & ~0xFF) | 0x100 | IDT_SPURIOUS); + uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR); + sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS); + sivr |= mmio_field(APIC_SIVR_ENABLE, 1); + lapic_write_reg(lapic, APIC_REG_SIVR, sivr); if (lapic->xapic_regs) { lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000); @@ -88,6 +102,9 @@ void lapic_eoi(struct local_apic* lapic) { lapic_write_reg(lapic, APIC_REG_EOI, 0); } +constexpr uint64_t the_icr = + mmio_field(APIC_ICR_VECTOR, 128) | mmio_field(APIC_ICR_DELIVERY_MODE, APIC_DELIVERY_STARTUP); + void lapic_send_ipi( struct local_apic* lapic, uint32_t target, @@ -97,21 +114,21 @@ void lapic_send_ipi( enum apic_level level, enum apic_trigger_mode trigger_mode ) { - uint64_t icr = vector; - icr |= (uint64_t)delivery_mode << 8; - icr |= (uint64_t)dest_mode << 11; - icr |= (uint64_t)level << 14; - icr |= (uint64_t)trigger_mode << 15; + uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector); + icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode); + icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode); + icr |= mmio_field(APIC_ICR_LEVEL, level); + icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode); if (lapic->xapic_regs) - icr |= (uint64_t)(uint8_t)target << 56; + icr |= mmio_field(APIC_ICR_DEST_XAPIC, target); else - icr |= (uint64_t)target << 32; + icr |= mmio_field(APIC_ICR_DEST_X2APIC, target); - lapic_write_reg(lapic, APIC_REG_ICR, icr); + lapic_write_icr(lapic, icr); if (lapic->xapic_regs) { - while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12)) + while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING) asm volatile("pause"); } } diff --git a/arch/x86_64/hpet.c b/arch/x86_64/hpet.c index 6b1ab54..047352f 100644 --- a/arch/x86_64/hpet.c +++ b/arch/x86_64/hpet.c @@ -26,18 +26,20 @@ struct hpet { uint32_t period; }; -constexpr uint32_t HPET_CAP = 0x0; -constexpr uint32_t HPET_CFG = 0x10; -constexpr uint32_t HPET_COUNTER = 0xF0; +constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0}; +constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32}; +constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10}; +constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1}; +constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0}; static void hpet_reset(struct clock* c) { struct hpet* hpet = containerof(c, struct hpet, clock); - mmio_write64(hpet->regs + HPET_COUNTER, 0); + mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0); } static uint64_t hpet_get_elapsed_ns(struct clock* c) { struct hpet* hpet = containerof(c, struct hpet, clock); - uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER); + uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER); return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000); } @@ -69,10 +71,12 @@ static void hpet_init(struct boot_info*) { uacpi_table_unref(&table); log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs); - mmio_write64(hpet_clock.regs + HPET_CFG, mmio_read64(hpet_clock.regs + HPET_CFG) | 1); + uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG); + cfg |= mmio_field(HPET_CFG_ENABLE, 1); + mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg); - uint64_t caps = mmio_read64(hpet_clock.regs + HPET_CAP); - hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF; + uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP); + hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap); assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!"); } diff --git a/arch/x86_64/include/x86_64/apic.h b/arch/x86_64/include/x86_64/apic.h index 5abdaaf..967b22c 100644 --- a/arch/x86_64/include/x86_64/apic.h +++ b/arch/x86_64/include/x86_64/apic.h @@ -1,5 +1,6 @@ #pragma once +#include #include struct local_apic { @@ -37,21 +38,32 @@ enum apic_trigger_mode { APIC_TRIGGER_LEVEL = 1, }; -enum apic_regs { - APIC_REG_ID = 0x20, - APIC_REG_TPR = 0x80, - APIC_REG_EOI = 0xB0, - APIC_REG_LDR = 0xD0, - APIC_REG_DFR = 0xE0, - APIC_REG_SIVR = 0xF0, - APIC_REG_ESR = 0x280, - APIC_REG_ICR = 0x300, - APIC_REG_ICR_HI = 0x310, - APIC_REG_LVT_TR = 0x320, - APIC_REG_ICR_TIMER = 0x380, - APIC_REG_CCR = 0x390, - APIC_REG_DCR = 0x3E0, -}; +constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20}; +constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8}; +constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80}; +constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0}; +constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0}; +constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0}; +constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0}; +constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8}; +constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1}; +constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280}; +constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300}; +constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8}; +constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3}; +constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1}; +constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1}; +constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1}; +constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1}; +constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8}; +constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32}; +constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300}; +constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310}; +constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320}; +constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8}; +constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380}; +constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390}; +constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0}; void lapic_init(struct local_apic* lapic); void lapic_eoi(struct local_apic* lapic); diff --git a/arch/x86_64/irq.c b/arch/x86_64/irq.c index 1179d8b..72f01d3 100644 --- a/arch/x86_64/irq.c +++ b/arch/x86_64/irq.c @@ -1,6 +1,12 @@ +#include +#include +#include +#include #include #include +#include #include +#include bool arch_irq_get_state() { uint64_t flags; @@ -20,3 +26,68 @@ bool arch_irq_set_state(bool state) { void arch_irq_wait() { asm volatile("hlt"); } + +struct apic_msi_line { + struct msi_line msi; + uint8_t vector; + uint32_t lapic; +}; + +static phys_t apic_msi_msg_addr(struct msi_line* self) { + struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi); + return 0xFEE0'0000 | ((phys_t)line->lapic << 12); +} + +static uint32_t apic_msi_msg_data(struct msi_line* self) { + return containerof(self, struct apic_msi_line, msi)->vector; +} + +static enum irq_mode apic_msi_set_config( + struct irq_line* self, + enum irq_trigger_mode trigger, + enum irq_polarity polarity +) { + assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered"); + return IRQ_MODE_EOI; +} + +static void apic_msi_mask(struct irq_line* self) {} + +static void apic_msi_unmask(struct irq_line* self) {} + +static void apic_msi_eoi(struct irq_line* self) { + lapic_eoi(&percpu_ref(arch_data)->lapic); +} + +errno_t arch_irq_alloc_msi(struct msi_line** out) { + struct apic_msi_line* line = mem_alloc(sizeof(*line), 0); + if (!line) + return ENOMEM; + + irq_lock(); + struct arch_percpu* arch_cpu = percpu_ref(arch_data); + size_t slot; + for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) { + if (!arch_cpu->irq_lines[slot]) + break; + } + if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) { + irq_unlock(); + mem_free(line); + return ENOSPC; + } + + line->msi.line.set_config = apic_msi_set_config; + line->msi.line.mask = apic_msi_mask; + line->msi.line.unmask = apic_msi_unmask; + line->msi.line.eoi = apic_msi_eoi; + line->msi.msg_addr = apic_msi_msg_addr; + line->msi.msg_data = apic_msi_msg_data; + line->vector = slot + 0x20; + line->lapic = arch_cpu->lapic.id; + arch_cpu->irq_lines[slot] = &line->msi.line; + irq_unlock(); + + *out = &line->msi; + return 0; +} diff --git a/arch/x86_64/sched.c b/arch/x86_64/sched.c index ed42a09..ba29206 100644 --- a/arch/x86_64/sched.c +++ b/arch/x86_64/sched.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -166,6 +168,21 @@ void arch_task_fini(struct arch_task_context* context) { context->fpu_region = nullptr; } +void arch_task_fork(struct arch_task_context* dst) { + if (dst->fpu_region) + percpu_ref(arch_data)->fpu_save(dst->fpu_region); +} + +void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) { + *ctx = (struct arch_context){ + .rip = ip, + .rsp = sp, + .rflags = 0x202, + .cs = offsetof(struct gdt, user_code64) | CPL_USER, + .ss = offsetof(struct gdt, user_data) | CPL_USER, + }; +} + [[__naked]] void arch_sched_preempt_disable() { asm volatile( @@ -189,6 +206,23 @@ bool arch_sched_preempt_enable() { ); } +void arch_sched_kick(struct percpu* cpu) { + if (!cpu->online) + return; + // The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler. + irq_lock(); + lapic_send_ipi( + &percpu_ref(arch_data)->lapic, + percpu_ref_on(cpu, arch_data)->lapic.id, + IDT_IPI_RESCHED, + APIC_DELIVERY_FIXED, + APIC_DEST_PHYSICAL, + APIC_LEVEL_ASSERT, + APIC_TRIGGER_EDGE + ); + irq_unlock(); +} + [[__naked]] void arch_sched_jump_to_context(struct arch_context* context) { asm volatile( diff --git a/include/arch/irq.h b/include/arch/irq.h index 023b190..a356453 100644 --- a/include/arch/irq.h +++ b/include/arch/irq.h @@ -5,3 +5,4 @@ bool arch_irq_set_state(bool state); bool arch_irq_get_state(); void arch_irq_wait(); +errno_t arch_irq_alloc_msi(struct msi_line** out); diff --git a/include/arch/sched.h b/include/arch/sched.h index 889efb4..a5fced7 100644 --- a/include/arch/sched.h +++ b/include/arch/sched.h @@ -3,8 +3,11 @@ #include #include +struct percpu; + void arch_sched_preempt_disable(); bool arch_sched_preempt_enable(); +void arch_sched_kick(struct percpu* cpu); struct task* arch_sched_switch(struct task* from, struct task* to); errno_t arch_task_init( struct arch_task_context* context, @@ -14,6 +17,8 @@ errno_t arch_task_init( bool is_user ); void arch_task_fini(struct arch_task_context* context); +void arch_task_fork(struct arch_task_context* dst); +void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp); [[noreturn]] void arch_sched_jump_to_context(struct arch_context* context); [[noreturn]] diff --git a/include/zinnia/irq.h b/include/zinnia/irq.h index 1a8ca45..03bd7bc 100644 --- a/include/zinnia/irq.h +++ b/include/zinnia/irq.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include