diff --git a/arch/x86_64/idt.c b/arch/x86_64/idt.c index f02a440..8a89001 100644 --- a/arch/x86_64/idt.c +++ b/arch/x86_64/idt.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -93,6 +94,10 @@ void interrupt_handler(struct arch_context* ctx) { if (arch_sched_preempt_enable()) sched_reschedule(&cpu->sched); break; + case IDT_IPI_SHOOTDOWN: + lapic_eoi(&arch_cpu->lapic); + pmap_shootdown_handle(); + break; default: if (isr < 0x20) panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error); diff --git a/arch/x86_64/pmap.c b/arch/x86_64/pmap.c index d5418a1..aa3b38c 100644 --- a/arch/x86_64/pmap.c +++ b/arch/x86_64/pmap.c @@ -1,5 +1,10 @@ +#include +#include #include #include +#include +#include +#include #include void pte_clear(pte_t* pte) { @@ -10,6 +15,26 @@ void pte_flush(uintptr_t vaddr) { asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory"); } +void pte_flush_all() { + uint64_t cr3; + asm volatile("mov %0, cr3" : "=r"(cr3)); + asm volatile("mov cr3, %0" ::"r"(cr3) : "memory"); +} + +void pte_shootdown_kick(struct percpu* cpu) { + irq_lock(); + lapic_send_ipi( + &percpu_ref(arch_data)->lapic, + percpu_ref_on(cpu, arch_data)->lapic.id, + IDT_IPI_SHOOTDOWN, + APIC_DELIVERY_FIXED, + APIC_DEST_PHYSICAL, + APIC_LEVEL_ASSERT, + APIC_TRIGGER_EDGE + ); + irq_unlock(); +} + pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) { pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT; diff --git a/arch/x86_64/sched.c b/arch/x86_64/sched.c index ba29206..1bfc345 100644 --- a/arch/x86_64/sched.c +++ b/arch/x86_64/sched.c @@ -162,6 +162,7 @@ void arch_task_fini(struct arch_task_context* context) { const uintptr_t fpu_base = (uintptr_t)context->fpu_region; for (size_t i = 0; i < fpu_size; i += page_size) pmap_unmap(&kernel_vma.pmap, fpu_base + i); + pmap_shootdown(&kernel_vma.pmap); mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size); vm_free(fpu_base, fpu_size); diff --git a/core/mem/pmap.c b/core/mem/pmap.c index 5a25b44..88e9397 100644 --- a/core/mem/pmap.c +++ b/core/mem/pmap.c @@ -1,5 +1,8 @@ #include +#include #include +#include +#include #include #include #include @@ -133,3 +136,103 @@ fail: spin_unlock(&pt->lock); return status; } + +errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out) { + spin_lock(&pt->lock); + + errno_t status = 0; + pte_t* pte = get_pte(pt, vaddr, false, true); + if (pte == nullptr || !pte_is_present(pte)) + status = ENOENT; + else + *out = pte_address(pte); + + spin_unlock(&pt->lock); + return status; +} + +errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) { + spin_lock(&pt->lock); + + errno_t status = 0; + pte_t* pte = get_pte(pt, vaddr, false, true); + if (pte == nullptr || !pte_is_present(pte)) { + status = ENOENT; + goto fail; + } + + *pte = pte_build(pte_address(pte), flags, CACHE_ANY); + pte_flush(vaddr); + +fail: + spin_unlock(&pt->lock); + return status; +} + +static void free_table(phys_t table, int8_t level) { + if (level > 0) { + pte_t* entries = HHDM_PTR(table); + const size_t num_entries = (size_t)1 << arch_mem_level_bits(); + for (size_t i = 0; i < num_entries; i++) { + if (pte_is_present(&entries[i])) + free_table(pte_address(&entries[i]), level - 1); + } + } + mem_phys_free(table, 1); +} + +void pmap_destroy(struct pmap* pt) { + if (!pt->is_user) + return; + + pte_t* root = HHDM_PTR(pt->root); + const size_t num_entries = (size_t)1 << arch_mem_level_bits(); + for (size_t i = 0; i < num_entries / 2; i++) { + if (pte_is_present(&root[i])) + free_table(pte_address(&root[i]), arch_mem_num_levels() - 2); + } + mem_phys_free(pt->root, 1); + pt->root = 0; +} + +[[__percpu]] +static atomic_size_t shootdown_req; + +[[__percpu]] +static atomic_size_t shootdown_ack; + +void pmap_shootdown_handle() { + const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE); + if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req) + return; + pte_flush_all(); + atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE); +} + +void pmap_shootdown(struct pmap* pt) { + const size_t count = percpu_count(); + if (count < 2) + return; + + irq_lock(); + struct percpu* self = percpu_get(); + for (size_t i = 0; i < count; i++) { + struct percpu* cpu = percpu_cpu(i); + if (cpu == self || !cpu->online) + continue; + atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE); + pte_shootdown_kick(cpu); + } + + for (size_t i = 0; i < count; i++) { + struct percpu* cpu = percpu_cpu(i); + if (cpu == self || !cpu->online) + continue; + const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED); + while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) { + pmap_shootdown_handle(); + asm volatile("pause"); + } + } + irq_unlock(); +} diff --git a/core/mem/stack.c b/core/mem/stack.c index 96c55fe..e1c79b7 100644 --- a/core/mem/stack.c +++ b/core/mem/stack.c @@ -52,6 +52,7 @@ void stack_destroy(struct stack* ks) { const size_t page_size = arch_mem_page_size(); for (size_t i = 0; i < ks->size; i += page_size) pmap_unmap(&kernel_vma.pmap, ks->base + i); + pmap_shootdown(&kernel_vma.pmap); mem_phys_free(ks->phys, ks->size / page_size); vm_free(ks->base - page_size, ks->size + page_size); diff --git a/core/module.c b/core/module.c index d358ddc..1c36ee8 100644 --- a/core/module.c +++ b/core/module.c @@ -204,6 +204,10 @@ static void module_free(struct module* mod) { const struct module_segment* seg = &mod->segments[i]; for (size_t off = 0; off < seg->size; off += page_size) pmap_unmap(&kernel_vma.pmap, seg->virt + off); + } + pmap_shootdown(&kernel_vma.pmap); + for (size_t i = 0; i < mod->num_segments; i++) { + const struct module_segment* seg = &mod->segments[i]; mem_phys_free(seg->phys, seg->size / page_size); } mem_free(mod->segments); diff --git a/core/percpu.c b/core/percpu.c index 069f483..2e11fd1 100644 --- a/core/percpu.c +++ b/core/percpu.c @@ -3,12 +3,11 @@ #include #include #include +#include #include +#include #include -extern uint8_t __ld_percpu_start[]; -extern uint8_t __ld_percpu_end[]; - static _Atomic(size_t) num_cpus = 1; static size_t max_cpus = SIZE_MAX; @@ -25,20 +24,33 @@ struct percpu percpu_bsp = { }; struct percpu* percpu_new() { - const size_t size = __ld_percpu_end - __ld_percpu_start; - const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size(); - - // TODO: Map pages past the end of the kernel. - // TODO: Document how allocations work. - - phys_t mem; - if (mem_phys_alloc(num_pages, 0, &mem)) + const size_t pgsz = arch_mem_page_size(); + const size_t stride = percpu_stride(); + const size_t id = atomic_load(&num_cpus, ATOMIC_RELAXED); + if (id >= (UINTPTR_MAX - (uintptr_t)__ld_percpu_start) / stride) return nullptr; - struct percpu* cpu = HHDM_PTR(mem); + phys_t mem; + if (mem_phys_alloc(stride / pgsz, 0, &mem)) + return nullptr; + + const uintptr_t base = (uintptr_t)percpu_cpu(id); + for (size_t offset = 0; offset < stride; offset += pgsz) { + if (pmap_map(&kernel_vma.pmap, base + offset, mem + offset, PTE_READ | PTE_WRITE, CACHE_ANY)) { + while (offset) { + offset -= pgsz; + pmap_unmap(&kernel_vma.pmap, base + offset); + } + mem_phys_free(mem, stride / pgsz); + return nullptr; + } + } + + struct percpu* cpu = (struct percpu*)base; cpu->self = cpu; - cpu->id = atomic_fetch_add(&num_cpus, 1, ATOMIC_RELAXED); + cpu->id = id; cpu->online = false; + atomic_store(&num_cpus, id + 1, ATOMIC_RELEASE); return cpu; } @@ -46,6 +58,10 @@ size_t percpu_max_cpus() { return max_cpus; } +size_t percpu_count() { + return atomic_load(&num_cpus, ATOMIC_RELAXED); +} + struct percpu* percpu_get() { return arch_percpu_get(); } diff --git a/include/zinnia/percpu.h b/include/zinnia/percpu.h index e0d7df4..e93fa93 100644 --- a/include/zinnia/percpu.h +++ b/include/zinnia/percpu.h @@ -5,6 +5,7 @@ #include #include #include +#include // # On using per-CPU data // @@ -37,8 +38,20 @@ struct percpu { struct scheduler sched; // Scheduler information. }; +extern uint8_t __ld_percpu_start[]; +extern uint8_t __ld_percpu_end[]; + +static inline size_t percpu_stride() { + return __ld_percpu_end - __ld_percpu_start; +} + +static inline struct percpu* percpu_cpu(size_t id) { + return (struct percpu*)(__ld_percpu_start + id * percpu_stride()); +} + struct percpu* percpu_new(); size_t percpu_max_cpus(); +size_t percpu_count(); struct percpu* percpu_get(); void percpu_bsp_init(); void percpu_init(struct boot_info* info); diff --git a/include/zinnia/pmap.h b/include/zinnia/pmap.h index 4c19d05..5e2c9e9 100644 --- a/include/zinnia/pmap.h +++ b/include/zinnia/pmap.h @@ -6,6 +6,8 @@ #include #include +struct percpu; + enum pte_flags : uint8_t { PTE_READ = 1 << 0, // Can read from this page. PTE_WRITE = 1 << 1, // Can write to this page. @@ -33,10 +35,16 @@ errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags); errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache); errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags); errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr); +errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out); +void pmap_destroy(struct pmap* pt); void pmap_set(struct pmap* pt); +void pmap_shootdown(struct pmap* pt); +void pmap_shootdown_handle(); void pte_clear(pte_t* pte); void pte_flush(uintptr_t vaddr); +void pte_flush_all(); +void pte_shootdown_kick(struct percpu* cpu); pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache); bool pte_is_present(pte_t* pte); bool pte_is_dir(pte_t* pte);