From 4a20a5b0afa4c069ae95a51074cd721d23b29a19 Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Mon, 28 Sep 2026 17:46:45 +0200 Subject: [PATCH] foo --- arch/x86_64/apic.c | 32 +++ arch/x86_64/gdt.c | 6 +- arch/x86_64/hpet.c | 2 +- arch/x86_64/idt.c | 64 ++++- arch/x86_64/include/x86_64/apic.h | 11 + arch/x86_64/include/x86_64/asm.h | 4 +- arch/x86_64/include/x86_64/gdt.h | 4 +- arch/x86_64/include/x86_64/percpu.h | 4 + arch/x86_64/include/x86_64/pmap.h | 19 ++ arch/x86_64/include/x86_64/smp.h | 37 +-- arch/x86_64/irq.c | 8 +- arch/x86_64/mem.c | 2 +- arch/x86_64/meson.build | 1 - arch/x86_64/percpu.c | 6 +- arch/x86_64/pmap.c | 18 +- arch/x86_64/sched.c | 56 +++-- arch/x86_64/smp.c | 349 +++++++++++++++++++++++++++- arch/x86_64/syscall.c | 33 ++- arch/x86_64/trampoline.S | 107 --------- arch/x86_64/tsc.c | 5 +- arch/x86_64/usercopy.c | 22 +- boot/cmdline.c | 2 + boot/limine/entry.c | 13 +- boot/main.c | 6 +- core/clock.c | 30 ++- core/console.c | 4 +- core/elf.c | 117 +++++++--- core/irq.c | 66 ++++-- core/lib/string.c | 13 +- core/mem/init.c | 5 +- core/mem/pmap.c | 12 +- core/mem/pmm.c | 65 +++++- core/mem/slab.c | 8 +- core/mem/usercopy.c | 20 ++ core/mem/virt.c | 7 +- core/mem/vma.c | 142 +++++++---- core/mem/vmo.c | 56 ++++- core/meson.build | 1 + core/percpu.c | 34 ++- core/print.c | 107 +++++---- core/refcount.c | 17 ++ core/sched/sched_dumb.c | 40 +++- core/spin.c | 10 +- dev/acpi/uacpi_glue.c | 11 +- include/arch/irq.h | 2 +- include/arch/mem.h | 17 +- include/zinnia/atomic.h | 21 -- include/zinnia/compiler.h | 1 + include/zinnia/hashmap.h | 49 ++-- include/zinnia/init.h | 5 +- include/zinnia/irq.h | 36 ++- include/zinnia/percpu.h | 3 +- include/zinnia/print.h | 2 +- include/zinnia/refcount.h | 7 +- include/zinnia/sched.h | 1 - include/zinnia/spin.h | 5 +- include/zinnia/syscall_list.inc | 4 +- include/zinnia/vec.h | 34 +-- include/zinnia/vma.h | 21 +- include/zinnia/vmo.h | 6 + include/zinnia/waitq.h | 6 +- syscalls/uname.c | 24 +- 62 files changed, 1266 insertions(+), 554 deletions(-) create mode 100644 arch/x86_64/include/x86_64/pmap.h delete mode 100644 arch/x86_64/trampoline.S create mode 100644 core/refcount.c delete mode 100644 include/zinnia/atomic.h diff --git a/arch/x86_64/apic.c b/arch/x86_64/apic.c index b618f0d..1bfa471 100644 --- a/arch/x86_64/apic.c +++ b/arch/x86_64/apic.c @@ -54,6 +54,10 @@ void lapic_init(struct local_apic* lapic) { asm_wrmsr(0x1B, apic_msr); + lapic->id = lapic_read_reg(lapic, APIC_REG_ID); + if (lapic->xapic_regs) + lapic->id >>= 24; + // Reset the TPR. lapic_write_reg(lapic, APIC_REG_TPR, 0); // Enable APIC bit in the SIVR. @@ -84,6 +88,34 @@ void lapic_eoi(struct local_apic* lapic) { lapic_write_reg(lapic, APIC_REG_EOI, 0); } +void lapic_send_ipi( + struct local_apic* lapic, + uint32_t target, + uint8_t vector, + enum apic_delivery_mode delivery_mode, + enum apic_dest_mode dest_mode, + enum apic_level level, + enum apic_trigger_mode trigger_mode +) { + uint64_t icr = vector; + icr |= (uint64_t)delivery_mode << 8; + icr |= (uint64_t)dest_mode << 11; + icr |= (uint64_t)level << 14; + icr |= (uint64_t)trigger_mode << 15; + + if (lapic->xapic_regs) + icr |= (uint64_t)(uint8_t)target << 56; + else + icr |= (uint64_t)target << 32; + + lapic_write_reg(lapic, APIC_REG_ICR, icr); + + if (lapic->xapic_regs) { + while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12)) + asm volatile("pause"); + } +} + constexpr uint16_t PIC1_COMMAND_PORT = 0x20; constexpr uint16_t PIC1_DATA_PORT = 0x21; constexpr uint16_t PIC2_COMMAND_PORT = 0xA0; diff --git a/arch/x86_64/gdt.c b/arch/x86_64/gdt.c index cc24f64..13dd36d 100644 --- a/arch/x86_64/gdt.c +++ b/arch/x86_64/gdt.c @@ -1,13 +1,13 @@ #include -#include #include +#include #include #include #include #include #include -static const struct gdt main_gdt = { +const struct gdt gdt_base = { .null = {0}, .kernel_code32 = {.val = 0x00cf9b000000ffff}, .kernel_data32 = {.val = 0x00cf93000000ffff}, @@ -26,7 +26,7 @@ static const struct gdt main_gdt = { void gdt_load() { struct gdt* gdt = &percpu_ref(arch_data)->gdt; - memcpy(gdt, &main_gdt, sizeof(main_gdt)); + memcpy(gdt, &gdt_base, sizeof(gdt_base)); struct tss* tss = &percpu_ref(arch_data)->tss; tss->iopb = sizeof(struct tss); diff --git a/arch/x86_64/hpet.c b/arch/x86_64/hpet.c index cf87521..9f9ceed 100644 --- a/arch/x86_64/hpet.c +++ b/arch/x86_64/hpet.c @@ -35,7 +35,7 @@ static void hpet_reset(struct clock* c) { static uint64_t hpet_get_elapsed_ns(struct clock* c) { struct hpet* hpet = containerof(c, struct hpet, clock); uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER); - return (counter * (uint64_t)hpet->period / 1'000'000); + return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000); } static struct hpet hpet_clock = { diff --git a/arch/x86_64/idt.c b/arch/x86_64/idt.c index 43633ba..ed567b5 100644 --- a/arch/x86_64/idt.c +++ b/arch/x86_64/idt.c @@ -5,6 +5,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -36,29 +39,68 @@ static_assert(sizeof(struct idtr) == 10); void interrupt_handler(struct arch_context* ctx) { struct percpu* cpu = percpu_get(); struct arch_percpu* arch_cpu = percpu_ref(arch_data); - bool old = irq_set_interrupted(true); - uint64_t isr = ctx->isr; switch (isr) { case IDT_PF: uint64_t cr2; asm volatile("mov %0, cr2" : "=r"(cr2)); - panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2); - case IDT_IPI_RESCHED: - arch_sched_preempt_disable(); - if (arch_sched_preempt_enable()) { - lapic_eoi(&arch_cpu->lapic); - sched_reschedule(&cpu->sched); + + const struct vma_fault_info info = { + .ip = ctx->rip, + .addr = cr2, + .caused_by_user = (ctx->cs & CPL_USER) == CPL_USER, + .caused_by_write = ctx->error & (1 << 1), + .caused_by_fetch = ctx->error & (1 << 4), + .page_was_present = ctx->error & (1 << 0), + }; + + struct task* current = cpu->sched.current; + const bool irqs_were_on = ctx->rflags & RFLAGS_IF; + if (irqs_were_on) + irq_set_state(true); + + const bool handled = current && current->space && vma_page_fault(current->space, &info); + + if (irqs_were_on) + irq_set_state(false); + + if (handled) + break; + + if (current && current->usercopy_region) { + struct usercopy_region* region = current->usercopy_region; + if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) { + ctx->rip = (uintptr_t)region->fault_ip; + break; + } } + + panic( + "Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n", + info.caused_by_write ? "write to" + : info.caused_by_fetch ? "execute on" + : "read from", + info.page_was_present ? "present" : "non-present", + info.addr, + info.ip + ); + case IDT_SPURIOUS: + break; + case IDT_IPI_RESCHED: + lapic_eoi(&arch_cpu->lapic); + arch_sched_preempt_disable(); + if (arch_sched_preempt_enable()) + sched_reschedule(&cpu->sched); break; default: + if (isr < 0x20) + panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error); + assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu\n", isr); struct irq_line* line = arch_cpu->irq_lines[isr - 0x20]; assert(line, "Unhandled interrupt on ISR %lu\n", isr); - // TODO: irq_raise(line); + irq_line_raise(line); } - - irq_set_interrupted(old); } #define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs)) diff --git a/arch/x86_64/include/x86_64/apic.h b/arch/x86_64/include/x86_64/apic.h index 18b73fd..5abdaaf 100644 --- a/arch/x86_64/include/x86_64/apic.h +++ b/arch/x86_64/include/x86_64/apic.h @@ -55,4 +55,15 @@ enum apic_regs { void lapic_init(struct local_apic* lapic); void lapic_eoi(struct local_apic* lapic); + +void lapic_send_ipi( + struct local_apic* lapic, + uint32_t target, + uint8_t vector, + enum apic_delivery_mode delivery_mode, + enum apic_dest_mode dest_mode, + enum apic_level level, + enum apic_trigger_mode trigger_mode +); + void pic_disable(); diff --git a/arch/x86_64/include/x86_64/asm.h b/arch/x86_64/include/x86_64/asm.h index 76ddcde..f5085cf 100644 --- a/arch/x86_64/include/x86_64/asm.h +++ b/arch/x86_64/include/x86_64/asm.h @@ -55,13 +55,13 @@ static inline void asm_fxrstor(const void* mem) { // Saves the FPU state to a region of memory using XSAVE. // Pointer must be 16-byte aligned. static inline void asm_xsave(void* mem) { - asm volatile("xsave [%0]" ::"r"(mem) : "memory"); + asm volatile("xsave [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory"); } // Restores the FPU state from a region of memory using XRSTOR. // Pointer must be 16-byte aligned. static inline void asm_xrstor(const void* mem) { - asm volatile("xrstor [%0]" ::"r"(mem) : "memory"); + asm volatile("xrstor [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory"); } static inline uint8_t asm_inb(uint16_t port) { diff --git a/arch/x86_64/include/x86_64/gdt.h b/arch/x86_64/include/x86_64/gdt.h index 0d7ca0c..33dad5b 100644 --- a/arch/x86_64/include/x86_64/gdt.h +++ b/arch/x86_64/include/x86_64/gdt.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include union gdt_desc { uint64_t val; @@ -50,7 +50,7 @@ struct [[__packed]] gdtr { struct gdt* base; }; -struct percpu; +extern const struct gdt gdt_base; // Initializes a GDT on the local CPU. void gdt_load(); diff --git a/arch/x86_64/include/x86_64/percpu.h b/arch/x86_64/include/x86_64/percpu.h index d0aa897..2f52038 100644 --- a/arch/x86_64/include/x86_64/percpu.h +++ b/arch/x86_64/include/x86_64/percpu.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -18,3 +19,6 @@ struct arch_percpu { }; extern struct arch_percpu arch_data; + +// Initializes the calling CPU and makes `cpu` its per-CPU block. +void x86_64_cpu_setup(struct percpu* cpu); diff --git a/arch/x86_64/include/x86_64/pmap.h b/arch/x86_64/include/x86_64/pmap.h new file mode 100644 index 0000000..70ba4ed --- /dev/null +++ b/arch/x86_64/include/x86_64/pmap.h @@ -0,0 +1,19 @@ +#pragma once +#include + +enum : pte_t { + ARCH_FLAG_PRESENT = 1 << 0, + ARCH_FLAG_READ_WRITE = 1 << 1, + ARCH_FLAG_USER_MODE = 1 << 2, + ARCH_FLAG_WRITE_THROUGH = 1 << 3, + ARCH_FLAG_CACHE_DISABLE = 1 << 4, + ARCH_FLAG_ACCESSED = 1 << 5, + ARCH_FLAG_DIRTY = 1 << 6, + ARCH_FLAG_SIZE = 1 << 7, + ARCH_FLAG_GLOBAL = 1 << 8, + ARCH_FLAG_AVAILABLE = 1 << 9, + ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10, + ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63, + + ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000, +}; diff --git a/arch/x86_64/include/x86_64/smp.h b/arch/x86_64/include/x86_64/smp.h index cdd5924..9fd5b2b 100644 --- a/arch/x86_64/include/x86_64/smp.h +++ b/arch/x86_64/include/x86_64/smp.h @@ -1,23 +1,10 @@ #pragma once -#define SMP_GDTR_OFFSET 80 -#define SMP_FARJMP_OFFSET 90 -#define SMP_TEMP_STACK_OFFSET 98 -#define SMP_TEMP_CR3_OFFSET 102 -#define SMP_ENTRY_OFFSET 106 -#define SMP_HHDM_OFFSET 114 -#define SMP_KERNEL32_DS 16 -#define SMP_KERNEL64_CS 24 -#define SMP_KERNEL64_DS 32 -#define SMP_INFO_SIZE 127 - -#ifndef __ASSEMBLER__ - +#include +#include #include #include #include -#include -#include struct [[__packed]] smp_info { struct gdt gdt; @@ -28,22 +15,10 @@ struct [[__packed]] smp_info { uint32_t temp_cr3; uint64_t entry; uint64_t hhdm_offset; + uint64_t percpu; uint32_t lapic_id; - uint8_t booted; + _Atomic(uint8_t) booted; }; -static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr)); -static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset)); -static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack)); -static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3)); -static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry)); -static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset)); -static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32)); -static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64)); -static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64)); -static_assert(SMP_INFO_SIZE == sizeof(struct smp_info)); - -// Initializes a LAPIC. -void x86_64_smp_init(uint32_t id); - -#endif +// Discovers all APs and starts them. +void x86_64_smp_init(); diff --git a/arch/x86_64/irq.c b/arch/x86_64/irq.c index f0cfb22..1179d8b 100644 --- a/arch/x86_64/irq.c +++ b/arch/x86_64/irq.c @@ -8,11 +8,13 @@ bool arch_irq_get_state() { return flags & RFLAGS_IF; } -void arch_irq_set_state(bool state) { +bool arch_irq_set_state(bool state) { + bool old = arch_irq_get_state(); if (state) - asm volatile("sti"); + asm volatile("sti" ::: "memory"); else - asm volatile("cli"); + asm volatile("cli" ::: "memory"); + return old; } void arch_irq_wait() { diff --git a/arch/x86_64/mem.c b/arch/x86_64/mem.c index bf9a401..462efd5 100644 --- a/arch/x86_64/mem.c +++ b/arch/x86_64/mem.c @@ -32,5 +32,5 @@ uintptr_t arch_mem_mapping_addr() { } bool arch_mem_addr_is_user(uintptr_t addr) { - return addr < 0x0000'8000'0000'0000; + return addr < 0x0000'7FFF'FFFF'F000; } diff --git a/arch/x86_64/meson.build b/arch/x86_64/meson.build index 1dfb718..f5902bf 100644 --- a/arch/x86_64/meson.build +++ b/arch/x86_64/meson.build @@ -20,7 +20,6 @@ kernel_sources += files( 'smp.c', 'sys.c', 'syscall.c', - 'trampoline.S', 'tsc.c', 'usercopy.c', ) diff --git a/arch/x86_64/percpu.c b/arch/x86_64/percpu.c index 49b7a99..ff16028 100644 --- a/arch/x86_64/percpu.c +++ b/arch/x86_64/percpu.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -43,7 +44,7 @@ static_assert(sizeof(union ia32_star) == 8); [[__percpu]] struct arch_percpu arch_data = {0}; -static void setup_cpu(struct percpu* cpu) { +void x86_64_cpu_setup(struct percpu* cpu) { asm_wrmsr(MSR_KERNEL_GS_BASE, 0); asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu); asm_wrmsr(MSR_FS_BASE, 0); @@ -131,7 +132,8 @@ void arch_percpu_init() { #endif // Init the BSP. - setup_cpu(&percpu_bsp); + x86_64_cpu_setup(&percpu_bsp); // Init any APs. + x86_64_smp_init(); } diff --git a/arch/x86_64/pmap.c b/arch/x86_64/pmap.c index f794929..94194af 100644 --- a/arch/x86_64/pmap.c +++ b/arch/x86_64/pmap.c @@ -1,22 +1,6 @@ #include #include - -enum : pte_t { - ARCH_FLAG_PRESENT = 1 << 0, - ARCH_FLAG_READ_WRITE = 1 << 1, - ARCH_FLAG_USER_MODE = 1 << 2, - ARCH_FLAG_WRITE_THROUGH = 1 << 3, - ARCH_FLAG_CACHE_DISABLE = 1 << 4, - ARCH_FLAG_ACCESSED = 1 << 5, - ARCH_FLAG_DIRTY = 1 << 6, - ARCH_FLAG_SIZE = 1 << 7, - ARCH_FLAG_GLOBAL = 1 << 8, - ARCH_FLAG_AVAILABLE = 1 << 9, - ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10, - ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63, - - ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000, -}; +#include void pte_clear(pte_t* pte) { *pte = 0; diff --git a/arch/x86_64/sched.c b/arch/x86_64/sched.c index babdf01..1ea6dd5 100644 --- a/arch/x86_64/sched.c +++ b/arch/x86_64/sched.c @@ -25,30 +25,30 @@ struct task_frame { static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) { asm volatile( "sub rsp, 0x30\n" // Make room for all regs (except RIP). - "mov [rsp + %c0], rbx\n" - "mov [rsp + %c1], rbp\n" - "mov [rsp + %c2], r12\n" - "mov [rsp + %c3], r13\n" - "mov [rsp + %c4], r14\n" - "mov [rsp + %c5], r15\n" + "mov [rsp + %c[rbx]], rbx\n" + "mov [rsp + %c[rbp]], rbp\n" + "mov [rsp + %c[r12]], r12\n" + "mov [rsp + %c[r13]], r13\n" + "mov [rsp + %c[r14]], r14\n" + "mov [rsp + %c[r15]], r15\n" "mov [rdi], rsp\n" // rdi = old_rsp "mov rsp, rsi\n" // rsi = new_rsp - "mov rbx, [rsp + %c0]\n" - "mov rbp, [rsp + %c1]\n" - "mov r12, [rsp + %c2]\n" - "mov r13, [rsp + %c3]\n" - "mov r14, [rsp + %c4]\n" - "mov r15, [rsp + %c5]\n" + "mov rbx, [rsp + %c[rbx]]\n" + "mov rbp, [rsp + %c[rbp]]\n" + "mov r12, [rsp + %c[r12]]\n" + "mov r13, [rsp + %c[r13]]\n" + "mov r14, [rsp + %c[r14]]\n" + "mov r15, [rsp + %c[r15]]\n" "add rsp, 0x30\n" "mov rax, rdx\n" "ret" // This will conveniently move us to the RIP we put at this stack entry. : - : "i"(offsetof(struct task_frame, rbx)), - "i"(offsetof(struct task_frame, rbp)), - "i"(offsetof(struct task_frame, r12)), - "i"(offsetof(struct task_frame, r13)), - "i"(offsetof(struct task_frame, r14)), - "i"(offsetof(struct task_frame, r15)) + : [rbx] "i"(offsetof(struct task_frame, rbx)), + [rbp] "i"(offsetof(struct task_frame, rbp)), + [r12] "i"(offsetof(struct task_frame, r12)), + [r13] "i"(offsetof(struct task_frame, r13)), + [r14] "i"(offsetof(struct task_frame, r14)), + [r15] "i"(offsetof(struct task_frame, r15)) : "memory" ); } @@ -60,7 +60,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) { cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE; - if (from->user_stack) { + if (from_ctx->fpu_region != nullptr) { cpu->fpu_save(from_ctx->fpu_region); from_ctx->ds = asm_read_ds(); from_ctx->es = asm_read_es(); @@ -70,7 +70,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) { from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE); } - if (to->user_stack) { + if (to_ctx->fpu_region != nullptr) { cpu->fpu_restore(to_ctx->fpu_region); asm_write_ds(to_ctx->ds); asm_write_es(to_ctx->es); @@ -124,16 +124,14 @@ errno_t arch_task_init( uintptr_t fpu_base = vm_alloc(fpu_size); if (!fpu_base) return ENOMEM; - for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) { - phys_t page; - errno_t s = mem_phys_alloc(1, 0, &page); - if (s) { - vm_free(fpu_base, fpu_size); - return s; - } - - pmap_map(&kernel_vma.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK); + phys_t pages; + errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages); + if (s) { + vm_free(fpu_base, fpu_size); + return s; } + for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) + pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK); context->fpu_region = (void*)fpu_base; context->ds = asm_read_ds(); diff --git a/arch/x86_64/smp.c b/arch/x86_64/smp.c index fd8aa79..5b311ef 100644 --- a/arch/x86_64/smp.c +++ b/arch/x86_64/smp.c @@ -1,17 +1,352 @@ +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include extern uint8_t smp_trampoline_start[]; +extern uint8_t smp_trampoline_data[]; extern uint8_t smp_trampoline_end[]; -[[noreturn]] -static void ap_entry(phys_t info) { - // TODO +[[__naked, __used]] +static void smp_trampoline() { + asm volatile( + ".pushsection .rodata\n" + ".code16\n" + ".global smp_trampoline_start\n" + "smp_trampoline_start:\n" + " cli\n" + " cld\n" + " jmp .Lmode16\n" - while (1) { - asm volatile("hlt"); - } + ".global smp_trampoline_data\n" + "smp_trampoline_data:\n" + " .skip %c[info_size]\n" + + ".set .Ldata_offset, (smp_trampoline_data - smp_trampoline_start)\n" + ".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n" + ".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n" + ".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n" + ".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n" + ".set .Lentry_offset, (.Ldata_offset + %c[entry])\n" + ".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n" + + ".Lmode16:\n" + " mov bx, cs\n" + " shl ebx, 4\n" + + " lidtd cs:(.Linvalid_idtr - smp_trampoline_start)\n" + " lgdtd cs:.Lgdtr_offset\n" + + " lea eax, [ebx + (.Lmode32 - smp_trampoline_start)]\n" + " mov dword ptr cs:.Lfarjmp_offset, eax\n" + + " mov eax, %c[cr0]\n" + " mov cr0, eax\n" + " jmp fword ptr cs:.Lfarjmp_offset\n" + + ".code32\n" + ".Lmode32:\n" + " mov ax, %c[kernel32_ds]\n" + " mov ds, ax\n" + " mov es, ax\n" + " mov fs, ax\n" + " mov gs, ax\n" + " mov ss, ax\n" + + " xor eax, eax\n" + " lldt ax\n" + + " mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n" + + " xor eax, eax\n" + " mov cr4, eax\n" + + " or eax, %c[cr4_pae]\n" + " mov cr4, eax\n" + + " mov ecx, %c[msr_efer]\n" + " mov eax, %c[efer_lme]\n" + " xor edx, edx\n" + " wrmsr\n" + + " mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n" + " mov cr3, eax\n" + + " mov eax, cr0\n" + " or eax, %c[cr0_pg]\n" + " mov cr0, eax\n" + + " lea eax, [ebx + (.Lmode64 - smp_trampoline_start)]\n" + " push %c[kernel64_cs]\n" + " push eax\n" + " retf\n" + + ".code64\n" + ".Lmode64:\n" + " mov ax, %c[kernel64_ds]\n" + " mov ds, ax\n" + " mov es, ax\n" + " mov fs, ax\n" + " mov gs, ax\n" + " mov ss, ax\n" + + " mov ecx, %c[msr_efer]\n" + " rdmsr\n" + " or eax, %c[efer_nxe]\n" + " wrmsr\n" + + " xor ebp, ebp\n" + + " lea rdi, [rbx + .Ldata_offset]\n" + " mov rax, [rbx + .Lentry_offset]\n" + " add rsp, [rbx + .Lhhdm_offset]\n" + " jmp rax\n" + + ".Linvalid_idtr:\n" + " .word 0\n" + " .quad 0\n" + + ".global smp_trampoline_end\n" + "smp_trampoline_end:\n" + ".popsection\n" + " ud2\n" + : + : [info_size] "i"(sizeof(struct smp_info)), + [gdtr] "i"(offsetof(struct smp_info, gdtr)), + [farjmp] "i"(offsetof(struct smp_info, farjmp_offset)), + [temp_stack] "i"(offsetof(struct smp_info, temp_stack)), + [temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)), + [entry] "i"(offsetof(struct smp_info, entry)), + [hhdm] "i"(offsetof(struct smp_info, hhdm_offset)), + [kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)), + [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)), + [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)), + [cr0] "i"(CR0_PE | CR0_ET), + [cr0_pg] "i"(CR0_PG), + [cr4_pae] "i"(CR4_PAE), + [msr_efer] "i"(MSR_EFER), + [efer_lme] "i"(MSR_EFER_LME), + [efer_nxe] "i"(MSR_EFER_NXE) + ); } -void x86_64_smp_init(uint32_t id) {} +constexpr size_t AP_BOOT_STACK_SIZE = 0x8000; +constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000; + +[[noreturn]] +static void ap_entry(phys_t info_addr) { + // We're still running on the temporary page table. + pmap_set(&kernel_vma.pmap); + + struct smp_info* info = HHDM_PTR(info_addr); + struct percpu* cpu = (struct percpu*)info->percpu; + + x86_64_cpu_setup(cpu); + irq_lock(); + + // This task keeps running on the boot stack, but is never scheduled again after the first yield. + sched_init(&cpu->sched); + + // Let the BSP know that we're alive. + // `info` must not be touched after this point. + atomic_store_explicit(&info->booted, 1, memory_order_release); + + log("smp: CPU %zu online\n", cpu->id); + + sched_yield(&cpu->sched); + unreachable(); +} + +static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) { + const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start; + + phys_t stack; + if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) { + log("smp: Failed to allocate a boot stack for LAPIC %u\n", lapic_id); + return false; + } + + struct percpu* cpu = percpu_new(); + if (!cpu) { + log("smp: Failed to allocate per-CPU data for LAPIC %u\n", lapic_id); + mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size()); + return false; + } + + log("smp: Starting CPU %zu (LAPIC %u)\n", cpu->id, lapic_id); + + memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size); + + const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start); + struct smp_info* info = HHDM_PTR(info_addr); + memcpy(&info->gdt, &gdt_base, sizeof(info->gdt)); + info->gdtr.limit = sizeof(struct gdt) - 1; + info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt)); + info->farjmp_offset = 0; // Filled in by the trampoline. + info->farjmp_segment = offsetof(struct gdt, kernel_code32); + info->temp_stack = stack + AP_BOOT_STACK_SIZE; + info->temp_cr3 = temp_cr3; + info->entry = (uint64_t)ap_entry; + info->hhdm_offset = mem_hhdm_base; + info->percpu = (uint64_t)cpu; + info->lapic_id = lapic_id; + atomic_init(&info->booted, 0); + + struct local_apic* lapic = &percpu_ref(arch_data)->lapic; + + lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE); + clock_spin_ns(10'000'000); + + lapic_send_ipi( + lapic, + lapic_id, + trampoline >> 12, + APIC_DELIVERY_STARTUP, + APIC_DEST_PHYSICAL, + APIC_LEVEL_ASSERT, + APIC_TRIGGER_EDGE + ); + clock_spin_ns(10'000'000); + + for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) { + if (waited >= AP_BOOT_TIMEOUT_NS) { + // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up. + log("smp: CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id); + return false; + } + clock_spin_ns(1'000'000); + } + + return true; +} + +static errno_t build_temp_table(phys_t* out) { + const size_t pgsz = arch_mem_page_size(); + + phys_t mem; + errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem); + if (status) + return status; + + uint64_t* l4 = HHDM_PTR(mem); + uint64_t* l3 = HHDM_PTR(mem + pgsz); + const phys_t l2_base = mem + 2 * pgsz; + + // Use 2 MiB pages. + l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE; + for (size_t i = 0; i < 4; i++) { + uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz); + l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE; + for (size_t j = 0; j < 512; j++) + l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE; + } + + // Copy over the higher half from the kernel page table. + const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root); + for (size_t i = 256; i < 512; i++) + l4[i] = kernel_l4[i]; + + *out = mem; + return 0; +} + +void x86_64_smp_init() { + const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start; + assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page\n"); + + const size_t max_cpus = percpu_max_cpus(); + if (max_cpus <= 1) + return; + + uacpi_table table; + if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) { + log("smp: No MADT found, not starting any APs\n"); + return; + } + + phys_t trampoline; + if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) { + log("smp: Failed to allocate 20-bit memory for the AP trampoline\n"); + uacpi_table_unref(&table); + return; + } + + phys_t temp_table; + if (build_temp_table(&temp_table)) { + log("smp: Failed to allocate a temporary page table\n"); + mem_phys_free(trampoline, 1); + uacpi_table_unref(&table); + return; + } + + const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id; + bool all_booted = true; + size_t num_cpus = 1; + + const struct acpi_madt* madt = table.ptr; + const uint8_t* entry = (const uint8_t*)madt->entries; + const uint8_t* end = (const uint8_t*)madt + madt->hdr.length; + + while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) { + const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry; + + // Don't trust malformed firmware tables. + if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) { + log("smp: MADT contains a malformed entry\n"); + break; + } + + uint32_t lapic_id = 0; + bool enabled = false; + switch (hdr->type) { + case ACPI_MADT_ENTRY_TYPE_LAPIC: + const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr; + lapic_id = lapic->id; + enabled = lapic->flags & ACPI_PIC_ENABLED; + break; + case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC: + const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr; + lapic_id = x2apic->id; + enabled = x2apic->flags & ACPI_PIC_ENABLED; + break; + } + + if (enabled && lapic_id != bsp_id) { + if (start_ap(trampoline, temp_table, lapic_id)) + num_cpus++; + else + all_booted = false; + } + + entry += hdr->length; + } + + uacpi_table_unref(&table); + + // An AP which didn't respond might still be using these. + if (all_booted) { + mem_phys_free(temp_table, 6); + mem_phys_free(trampoline, 1); + } + + log("smp: %zu CPU(s) online\n", num_cpus); +} diff --git a/arch/x86_64/syscall.c b/arch/x86_64/syscall.c index 7bff4e6..9b73b1c 100644 --- a/arch/x86_64/syscall.c +++ b/arch/x86_64/syscall.c @@ -12,17 +12,17 @@ void arch_syscall_stub() { asm volatile( " swapgs\n" - " mov gs:%c0, rsp\n" // user_stack - " mov rsp, gs:%c1\n" // kernel_stack + " mov gs:%c[user_stack], rsp\n" + " mov rsp, gs:%c[kernel_stack]\n" " cld\n" "1:\n" - " push %c3\n" // ss - " push gs:%c0\n" - " push r11\n" // rflags - " push %c2\n" // cs - " push rcx\n" // rip - " push 0x00\n" // arch_context::error - " push 0x00\n" // arch_context::isr + " push %c[user_data]\n" // ss + " push gs:%c[user_stack]\n" + " push r11\n" // rflags + " push %c[user_code64]\n" // cs + " push rcx\n" // rip + " push 0x00\n" // arch_context::error + " push 0x00\n" // arch_context::isr " push rax\n" " push rbx\n" " push rcx\n" @@ -39,8 +39,8 @@ void arch_syscall_stub() { " push r14\n" " push r15\n" " xor rbp, rbp\n" - " mov rdi, rsp\n" // SysV ABI first argument. - " call %4\n" // Call syscall handler. + " mov rdi, rsp\n" // SysV ABI first argument. + " call syscall_dispatch\n" // Call syscall handler. " cli\n" " pop r15\n" " pop r14\n" @@ -59,14 +59,13 @@ void arch_syscall_stub() { " pop rax\n" " add rsp, 0x10\n" // Skip .error and .isr fields "2:\n" - " mov rsp, gs:%c0\n" // user_stack + " mov rsp, gs:%c[user_stack]\n" " swapgs\n" " sysretq\n" : - : "i"(offsetof(struct percpu, user_stack)), // 0 - "i"(offsetof(struct percpu, kernel_stack)), // 1 - "i"(offsetof(struct gdt, user_code64) | CPL_USER), // 2 - "i"(offsetof(struct gdt, user_data) | CPL_USER), // 3 - "r"(syscall_dispatch) // 4 + : [user_stack] "i"(offsetof(struct percpu, user_stack)), + [kernel_stack] "i"(offsetof(struct percpu, kernel_stack)), + [user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER), + [user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER) ); } diff --git a/arch/x86_64/trampoline.S b/arch/x86_64/trampoline.S deleted file mode 100644 index f60b280..0000000 --- a/arch/x86_64/trampoline.S +++ /dev/null @@ -1,107 +0,0 @@ -#include -#include - -.intel_syntax noprefix - -.global smp_trampoline_start -.global smp_trampoline_end - -.section .rodata -.code16 - -smp_trampoline_start: - cli - cld - jmp 1f - -smp_trampoline_data: - .skip SMP_INFO_SIZE - -.set data_offset, (smp_trampoline_data - smp_trampoline_start) -.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET) -.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET) -.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET) -.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET) -.set entry_offset, (data_offset + SMP_ENTRY_OFFSET) -.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET) - -1: - mov bx, cs - shl ebx, 4 - -.set idtr_offset, (invalid_idtr - smp_trampoline_start) - lidtd cs:idtr_offset - lgdtd cs:gdtr_offset - -.set mode32_offset, (mode32 - smp_trampoline_start) - lea eax, [ebx + mode32_offset] - mov dword ptr cs:farjmp_offset, eax - - mov eax, CR0_PE | CR0_ET - mov cr0, eax - jmp fword ptr cs:farjmp_offset - -.code32 -mode32: - mov ax, SMP_KERNEL32_DS - mov ds, ax - mov es, ax - mov fs, ax - mov gs, ax - mov ss, ax - - xor eax, eax - lldt ax - - mov esp, dword ptr [ebx + temp_stack_offset] - - xor eax, eax - mov cr4, eax - - or eax, CR4_PAE - mov cr4, eax - - mov ecx, MSR_EFER - mov eax, MSR_EFER_LME - xor edx, edx - wrmsr - - mov eax, dword ptr [ebx + temp_cr3_offset] - mov cr3, eax - - mov eax, cr0 - or eax, CR0_PG - mov cr0, eax - -.set mode64_offset, (mode64 - smp_trampoline_start) - lea eax, [ebx + mode64_offset] - push SMP_KERNEL64_CS - push eax - retf - -.code64 -mode64: - mov ax, SMP_KERNEL64_DS - mov ds, ax - mov es, ax - mov fs, ax - mov gs, ax - mov ss, ax - - mov ecx, MSR_EFER - rdmsr - or eax, MSR_EFER_NXE - wrmsr - - xor ebp, ebp - - lea rdi, [rbx + data_offset] - mov rax, [rbx + entry_offset] - add rsp, [rbx + hhdm_offset] - jmp rax - -invalid_idtr: - .word 0 - .quad 0 - -smp_trampoline_end: diff --git a/arch/x86_64/tsc.c b/arch/x86_64/tsc.c index fd3b272..d3e2f02 100644 --- a/arch/x86_64/tsc.c +++ b/arch/x86_64/tsc.c @@ -53,8 +53,9 @@ void tsc_init() { // Check for the TSC info leaf. bool has_tsc_leaf = false; struct cpuid tsc_leaf = {0}; - if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) { + if (asm_cpuid(0, 0).eax >= 0x15) { tsc_leaf = asm_cpuid(0x15, 0); + has_tsc_leaf = true; } if (clock_available()) { @@ -69,7 +70,7 @@ void tsc_init() { // since at this point we have at least the HPET timer. if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) { log("x86_64/tsc: Calibrating TSC using CPUID 0x15"); - tsc_freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax; + tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax; } else { log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information"); return; diff --git a/arch/x86_64/usercopy.c b/arch/x86_64/usercopy.c index dc367ff..88d1266 100644 --- a/arch/x86_64/usercopy.c +++ b/arch/x86_64/usercopy.c @@ -1,7 +1,12 @@ #include +#include #include +#include void arch_usercopy_allow(bool state) { + if (!percpu_ref(arch_data)->can_smap) + return; + if (state) { asm volatile("stac" ::: "memory"); } else { @@ -24,7 +29,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us asm volatile( // Setup regs for `rep movsb`. " xchg rcx, rdx\n" - " mov rax, [rip + %0]\n" + " lea rax, [rip + %0]\n" " xchg [rdx], rax\n" ".global x86_64_read_start\n" @@ -41,7 +46,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us ".global x86_64_read_fault\n" "x86_64_read_fault:\n" " xor rax, rax\n" - " xchg [rdx], rax\n" + " mov [rdx], rax\n" " ret\n" : : "i"(&read_region) @@ -64,7 +69,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u asm volatile( // Setup regs for `rep movsb`. " xchg rcx, rdx\n" - " mov rax, [rip + %0]\n" + " lea rax, [rip + %0]\n" " mov [rdx], rax\n" ".global x86_64_write_start\n" @@ -81,7 +86,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u ".global x86_64_write_fault\n" "x86_64_write_fault:\n" " xor rax, rax\n" - " xchg [rdx], rax\n" + " mov [rdx], rax\n" " ret\n" : : "i"(&write_region) @@ -102,18 +107,19 @@ static const struct usercopy_region strlen_region = { [[__naked]] bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) { asm volatile( - " mov rax, [rip + %0]\n" + " lea rax, [rip + %0]\n" " xchg [rcx], rax\n" ".global x86_64_strlen_start\n" " x86_64_strlen_start:\n" " xor r8, r8\n" ".Lloop:\n" + " cmp r8, rsi\n" + " je .Lleave\n" " cmp byte ptr [rdi + r8], 0\n" " je .Lleave\n" " inc r8\n" - " cmp rsi, r8\n" - " jne .Lloop\n" + " jmp .Lloop\n" ".Lleave:\n" " mov [rdx], r8\n" @@ -127,7 +133,7 @@ bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struc ".global x86_64_strlen_fault\n" "x86_64_strlen_fault:\n" " xor rax, rax\n" - " xchg [rcx], rax\n" + " mov [rcx], rax\n" " ret\n" : : "i"(&strlen_region) diff --git a/boot/cmdline.c b/boot/cmdline.c index 34371fb..f299cb8 100644 --- a/boot/cmdline.c +++ b/boot/cmdline.c @@ -65,6 +65,8 @@ void cmdline_parse(const char* cmdline) { while (opt < __ld_cmdline_end) { const struct cmdline_option* current = *opt; if (!strcmp(current->name, name)) { + if (value == nullptr) + break; switch (current->option_type) { case CMDLINE_STRING: current->func_str(value); diff --git a/boot/limine/entry.c b/boot/limine/entry.c index 2928462..885466b 100644 --- a/boot/limine/entry.c +++ b/boot/limine/entry.c @@ -39,7 +39,7 @@ static struct boot_file files[8]; void kernel_entry() { kernel_early_init(); - struct boot_info info; + struct boot_info info = {}; struct limine_memmap_response* mm_res = memmap_request.response; struct limine_module_response* module_res = module_request.response; @@ -76,7 +76,8 @@ void kernel_entry() { for (size_t i = 0; i < info.num_files; i++) { files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset; files[i].length = module_res->modules[i]->size; - strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path)); + strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1); + files[i].path[sizeof(files[i].path) - 1] = 0; } } @@ -85,7 +86,13 @@ void kernel_entry() { info.virt_base = (uintptr_t)exec_res->virtual_base; info.hhdm_base = (uintptr_t)hhdm_request.response->offset; - info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base; +#if CONFIG_ACPI + if (rsdp_request.response) { + info.rsdp = (phys_t)rsdp_request.response->address; + if (info.rsdp >= info.hhdm_base) + info.rsdp -= info.hhdm_base; + } +#endif kernel_main(&info); } diff --git a/boot/main.c b/boot/main.c index da38e4c..2440256 100644 --- a/boot/main.c +++ b/boot/main.c @@ -79,13 +79,13 @@ void kernel_main(struct boot_info* info) { mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base); - percpu_init(); - sched_init(&percpu_get()->sched); - #if CONFIG_ACPI acpi_init(info); #endif + percpu_init(); + sched_init(&percpu_get()->sched); + struct task* main_task; task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task); sched_add_task(&percpu_get()->sched, main_task); diff --git a/core/clock.c b/core/clock.c index 483cca4..17d63a8 100644 --- a/core/clock.c +++ b/core/clock.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -9,16 +8,25 @@ static struct clock* active_clock = nullptr; static uint64_t counter_base = 0; static struct spinlock lock = {}; +static uint64_t clock_get_elapsed_ns_locked() { + if (__unlikely(!active_clock)) + return 0; + + return counter_base + active_clock->get_elapsed_ns(active_clock); +} + bool clock_available() { - return active_clock != nullptr; + spin_lock(&lock); + bool available = active_clock != nullptr; + spin_unlock(&lock); + return available; } uint64_t clock_get_elapsed_ns() { - struct clock* active = atomic_load(&active_clock, ATOMIC_RELAXED); - if (__unlikely(!active)) - return 0; - - return atomic_load(&counter_base, ATOMIC_RELAXED) + active->get_elapsed_ns(active); + spin_lock(&lock); + uint64_t elapsed = clock_get_elapsed_ns_locked(); + spin_unlock(&lock); + return elapsed; } bool clock_switch(struct clock* clock) { @@ -33,16 +41,18 @@ bool clock_switch(struct clock* clock) { log("clock: Switching to clock source \"%s\"\n", clock->name); - uint64_t elapsed = clock_get_elapsed_ns(); - counter_base += elapsed; + uint64_t elapsed = clock_get_elapsed_ns_locked(); + clock->reset(clock); + counter_base = elapsed; active_clock = clock; + spin_unlock(&lock); return true; } void clock_spin_ns(uint64_t ns) { - if (active_clock == nullptr) { + if (!clock_available()) { log("clock: Unable to sleep for %lu ns, no clock available!\n", ns); return; } diff --git a/core/console.c b/core/console.c index be4e6b5..faeb189 100644 --- a/core/console.c +++ b/core/console.c @@ -35,8 +35,10 @@ void console_init() { active = __ld_console_start[0]; if (active) { - if (active->init(active) == false) + if (active->init(active) == false) { active = nullptr; + return; + } log("console: Logging on console \"%s\"\n", active->name); } diff --git a/core/elf.c b/core/elf.c index edd03b3..4a8b341 100644 --- a/core/elf.c +++ b/core/elf.c @@ -7,11 +7,12 @@ #include #include #include +#include #include #include #include #include -#include +#include struct exec_info { struct vma* space; @@ -21,11 +22,15 @@ struct exec_info { }; errno_t elf_load(struct exec_info* info, struct task** out) { - errno_t status; + errno_t err; size_t read; struct elf_ehdr ehdr = {}; - vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read); + err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read); + if (err) + return err; + if (read != sizeof(ehdr)) + return ENOEXEC; // Check if the file is an ELF. if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0) @@ -40,13 +45,48 @@ errno_t elf_load(struct exec_info* info, struct task** out) { return EINVAL; if (ehdr.e_type != ET_EXEC) return EINVAL; + if (!arch_mem_addr_is_user(ehdr.e_entry)) + return ENOEXEC; + + const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize; + uintptr_t phdrs_end; + if (ehdr.e_phentsize < sizeof(struct elf_phdr) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024) + return ENOEXEC; + if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size)) + return ENOEXEC; const size_t page_size = arch_mem_page_size(); + uintptr_t phdr_addr = 0; for (size_t i = 0; i < ehdr.e_phnum; i++) { struct elf_phdr phdr = {}; - vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read); + err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read); + if (err) + return err; + if (read != sizeof(phdr)) + return ENOEXEC; + + if (phdr.p_type == PT_PHDR) + phdr_addr = phdr.p_vaddr; + if (phdr.p_type == PT_LOAD) { + if (phdr.p_memsz == 0) + continue; + if (phdr.p_filesz > phdr.p_memsz) + return ENOEXEC; + + uintptr_t seg_end; + if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz)) + return ENOEXEC; + if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1)) + return ENOEXEC; + + if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align) + return ENOEXEC; + + if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz) + phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset); + enum prot_flags prot = 0; if (phdr.p_flags & PF_R) prot |= PROT_READ; @@ -55,8 +95,6 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if (phdr.p_flags & PF_X) prot |= PROT_EXEC; - assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, ""); - const uintptr_t misalign = phdr.p_vaddr & (page_size - 1); const uintptr_t map_address = phdr.p_vaddr - misalign; const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1); @@ -64,18 +102,21 @@ errno_t elf_load(struct exec_info* info, struct task** out) { // Copy the file data into its own mapping. struct paged_vmo* phdr_obj; - status = vmo_new_phys(&phdr_obj); - if (status) - return status; + err = vmo_new_phys(&phdr_obj); + if (err) + return err; - status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr); - if (status) - return status; + err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr); + if (err) { + vmo_unref(&phdr_obj->object); + return err; + } // We map more than we copied so the rest is filled with zeroed pages. - status = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset); - if (status) - return status; + err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset); + vmo_unref(&phdr_obj->object); + if (err) + return err; } } @@ -85,9 +126,9 @@ errno_t elf_load(struct exec_info* info, struct task** out) { // Allocate stack. struct paged_vmo* stack; - status = vmo_new_phys(&stack); - if (status) - return status; + err = vmo_new_phys(&stack); + if (err) + return err; // Fill in stack info. uintptr_t stack_off = stack_size; @@ -101,8 +142,21 @@ errno_t elf_load(struct exec_info* info, struct task** out) { for (const char** ptr = info->argv; *ptr != nullptr; ptr++) { num_argv++; } + + size_t strings_size = 0; + for (size_t env = 0; env < num_envp; env++) + strings_size += strlen(info->envp[env]) + 1; + for (size_t arg = 0; arg < num_argv; arg++) + strings_size += strlen(info->argv[arg]) + 1; + uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0); uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0); + if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) { + mem_free(envp_offsets); + mem_free(argv_offsets); + vmo_unref(&stack->object); + return ENOMEM; + } for (size_t env = 0; env < num_envp; env++) { const char nul = 0; @@ -153,22 +207,24 @@ errno_t elf_load(struct exec_info* info, struct task** out) { WRITE_AUXV(AT_PHNUM, ehdr.e_phnum); WRITE_AUXV(AT_PHENT, ehdr.e_phentsize); WRITE_AUXV(AT_ENTRY, ehdr.e_entry); + if (phdr_addr) + WRITE_AUXV(AT_PHDR, phdr_addr); // envp pointers. stack_off -= sizeof(uintptr_t); const uintptr_t zero = 0; vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); - for (size_t env = 0; env < num_envp; env++) { + for (size_t env = num_envp; env > 0; env--) { stack_off -= sizeof(uintptr_t); - vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr); + vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr); } // argv pointers. stack_off -= sizeof(uintptr_t); vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); - for (size_t arg = 0; arg < num_argv; arg++) { + for (size_t arg = num_argv; arg > 0; arg--) { stack_off -= sizeof(uintptr_t); - vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr); + vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr); } mem_free(envp_offsets); @@ -179,19 +235,24 @@ errno_t elf_load(struct exec_info* info, struct task** out) { uintptr_t argc = num_argv; vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr); - status = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0); - if (status) - return status; + err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0); + vmo_unref(&stack->object); + if (err) + return err; struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0); + if (!start_info) + return ENOMEM; start_info->ip = ehdr.e_entry; start_info->sp = highest - stack_size + stack_off; start_info->arg = 0; struct task* result; - status = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result); - if (status) - return status; + err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result); + if (err) { + mem_free(start_info); + return err; + } *out = result; return 0; diff --git a/core/irq.c b/core/irq.c index 24d54bf..d621c3a 100644 --- a/core/irq.c +++ b/core/irq.c @@ -1,10 +1,15 @@ -#include +#include #include +#include #include +#include #include -void irq_set_state(bool state) { - arch_irq_set_state(state); +[[__percpu]] +struct irq_percpu irq_data; + +bool irq_set_state(bool state) { + return arch_irq_set_state(state); } bool irq_get_state() { @@ -16,26 +21,49 @@ void irq_wait() { } void irq_lock() { - struct irq_percpu* cpu = &percpu_get()->irq; - if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE)) - return; - - arch_irq_set_state(false); - atomic_fetch_add(&percpu_get()->irq.level, 1, ATOMIC_ACQ_REL); + bool prev = arch_irq_set_state(false); + struct irq_percpu* state = percpu_ref(irq_data); + if (state->depth++ == 0) + state->saved_state = prev; } void irq_unlock() { - struct irq_percpu* cpu = &percpu_get()->irq; - if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE)) - return; - - uint32_t old_level = atomic_fetch_sub(&cpu->level, 1, ATOMIC_ACQ_REL); - // If it was 1, the new IRQ level is now 0. - if (old_level == 1) { + struct irq_percpu* state = percpu_ref(irq_data); + if (--state->depth == 0 && state->saved_state) arch_irq_set_state(true); - } } -bool irq_set_interrupted(bool is_interrupted) { - return atomic_exchange(&percpu_get()->irq.in_interrupt, is_interrupted, ATOMIC_RELEASE); +void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) { + irq_lock(); + line->mode = line->set_config(line, trigger, polarity); + irq_unlock(); +} + +void irq_line_raise(struct irq_line* line) { + assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled"); + + irq_lock(); + line->is_busy = true; + + bool claimed = false; + struct irq_handler* handler; + list_foreach(handler, &line->handlers, next) { + if (handler->raise(handler) == IRQ_STATUS_HANDLED) + claimed = true; + } + + if (!claimed) + log("irq: Spurious interrupt\n"); + + if ((line->mode & IRQ_MODE_EOI) && line->eoi) + line->eoi(line); + + line->is_busy = false; + irq_unlock(); +} + +void irq_line_attach(struct irq_line* line, struct irq_handler* handler) { + irq_lock(); + list_insert_head(&line->handlers, handler, next); + irq_unlock(); } diff --git a/core/lib/string.c b/core/lib/string.c index e07ed06..3539d62 100644 --- a/core/lib/string.c +++ b/core/lib/string.c @@ -24,15 +24,14 @@ size_t strnlen(const char* str, size_t len) { [[__hot]] int memcmp(const void* s1, const void* s2, size_t size) { - int diff = 0; const uint8_t* s1ptr = s1; const uint8_t* s2ptr = s2; for (size_t i = 0; i < size; i++) { if (s1ptr[i] != s2ptr[i]) - diff++; + return s1ptr[i] - s2ptr[i]; } - return diff; + return 0; } [[__hot]] @@ -103,7 +102,7 @@ void* memset(void* dest, int value, size_t n) { } size_t* qword_dest = (size_t*)d; - size_t qword_value = value; + size_t qword_value = (uint8_t)value; qword_value |= (qword_value << 8); qword_value |= (qword_value << 16); qword_value |= (qword_value << 32); @@ -126,8 +125,10 @@ void* memset(void* dest, int value, size_t n) { } char* strncpy(char* restrict dst, const char* restrict src, size_t len) { - size_t src_len = strnlen(src, len) + 1; - return memcpy(dst, src, MIN(len, src_len)); + size_t src_len = strnlen(src, len); + memcpy(dst, src, src_len); + memset(dst + src_len, 0, len - src_len); + return dst; } int strncmp(const char* str1, const char* str2, size_t len) { diff --git a/core/mem/init.c b/core/mem/init.c index 7ba512f..156b9ef 100644 --- a/core/mem/init.c +++ b/core/mem/init.c @@ -64,10 +64,11 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_ mem_hhdm_base = tmp_hhdm; assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table\n"); + tailq_init(&kernel_vma.mappings); // text log("mem: Mapping text segment at %p\n", __ld_text_start); - for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) { + for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) { errno_t status = pmap_map( &kernel_vma.pmap, (uintptr_t)p, @@ -116,7 +117,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_ for (size_t i = 0; i < map_len; i++) { if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC) continue; - for (size_t p = 0; p <= map[i].length; p += pgsz) { + for (size_t p = 0; p < map[i].length; p += pgsz) { uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm); phys_t paddr = (phys_t)(map[i].address + p); errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY); diff --git a/core/mem/pmap.c b/core/mem/pmap.c index ae4d88c..6e5e6b7 100644 --- a/core/mem/pmap.c +++ b/core/mem/pmap.c @@ -17,6 +17,16 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) { if (status) return status; + pte_t* root = HHDM_PTR(addr); + const size_t num_entries = (size_t)1 << arch_mem_level_bits(); + for (size_t i = num_entries / 2; i < num_entries; i++) { + phys_t dir; + status = mem_phys_alloc(1, flags, &dir); + if (status) + return status; + root[i] = pte_build(dir, PTE_DIR, CACHE_ANY); + } + struct pmap result = { .root = addr, .lock = (struct spinlock){0}, @@ -71,7 +81,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check if (pte_is_present(pte)) { // Get the next level. - *pte = pte_build(pte_address(pte), level_flags, CACHE_ANY); + *pte |= pte_build(0, level_flags, CACHE_ANY); current_head = HHDM_PTR(pte_address(pte)); } else { // If the current level isn't present, we can skip the rest. diff --git a/core/mem/pmm.c b/core/mem/pmm.c index 3ba9c7f..530281b 100644 --- a/core/mem/pmm.c +++ b/core/mem/pmm.c @@ -25,7 +25,7 @@ static struct phys_mem* bump_region = nullptr; static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) { const size_t bytes = num_pages * arch_mem_page_size(); - if (((intptr_t)bump_mem.length - bytes) <= 0) + if (bytes > bump_mem.length) return ENOMEM; *out = bump_mem.address; @@ -76,7 +76,7 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* while (it) { const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size(); // If the region is too high up or doesn't have enough pages left, continue searching. - if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) { + if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) { prev_it = it; it = it->freelist.next; continue; @@ -96,11 +96,26 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* *out = page_addr; } // If not, we can just shrink the block. - else { + else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) { it->freelist.count -= num_pages; if (__likely(out)) *out = page_addr + (it->freelist.count * arch_mem_page_size()); + } else { + struct page* rest = it + num_pages; + rest->freelist.count = it->freelist.count - num_pages; + rest->freelist.next = it->freelist.next; + + if (prev_it) + prev_it->freelist.next = rest; + else + pmm_head = rest; + + it->freelist.next = nullptr; + it->freelist.count = 0; + + if (__likely(out)) + *out = page_addr; } goto success; } @@ -118,13 +133,41 @@ success: return 0; } +static void freelist_insert(struct page* page, size_t num_pages) { + struct page* prev = nullptr; + struct page* next = pmm_head; + while (next && next < page) { + prev = next; + next = next->freelist.next; + } + + page->freelist.count = num_pages; + page->freelist.next = next; + + if (next && page + page->freelist.count == next) { + page->freelist.count += next->freelist.count; + page->freelist.next = next->freelist.next; + next->freelist.next = nullptr; + next->freelist.count = 0; + } + + if (prev && prev + prev->freelist.count == page) { + prev->freelist.count += page->freelist.count; + prev->freelist.next = page->freelist.next; + page->freelist.next = nullptr; + page->freelist.count = 0; + } else if (prev) { + prev->freelist.next = page; + } else { + pmm_head = page; + } +} + static errno_t freelist_free(phys_t addr, size_t num_pages) { spin_lock(&pmm_lock); - struct page* page = &vm_pfndb[addr / arch_mem_page_size()]; - page->freelist.count = num_pages; - page->freelist.next = pmm_head; - pmm_head = page; + freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages); + pmm_total_free += num_pages; spin_unlock(&pmm_lock); return 0; @@ -141,11 +184,9 @@ void mem_phys_init(struct phys_mem* map, size_t length) { if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE) continue; - struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()]; - page->freelist.count = map[i].length / arch_mem_page_size(); - page->freelist.next = pmm_head; - pmm_head = page; - pmm_total_free += page->freelist.count; + const size_t num_pages = map[i].length / arch_mem_page_size(); + freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages); + pmm_total_free += num_pages; } const size_t free_bytes = (pmm_total_free * arch_mem_page_size()); diff --git a/core/mem/slab.c b/core/mem/slab.c index 7dd4927..21edb1d 100644 --- a/core/mem/slab.c +++ b/core/mem/slab.c @@ -24,8 +24,6 @@ typedef struct { static struct slab slabs[8] = {0}; static errno_t slab_new(struct slab* slab, size_t size) { - slab->lock = (struct spinlock){0}; - // Allocate a new page for the head. phys_t addr; errno_t status = mem_phys_alloc(1, 0, &addr); @@ -68,8 +66,10 @@ void slab_init(void) { static void* slab_do_alloc(struct slab* slab) { spin_lock(&slab->lock); - if (__unlikely(slab->head == nullptr)) - slab_new(slab, slab->ent_size); + if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) { + spin_unlock(&slab->lock); + return nullptr; + } void** old_free = slab->head; slab->head = *old_free; diff --git a/core/mem/usercopy.c b/core/mem/usercopy.c index 3ff9ebf..9d2f71d 100644 --- a/core/mem/usercopy.c +++ b/core/mem/usercopy.c @@ -1,8 +1,18 @@ #include #include +#include #include +static bool is_user_range(uintptr_t addr, size_t len) { + if (len == 0) + return arch_mem_addr_is_user(addr); + return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1); +} + bool usercopy_read(void* dst, const __user void* src, size_t len) { + if (!is_user_range((uintptr_t)src, len)) + return false; + arch_usercopy_allow(true); bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region); arch_usercopy_allow(false); @@ -10,6 +20,9 @@ bool usercopy_read(void* dst, const __user void* src, size_t len) { } bool usercopy_write(__user void* dst, const void* src, size_t len) { + if (!is_user_range((uintptr_t)dst, len)) + return false; + arch_usercopy_allow(true); bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region); arch_usercopy_allow(false); @@ -17,6 +30,13 @@ bool usercopy_write(__user void* dst, const void* src, size_t len) { } bool usercopy_strlen(const __user char* str, size_t max, size_t* len) { + if (!arch_mem_addr_is_user((uintptr_t)str)) + return false; + + size_t unused; + if (len == nullptr) + len = &unused; + arch_usercopy_allow(true); bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region); arch_usercopy_allow(false); diff --git a/core/mem/virt.c b/core/mem/virt.c index f874634..61f9e4d 100644 --- a/core/mem/virt.c +++ b/core/mem/virt.c @@ -1,18 +1,19 @@ #include -#include #include #include +#include #include struct page* vm_pfndb = nullptr; // TODO: Replace with actual allocator. -static size_t mapping_offset = 0; +static _Atomic(size_t) mapping_offset = 0; uintptr_t vm_alloc(size_t length) { uintptr_t base = arch_mem_mapping_addr(); - size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE); + size_t offset = + atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire); return base + offset; } diff --git a/core/mem/vma.c b/core/mem/vma.c index 7effadf..0bbb352 100644 --- a/core/mem/vma.c +++ b/core/mem/vma.c @@ -5,6 +5,7 @@ #include #include #include +#include // Try to create a new, empty virtual address space. errno_t vma_new(struct vma** out) { @@ -22,6 +23,8 @@ errno_t vma_new(struct vma** out) { if (res) goto err1; + tailq_init(&vmspace->mappings); + *out = vmspace; return 0; @@ -32,20 +35,90 @@ err0: } // Delete a virtual address space. -void vma_free(struct vma* v) {} +void vma_free(struct vma* v) { + struct vma_mapping* mapping; + while ((mapping = tailq_first(&v->mappings)) != nullptr) { + tailq_remove(&v->mappings, mapping, link); + vmo_unref(mapping->object); + mem_free(mapping); + } +} -// Remove any mappings in the given address range. -// May fail if there is no memory and a mapping needs to be split. -errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size); +errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) { + panic("TODO"); +} -// Change the protection of an existing mapping. -// May fail if there is no memory and a mapping needs to be split. -errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags); +errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) { + panic("TODO"); +} + +static struct vma_mapping* find_mapping(struct vma* v, size_t page) { + for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) { + if (page >= m->start_page && page < m->end_page) + return m; + } + return nullptr; +} + +static bool access_allowed(enum prot_flags prot, const struct vma_fault_info* info) { + if (info->caused_by_write) + return prot & PROT_WRITE; + if (info->caused_by_fetch) + return prot & PROT_EXEC; + return prot & (PROT_READ | PROT_WRITE); +} + +static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) { + enum pte_flags pf = 0; + if (prot & PROT_READ) + pf |= PTE_READ; + if (prot & PROT_WRITE) + pf |= PTE_WRITE; + if (prot & PROT_EXEC) + pf |= PTE_EXEC; + if (v->pmap.is_user) + pf |= PTE_USER; + return pf; +} -// Try to handle a page fault. bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) { - // TODO - return false; + const size_t page_size = arch_mem_page_size(); + const size_t faulty_page = info->addr / page_size; + + spin_lock(&v->lock); + struct vma_mapping* mapping = find_mapping(v, faulty_page); + if (!mapping || !access_allowed(mapping->prot, info)) { + spin_unlock(&v->lock); + return false; + } + + struct vmo* object = mapping->object; + const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page; + const enum prot_flags prot = mapping->prot; + vmo_ref(object); + spin_unlock(&v->lock); + + struct page* page; + if (object->get_page(object, object_page, &page)) { + vmo_unref(object); + return false; + } + + spin_lock(&v->lock); + mapping = find_mapping(v, faulty_page); + const bool still_valid = mapping && mapping->object == object && + (faulty_page - mapping->start_page) + mapping->offset_page == object_page && + access_allowed(mapping->prot, info); + + if (still_valid) { + errno_t status = + pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY); + assert(status == 0, "Failed to map a demand-loaded page\n"); + } + spin_unlock(&v->lock); + + vmo_unref(object); + return true; } errno_t vma_map( @@ -56,44 +129,31 @@ errno_t vma_map( enum prot_flags flags, uintptr_t vmo_offset ) { - const size_t page_size = arch_mem_page_size(); - - if (addr % page_size != vmo_offset % page_size) - return EINVAL; - - size_t start_page = addr / page_size; - const size_t addr_offset = addr % page_size; - const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size; - size_t end_page = start_page + pages; - if (!vma || !vmo) return EINVAL; - const size_t first_vmo_page = vmo_offset / page_size; + uintptr_t end; + if (ckd_add(&end, addr, len)) + return ENOMEM; - for (size_t p = start_page; p < end_page; ++p) { - const size_t page_index = p - start_page; - struct page* page_obj; - errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj); - if (status != 0) - return status; + const size_t page_size = arch_mem_page_size(); + if (addr % page_size != vmo_offset % page_size) + return EINVAL; - phys_t paddr = (phys_t)vm_page_addr(page_obj); + struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0); + if (!mapping) + return ENOMEM; - enum pte_flags pf = 0; - if (flags & PROT_READ) - pf |= PTE_READ; - if (flags & PROT_WRITE) - pf |= PTE_WRITE; - if (flags & PROT_EXEC) - pf |= PTE_EXEC; - if (vma->pmap.is_user) - pf |= PTE_USER; + vmo_ref(vmo); + mapping->start_page = addr / page_size; + mapping->end_page = ALIGN_UP(end, page_size) / page_size; + mapping->offset_page = vmo_offset / page_size; + mapping->object = vmo; + mapping->prot = flags; - status = pmap_map(&vma->pmap, p * page_size, paddr, pf, CACHE_ANY); - if (status != 0) - return status; - } + spin_lock(&vma->lock); + tailq_insert_head(&vma->mappings, mapping, link); + spin_unlock(&vma->lock); return 0; } diff --git a/core/mem/vmo.c b/core/mem/vmo.c index 88db73a..9239056 100644 --- a/core/mem/vmo.c +++ b/core/mem/vmo.c @@ -5,6 +5,15 @@ #include #include +void vmo_ref(struct vmo* obj) { + refcount_inc(&obj->rc); +} + +void vmo_unref(struct vmo* obj) { + if (refcount_dec(&obj->rc)) + obj->destroy(obj); +} + errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) { const size_t page_size = arch_mem_page_size(); size_t progress = 0; @@ -104,14 +113,17 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page // Try to find the requested page in the cache. struct page_list* iter; + spin_lock(&paged->lock); list_foreach(iter, &paged->cache, next) { if (iter->offset != offset_idx) continue; // We found a cached page! *out = iter->value; + spin_unlock(&paged->lock); return 0; } + spin_unlock(&paged->lock); // Get page from out pager if we don't have one already. struct page* new_page; @@ -120,26 +132,57 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page return status; struct page_list* entry = mem_alloc(sizeof(struct page_list), 0); - if (!entry) + if (!entry) { + paged->source.put_page(&paged->source, offset_idx, new_page); return ENOMEM; + } entry->offset = offset_idx; entry->value = new_page; + spin_lock(&paged->lock); + list_foreach(iter, &paged->cache, next) { + if (iter->offset != offset_idx) + continue; + + *out = iter->value; + spin_unlock(&paged->lock); + mem_free(entry); + paged->source.put_page(&paged->source, offset_idx, new_page); + return 0; + } + // Add page to the object cache. list_insert_head(&paged->cache, entry, next); + spin_unlock(&paged->lock); *out = new_page; return 0; } +static void paged_destroy(struct vmo* vmo) { + struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object); + + struct page_list* iter = list_first(&paged->cache); + while (iter) { + struct page_list* next = list_next(iter, next); + paged->source.put_page(&paged->source, iter->offset, iter->value); + mem_free(iter); + iter = next; + } + + mem_free(paged); +} + errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) { struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0); if (!result) return ENOMEM; + refcount_init(&result->object.rc); result->source = pager; result->object.get_page = paged_get_page; + result->object.destroy = paged_destroy; *out = result; return 0; @@ -156,8 +199,7 @@ static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct p } static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) { - // We do nothing here. - return 0; + return mem_phys_free(vm_page_addr(page), 1); } static const struct pager_ops phys_pager = { @@ -177,16 +219,24 @@ struct phys_range_vmo { static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) { struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object); + if (offset_idx >= p->len / arch_mem_page_size()) + return EINVAL; *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx]; return 0; } +static void phys_range_destroy(struct vmo* vmo) { + mem_free(containerof(vmo, struct phys_range_vmo, object)); +} + errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) { struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0); if (!vmo) return ENOMEM; + refcount_init(&vmo->object.rc); vmo->object.get_page = phys_range_get_page; + vmo->object.destroy = phys_range_destroy; vmo->start = ALIGN_DOWN(addr, arch_mem_page_size()); vmo->len = ALIGN_UP(length, arch_mem_page_size()); diff --git a/core/meson.build b/core/meson.build index fade04c..6b7f3ef 100644 --- a/core/meson.build +++ b/core/meson.build @@ -6,6 +6,7 @@ kernel_sources += files( 'panic.c', 'percpu.c', 'print.c', + 'refcount.c', 'spin.c', 'wait.c', ) diff --git a/core/percpu.c b/core/percpu.c index 8fedaa3..6e01802 100644 --- a/core/percpu.c +++ b/core/percpu.c @@ -1,6 +1,22 @@ +#include +#include #include #include +#include +#include #include +#include + +extern uint8_t __ld_percpu_start[]; +extern uint8_t __ld_percpu_end[]; + +static _Atomic(size_t) num_cpus = 1; + +static size_t max_cpus = SIZE_MAX; +static void smp_option(uintptr_t value) { + max_cpus = value; +} +CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option); [[__used, __section(".percpu.init")]] struct percpu percpu_bsp = { @@ -10,8 +26,22 @@ struct percpu percpu_bsp = { }; struct percpu* percpu_new() { - // TODO - return nullptr; + const size_t size = __ld_percpu_end - __ld_percpu_start; + const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size(); + + phys_t mem; + if (mem_phys_alloc(num_pages, 0, &mem)) + return nullptr; + + struct percpu* cpu = HHDM_PTR(mem); + cpu->self = cpu; + cpu->id = atomic_fetch_add_explicit(&num_cpus, 1, memory_order_relaxed); + cpu->online = false; + return cpu; +} + +size_t percpu_max_cpus() { + return max_cpus; } struct percpu* percpu_get() { diff --git a/core/print.c b/core/print.c index 918b13e..19ecd71 100644 --- a/core/print.c +++ b/core/print.c @@ -10,6 +10,11 @@ typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len); +static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) { + for (size_t i = 0; i < count; i++) + callback(ctx, &c, 1); +} + static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) { // Amount of bytes written. int32_t written = 0; @@ -37,7 +42,7 @@ static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_ bool blank_sign = false; bool write_prefix = false; bool pad_zero = false; - bool has_width = false; + bool left_align = false; size_t width = 0; bool has_precision = false; size_t precision = 0; @@ -68,6 +73,11 @@ check_fmt: fmt++; goto check_fmt; } + case '-': { + left_align = true; + fmt++; + goto check_fmt; + } // Width case '1': case '2': @@ -78,14 +88,11 @@ check_fmt: case '7': case '8': case '9': { - has_width = true; - char number[10 + 1] = {0}; - size_t idx = 0; + width = 0; while (*fmt >= '0' && *fmt <= '9') { - number[idx++] = *fmt; + width = width * 10 + (*fmt - '0'); fmt++; } - width = atolu(number, 10); goto check_fmt; } @@ -93,16 +100,19 @@ check_fmt: case '.': { has_precision = true; fmt++; + precision = 0; if (*fmt == '*') { - precision = (char)va_arg(args, int32_t); + const int32_t arg = va_arg(args, int32_t); + if (arg < 0) + has_precision = false; + else + precision = arg; + fmt++; } else { - char number[10 + 1] = {0}; - size_t idx = 0; while (*fmt >= '0' && *fmt <= '9') { - number[idx++] = *fmt; + precision = precision * 10 + (*fmt - '0'); fmt++; } - precision = atolu(number, 10); } goto check_fmt; } @@ -133,10 +143,15 @@ check_fmt: // Character case 'c': { char c = (char)va_arg(args, int32_t); + const size_t pad = width > 1 ? width - 1 : 0; + if (!left_align) + print_pad(callback, ctx, ' ', pad); callback(ctx, &c, sizeof(c)); + if (left_align) + print_pad(callback, ctx, ' ', pad); - written++; + written += 1 + pad; fmt++; break; } @@ -150,10 +165,15 @@ check_fmt: len = strnlen(str, precision); else len = strlen(str); + const size_t pad = width > len ? width - len : 0; + if (!left_align) + print_pad(callback, ctx, ' ', pad); callback(ctx, str, len); + if (left_align) + print_pad(callback, ctx, ' ', pad); - written += len; + written += len + pad; fmt++; break; } @@ -212,36 +232,38 @@ check_fmt: break; } print_num: - size_t len = strlen(number); - if (has_precision) { - for (size_t i = 0; i < precision - len; i++) - callback(ctx, " ", 1); + const char* digits = number; + char sign = 0; + if (digits[0] == '-') { + sign = '-'; + digits++; + } else if (force_sign) { + sign = '+'; + } else if (blank_sign) { + sign = ' '; } - if (has_width) { - char c = pad_zero ? '0' : ' '; - // We don't have to pad anything. - if (len < width) { - for (size_t i = 0; i < width - len; i++) - callback(ctx, &c, 1); - } + const size_t len = strlen(digits); + size_t zeros = (has_precision && precision > len) ? precision - len : 0; + const size_t total = (sign ? 1 : 0) + (write_prefix ? 2 : 0) + zeros + len; + size_t pad = width > total ? width - total : 0; + if (pad_zero && !left_align && !has_precision) { + zeros += pad; + pad = 0; } - if (write_prefix) { + + if (!left_align) + print_pad(callback, ctx, ' ', pad); + if (sign) + callback(ctx, &sign, 1); + if (write_prefix) callback(ctx, "0x", 2); - } - if (force_sign) { - char c = '+'; - if (number[0] != '-') - callback(ctx, &c, 1); - } - if (blank_sign) { - char c = ' '; - if (number[0] != '-') - callback(ctx, &c, 1); - } - callback(ctx, number, len); + print_pad(callback, ctx, '0', zeros); + callback(ctx, digits, len); + if (left_align) + print_pad(callback, ctx, ' ', pad); - written += len; + written += width > total ? width : total; fmt++; break; } @@ -272,13 +294,14 @@ print_num: } // No format, just normal text. default: { - fmt = format_begun_at; - size_t len = strlen(fmt); + size_t len = fmt - format_begun_at; + if (*fmt) + len++; - callback(ctx, fmt, len); + callback(ctx, format_begun_at, len); written += len; - fmt += len; + fmt = format_begun_at + len; break; } } diff --git a/core/refcount.c b/core/refcount.c new file mode 100644 index 0000000..4655528 --- /dev/null +++ b/core/refcount.c @@ -0,0 +1,17 @@ +#include +#include + +void refcount_init(struct refcount* rc) { + atomic_store_explicit(&rc->count, 1, memory_order_relaxed); +} + +void refcount_inc(struct refcount* rc) { + atomic_fetch_add_explicit(&rc->count, 1, memory_order_relaxed); +} + +bool refcount_dec(struct refcount* rc) { + if (atomic_fetch_sub_explicit(&rc->count, 1, memory_order_release) != 1) + return false; + atomic_thread_fence(memory_order_acquire); + return true; +} diff --git a/core/sched/sched_dumb.c b/core/sched/sched_dumb.c index 012f5b9..3f1ddd9 100644 --- a/core/sched/sched_dumb.c +++ b/core/sched/sched_dumb.c @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -9,11 +8,12 @@ #include #include #include +#include #include #include #include +#include #include -#include static void idle_func(uintptr_t) { while (1) { @@ -39,7 +39,12 @@ void sched_init(struct scheduler* sched) { struct task* idle_task; assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n"); sched->idle_task = idle_task; - sched->current = idle_task; + + struct task* boot_task = mem_alloc(sizeof(struct task), 0); + assert(boot_task != nullptr, "Unable to create boot task!\n"); + strncpy(boot_task->name, "boot", sizeof(boot_task->name)); + boot_task->space = &kernel_vma; + sched->current = boot_task; tailq_init(&sched->run_queue); @@ -76,17 +81,19 @@ static void do_reschedule(struct scheduler* sched) { pmap_set(&to->space->pmap); struct percpu* cpu = percpu_get(); - from->kernel_stack = cpu->kernel_stack; from->user_stack = cpu->user_stack; - cpu->kernel_stack = to->kernel_stack; + cpu->kernel_stack = to->kernel_stack + KERNEL_STACK_SIZE; cpu->user_stack = to->user_stack; - struct task* old = arch_sched_switch(from, to); + struct irq_percpu irq_state = *percpu_ref(irq_data); + arch_sched_switch(from, to); + *percpu_ref(irq_data) = irq_state; + irq_unlock(); } void sched_reschedule(struct scheduler* sched) { irq_lock(); - if (sched->current->state != TASK_STATE_DEAD) + if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD) sched_add_task(sched, sched->current); do_reschedule(sched); } @@ -104,27 +111,38 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt if (new_task == nullptr) return ENOMEM; - static size_t last_tid = 0; - new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE); + static _Atomic(size_t) last_tid = 0; + new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire); new_task->space = space; strncpy(new_task->name, name, sizeof(new_task->name)); // Allocate a kernel stack. new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0); - if (!new_task->kernel_stack) + if (!new_task->kernel_stack) { + mem_free(new_task); return ENOMEM; + } new_task->user_stack = 0; // Initialize the arch-dependent parts. errno_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true); - if (s != 0) + if (s != 0) { + mem_free((void*)new_task->kernel_stack); + mem_free(new_task); return s; + } *out = new_task; return 0; } void task_entry(task_fn_t entry, uintptr_t arg0) { + *percpu_ref(irq_data) = (struct irq_percpu){ + .depth = 1, + .saved_state = true, + }; + irq_unlock(); + entry(arg0); struct scheduler* sched = &percpu_get()->sched; diff --git a/core/spin.c b/core/spin.c index 51127d4..13d9d90 100644 --- a/core/spin.c +++ b/core/spin.c @@ -1,15 +1,15 @@ -#include #include +#include void spin_lock(struct spinlock* spin) { - uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED); + uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed); - while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) { + while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) { asm volatile("pause"); } } void spin_unlock(struct spinlock* spin) { - uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED); - atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE); + uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed); + atomic_store_explicit(&spin->owner, val + 1, memory_order_release); } diff --git a/dev/acpi/uacpi_glue.c b/dev/acpi/uacpi_glue.c index f1c72fc..27ca035 100644 --- a/dev/acpi/uacpi_glue.c +++ b/dev/acpi/uacpi_glue.c @@ -23,12 +23,15 @@ void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) { uintptr_t difference = addr - aligned_addr; size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size()); - uintptr_t vaddr = vm_alloc(len); - for (size_t i = 0; i < len; i += arch_mem_page_size()) { - pmap_map(&kernel_vma.pmap, vaddr + i, addr + i, PROT_READ | PROT_WRITE, CACHE_ANY); + uintptr_t vaddr = vm_alloc(aligned_len); + if (!vaddr) + return nullptr; + + for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) { + pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PROT_READ | PROT_WRITE, CACHE_ANY); } - return (void*)vaddr; + return (void*)(vaddr + difference); } void uacpi_kernel_unmap(void* addr, uacpi_size len) { diff --git a/include/arch/irq.h b/include/arch/irq.h index 3935e69..023b190 100644 --- a/include/arch/irq.h +++ b/include/arch/irq.h @@ -2,6 +2,6 @@ #include -void arch_irq_set_state(bool state); +bool arch_irq_set_state(bool state); bool arch_irq_get_state(); void arch_irq_wait(); diff --git a/include/arch/mem.h b/include/arch/mem.h index 0a893c2..6811502 100644 --- a/include/arch/mem.h +++ b/include/arch/mem.h @@ -1,28 +1,29 @@ #pragma once +#include #include #include -[[gnu::pure]] +[[__pure]] bool arch_mem_addr_is_user(uintptr_t addr); -[[gnu::pure]] +[[__pure]] size_t arch_mem_page_size(); -[[gnu::pure]] +[[__pure]] size_t arch_mem_page_bits(); -[[gnu::pure]] +[[__pure]] size_t arch_mem_level_bits(); -[[gnu::pure]] +[[__pure]] size_t arch_mem_num_levels(); -[[gnu::pure]] +[[__pure]] uintptr_t arch_mem_hhdm_addr(); -[[gnu::pure]] +[[__pure]] uintptr_t arch_mem_pfndb_addr(); -[[gnu::pure]] +[[__pure]] uintptr_t arch_mem_mapping_addr(); diff --git a/include/zinnia/atomic.h b/include/zinnia/atomic.h deleted file mode 100644 index e53e6c7..0000000 --- a/include/zinnia/atomic.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -typedef enum memory_order { - ATOMIC_RELAXED = __ATOMIC_RELAXED, - ATOMIC_CONSUME = __ATOMIC_CONSUME, - ATOMIC_ACQUIRE = __ATOMIC_ACQUIRE, - ATOMIC_RELEASE = __ATOMIC_RELEASE, - ATOMIC_ACQ_REL = __ATOMIC_ACQ_REL, - ATOMIC_SEQ_CST = __ATOMIC_SEQ_CST -} memory_order; - -#define atomic(T) _Atomic(T) -#define atomic_store(ptr, des, order) __atomic_store_n(ptr, des, order) -#define atomic_load(ptr, order) __atomic_load_n(ptr, order) -#define atomic_exchange(ptr, des, order) __atomic_exchange_n(ptr, des, order) -#define atomic_fetch_add(ptr, op, order) __atomic_fetch_add(ptr, op, order) -#define atomic_fetch_sub(ptr, op, order) __atomic_fetch_sub(ptr, op, order) -#define atomic_fetch_or(ptr, op, order) __atomic_fetch_or(ptr, op, order) -#define atomic_fetch_xor(ptr, op, order) __atomic_fetch_xor(ptr, op, order) -#define atomic_fetch_and(ptr, op, order) __atomic_fetch_and(ptr, op, order) -#define atomic_thread_fence(order) __atomic_thread_fence(order) diff --git a/include/zinnia/compiler.h b/include/zinnia/compiler.h index 28f5a83..82a4b33 100644 --- a/include/zinnia/compiler.h +++ b/include/zinnia/compiler.h @@ -14,6 +14,7 @@ #define __format(like, ...) gnu::format(like, __VA_ARGS__) #define __always_inline gnu::always_inline #define __visibility gnu::visibility +#define __pure gnu::pure // Only clang has these hints, GCC would throw a warning here. // clang::noderef also don't support the C23 attribute format. diff --git a/include/zinnia/hashmap.h b/include/zinnia/hashmap.h index b8b94f9..9cac11e 100644 --- a/include/zinnia/hashmap.h +++ b/include/zinnia/hashmap.h @@ -46,26 +46,38 @@ // Internal: grow and rehash. #define _HASHMAP_GROW(map, hash_fn, eq_fn) \ - do { \ + ({ \ + errno_t _grow_status = 0; \ size_t _old_cap = (map)->capacity; \ size_t _new_cap = _old_cap == 0 ? 16 : _old_cap * 2; \ - typeof((map)->keys) _old_keys = (map)->keys; \ - typeof((map)->values) _old_values = (map)->values; \ - uint64_t* _old_hashes = (map)->hashes; \ - (map)->keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \ - (map)->values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \ - (map)->hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \ - (map)->capacity = _new_cap; \ - (map)->count = 0; \ - for (size_t _i = 0; _i < _old_cap; _i++) { \ - if (_old_hashes[_i] != 0) { \ - _HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \ + typeof((map)->keys) _new_keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \ + typeof((map)->values) _new_values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \ + uint64_t* _new_hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \ + if (_new_keys == nullptr || _new_values == nullptr || _new_hashes == nullptr) { \ + mem_free(_new_keys); \ + mem_free(_new_values); \ + mem_free(_new_hashes); \ + _grow_status = ENOMEM; \ + } else { \ + typeof((map)->keys) _old_keys = (map)->keys; \ + typeof((map)->values) _old_values = (map)->values; \ + uint64_t* _old_hashes = (map)->hashes; \ + (map)->keys = _new_keys; \ + (map)->values = _new_values; \ + (map)->hashes = _new_hashes; \ + (map)->capacity = _new_cap; \ + (map)->count = 0; \ + for (size_t _i = 0; _i < _old_cap; _i++) { \ + if (_old_hashes[_i] != 0) { \ + _HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \ + } \ } \ + mem_free(_old_keys); \ + mem_free(_old_values); \ + mem_free(_old_hashes); \ } \ - mem_free(_old_keys); \ - mem_free(_old_values); \ - mem_free(_old_hashes); \ - } while (0) + _grow_status; \ + }) // Internal: insert with a precomputed hash. Used by grow and the public INSERT macro. #define _HASHMAP_INSERT_INNER(map, k, v, h) \ @@ -126,10 +138,7 @@ } \ if (!_found) { \ if ((map)->count * 4 >= (map)->capacity * 3 || (map)->capacity == 0) { \ - _HASHMAP_GROW(map, hash_fn, eq_fn); \ - if ((map)->keys == nullptr || (map)->values == nullptr || (map)->hashes == nullptr) { \ - _status = ENOMEM; \ - } \ + _status = _HASHMAP_GROW(map, hash_fn, eq_fn); \ } \ if (_status == 0) { \ _HASHMAP_INSERT_INNER(map, _k, (val), _h); \ diff --git a/include/zinnia/init.h b/include/zinnia/init.h index ee7acf6..d011dca 100644 --- a/include/zinnia/init.h +++ b/include/zinnia/init.h @@ -26,16 +26,13 @@ struct boot_info { uintptr_t hhdm_base; struct boot_file* files; size_t num_files; -#if CONFIG_OF + void* dtb; -#endif #if CONFIG_ACPI phys_t rsdp; #endif }; -extern struct boot_info boot_info; - [[__visibility("hidden")]] extern uint8_t __ld_stack_top[]; [[__visibility("hidden")]] diff --git a/include/zinnia/irq.h b/include/zinnia/irq.h index 744799b..1a8ca45 100644 --- a/include/zinnia/irq.h +++ b/include/zinnia/irq.h @@ -2,11 +2,13 @@ #include #include +#include +#include #include enum irq_status : uint8_t { - IRQ_STATUS_HANDLED, IRQ_STATUS_IGNORED, + IRQ_STATUS_HANDLED, }; enum irq_polarity : uint8_t { @@ -19,34 +21,48 @@ enum irq_trigger_mode : uint8_t { IRQ_TRIGGER_LEVEL, }; +enum irq_mode : uint8_t { + IRQ_MODE_EOI = 1 << 0, + IRQ_MODE_MASKABLE = 1 << 1, +}; + struct irq_handler { - enum irq_status (*handler)(void* self); + enum irq_status (*raise)(struct irq_handler* self); list_link(struct irq_handler*) next; }; struct irq_line { - int (*set_config)(enum irq_trigger_mode mode, enum irq_polarity polarity); + enum irq_mode (*set_config)(struct irq_line* self, enum irq_trigger_mode trigger, enum irq_polarity polarity); void (*mask)(struct irq_line* self); void (*unmask)(struct irq_line* self); void (*eoi)(struct irq_line* self); - list_head(struct irq_handler*) handlers; bool is_busy; + list_head(struct irq_handler*) handlers; + enum irq_mode mode; +}; + +struct msi_line { + struct irq_line line; + phys_t (*msg_addr)(struct msi_line* self); + uint32_t (*msg_data)(struct msi_line* self); }; struct irq_percpu { - uint32_t level; - bool in_interrupt; + size_t depth; + bool saved_state; }; -void irq_line_attach(struct irq_line* self, struct irq_handler* handler); +extern struct irq_percpu irq_data; + +void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity); +void irq_line_raise(struct irq_line* line); +void irq_line_attach(struct irq_line* line, struct irq_handler* handler); void irq_lock(); void irq_unlock(); -bool irq_set_interrupted(bool is_interrupted); -void irq_line_raise(struct irq_line* line); // Hints to stop execution on this CPU until an interrupt happens. void irq_wait(); bool irq_get_state(); -void irq_set_state(bool state); +bool irq_set_state(bool state); diff --git a/include/zinnia/percpu.h b/include/zinnia/percpu.h index b9da4b3..ac3bc49 100644 --- a/include/zinnia/percpu.h +++ b/include/zinnia/percpu.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -33,11 +32,11 @@ struct percpu { bool online; // Whether this CPU is initialized and active. uintptr_t kernel_stack; // The kernel mode stack. uintptr_t user_stack; // The user mode stack. - struct irq_percpu irq; // IRQ information. struct scheduler sched; // Scheduler information. }; struct percpu* percpu_new(); +size_t percpu_max_cpus(); struct percpu* percpu_get(); void percpu_bsp_init(); void percpu_init(); diff --git a/include/zinnia/print.h b/include/zinnia/print.h index 94904df..03a886d 100644 --- a/include/zinnia/print.h +++ b/include/zinnia/print.h @@ -16,5 +16,5 @@ void kvprintf(const char* message, va_list args); auto _time = clock_get_elapsed_ns(); \ auto _cur = percpu_get()->sched.current; \ auto _tid = _cur ? _cur->id : 0; \ - kprintf("[%5lu.%06lu] [%5lu] " message, _time / 1'000'000'000, _time % 1'000'000, _tid, ##__VA_ARGS__); \ + kprintf("[%5lu.%06lu] [%5lu] " message, _time / 1'000'000'000, _time / 1'000 % 1'000'000, _tid, ##__VA_ARGS__); \ } while (0) diff --git a/include/zinnia/refcount.h b/include/zinnia/refcount.h index 1797e64..9e4a835 100644 --- a/include/zinnia/refcount.h +++ b/include/zinnia/refcount.h @@ -1,12 +1,13 @@ #pragma once -#include #include +#include #include struct refcount { - atomic(size_t) count; + _Atomic(size_t) count; }; +void refcount_init(struct refcount* rc); void refcount_inc(struct refcount* rc); -void refcount_dec(struct refcount* rc); +bool refcount_dec(struct refcount* rc); diff --git a/include/zinnia/sched.h b/include/zinnia/sched.h index 4d8beca..4dda92c 100644 --- a/include/zinnia/sched.h +++ b/include/zinnia/sched.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include diff --git a/include/zinnia/spin.h b/include/zinnia/spin.h index 0372df8..dfcc9f2 100644 --- a/include/zinnia/spin.h +++ b/include/zinnia/spin.h @@ -1,12 +1,13 @@ #pragma once +#include #include // Busy-waits in a loop until the lock is freed. // Does not put the CPU to sleep. struct spinlock { - uint32_t owner; - uint32_t next; + _Atomic(uint32_t) owner; + _Atomic(uint32_t) next; }; // Attempts to lock a spinlock. If it's already locked, waits until it's freed. diff --git a/include/zinnia/syscall_list.inc b/include/zinnia/syscall_list.inc index 397f575..332b342 100644 --- a/include/zinnia/syscall_list.inc +++ b/include/zinnia/syscall_list.inc @@ -1,11 +1,11 @@ #ifndef SYSCALL_DECLARE -#define SYSCALL_DECLARE(number, name, num_args) extern void sys_##name(); +#define SYSCALL_DECLARE(number, name, num_args) extern void sys_##name(...); #endif // SYSCALL_DECLARE(0, exit, -1) // SYSCALL_DECLARE(1, syslog, -1) SYSCALL_DECLARE(2, getuname, 1) -// SYSCALL_DECLARE(3, setuname, -1) +SYSCALL_DECLARE(3, setuname, 1) SYSCALL_DECLARE(4, archctl, 2) // SYSCALL_DECLARE(5, reboot, -1) // SYSCALL_DECLARE(6, mmap, -1) diff --git a/include/zinnia/vec.h b/include/zinnia/vec.h index 8f36630..4c9c344 100644 --- a/include/zinnia/vec.h +++ b/include/zinnia/vec.h @@ -34,27 +34,30 @@ // Internal: grow to at least `needed` capacity. #define _vec_grow(vec, needed) \ - do { \ + ({ \ + errno_t _grow_status = 0; \ size_t _new_cap = (vec)->capacity == 0 ? 8 : (vec)->capacity; \ while (_new_cap < (needed)) \ _new_cap *= 2; \ typeof((vec)->data) _new = mem_alloc(_new_cap * sizeof(*(vec)->data), ALLOC_NOZERO); \ - if (_new != nullptr && (vec)->length > 0) \ - memcpy(_new, (vec)->data, (vec)->length * sizeof(*(vec)->data)); \ - mem_free((vec)->data); \ - (vec)->data = _new; \ - (vec)->capacity = _new_cap; \ - } while (0) + if (_new == nullptr) { \ + _grow_status = ENOMEM; \ + } else { \ + if ((vec)->length > 0) \ + memcpy(_new, (vec)->data, (vec)->length * sizeof(*(vec)->data)); \ + mem_free((vec)->data); \ + (vec)->data = _new; \ + (vec)->capacity = _new_cap; \ + } \ + _grow_status; \ + }) // Append an element. Returns 0 or ENOMEM. #define vec_push(vec, val) \ ({ \ errno_t _status = 0; \ - if ((vec)->length >= (vec)->capacity) { \ - _vec_grow(vec, (vec)->length + 1); \ - if ((vec)->data == nullptr) \ - _status = ENOMEM; \ - } \ + if ((vec)->length >= (vec)->capacity) \ + _status = _vec_grow(vec, (vec)->length + 1); \ if (_status == 0) \ (vec)->data[(vec)->length++] = (val); \ _status; \ @@ -68,11 +71,8 @@ ({ \ errno_t _status = 0; \ size_t _idx = (i); \ - if ((vec)->length >= (vec)->capacity) { \ - _vec_grow(vec, (vec)->length + 1); \ - if ((vec)->data == nullptr) \ - _status = ENOMEM; \ - } \ + if ((vec)->length >= (vec)->capacity) \ + _status = _vec_grow(vec, (vec)->length + 1); \ if (_status == 0) { \ memmove(&(vec)->data[_idx + 1], &(vec)->data[_idx], ((vec)->length - _idx) * sizeof(*(vec)->data)); \ (vec)->data[_idx] = (val); \ diff --git a/include/zinnia/vma.h b/include/zinnia/vma.h index 3c39bae..78e3de1 100644 --- a/include/zinnia/vma.h +++ b/include/zinnia/vma.h @@ -4,12 +4,25 @@ #include #include #include +#include +#include #include +struct vma_mapping { + size_t start_page; + size_t end_page; + size_t offset_page; + struct vmo* object; + enum prot_flags prot; + tailq_link(struct vma_mapping) link; +}; + // Virtual address space. struct vma { // Architecture-specific virtual memory map implementation. struct pmap pmap; + tailq_head(struct vma_mapping) mappings; + struct spinlock lock; }; // Try to create a new, empty virtual address space. @@ -36,10 +49,10 @@ errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags struct vma_fault_info { uintptr_t ip; uintptr_t addr; - bool caused_by_user; - bool caused_by_write; - bool caused_by_fetch; - bool page_was_present; + bool caused_by_user:1; + bool caused_by_write:1; + bool caused_by_fetch:1; + bool page_was_present:1; }; // Try to handle a page fault. diff --git a/include/zinnia/vmo.h b/include/zinnia/vmo.h index 8c3b776..eea6465 100644 --- a/include/zinnia/vmo.h +++ b/include/zinnia/vmo.h @@ -2,15 +2,21 @@ #include #include +#include #include #include #include #include struct vmo { + struct refcount rc; errno_t (*get_page)(struct vmo* vmo, uintptr_t offset_idx, struct page** out); + void (*destroy)(struct vmo* vmo); }; +void vmo_ref(struct vmo* obj); +void vmo_unref(struct vmo* obj); + errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read); errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written); errno_t vmo_copy( diff --git a/include/zinnia/waitq.h b/include/zinnia/waitq.h index d2f5dee..dc7c901 100644 --- a/include/zinnia/waitq.h +++ b/include/zinnia/waitq.h @@ -1,14 +1,14 @@ #pragma once -#include #include #include #include +#include // A queue of tasks waiting for an event. struct waitq { - tailq_head(atomic(struct task)) waiters; - atomic(bool) woken; + tailq_head(_Atomic(struct task)) waiters; + _Atomic(bool) woken; }; void waitq_init(struct waitq* wq); diff --git a/syscalls/uname.c b/syscalls/uname.c index d8964e0..1807082 100644 --- a/syscalls/uname.c +++ b/syscalls/uname.c @@ -1,8 +1,8 @@ #include +#include #include #include #include -#include #include static struct spinlock uname_lock = {}; @@ -13,9 +13,10 @@ static struct __utsname uname_buf = { .version = CONFIG_KERNEL_FLAVOR, .nodename = "localhost", .machine = ZINNIA_ARCH, + .domainname = "none", }; -struct syscall_ret sys_setuname(__user const struct __utsname* ptr) { +struct syscall_ret sys_setuname(const __user struct __utsname* ptr) { struct syscall_ret ret = {}; if (!ptr) { @@ -33,11 +34,12 @@ struct syscall_ret sys_setuname(__user const struct __utsname* ptr) { spin_lock(&uname_lock); - strncpy(uname_buf.sysname, read_buf.sysname, sizeof(uname_buf.sysname)); - strncpy(uname_buf.release, read_buf.release, sizeof(uname_buf.release)); - strncpy(uname_buf.version, read_buf.version, sizeof(uname_buf.version)); - strncpy(uname_buf.nodename, read_buf.nodename, sizeof(uname_buf.nodename)); - strncpy(uname_buf.machine, read_buf.machine, sizeof(uname_buf.machine)); + strncpy(uname_buf.sysname, read_buf.sysname, sizeof(uname_buf.sysname) - 1); + strncpy(uname_buf.release, read_buf.release, sizeof(uname_buf.release) - 1); + strncpy(uname_buf.version, read_buf.version, sizeof(uname_buf.version) - 1); + strncpy(uname_buf.nodename, read_buf.nodename, sizeof(uname_buf.nodename) - 1); + strncpy(uname_buf.machine, read_buf.machine, sizeof(uname_buf.machine) - 1); + strncpy(uname_buf.domainname, read_buf.domainname, sizeof(uname_buf.domainname) - 1); spin_unlock(&uname_lock); @@ -53,11 +55,11 @@ struct syscall_ret sys_getuname(__user struct __utsname* ptr) { } spin_lock(&uname_lock); - - if (!usercopy_write(ptr, &uname_buf, sizeof(uname_buf))) - ret.errno = EFAULT; - + struct __utsname write_buf = uname_buf; spin_unlock(&uname_lock); + if (!usercopy_write(ptr, &write_buf, sizeof(write_buf))) + ret.errno = EFAULT; + return ret; }