#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // The task frame consists of registers that the C ABI marks as callee-saved. // If we don't save them, these registers are lost during a context switch. struct task_frame { uint64_t rbx, rbp, r12, r13, r14, r15, rip; }; [[__naked]] static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) { asm volatile( "sub rsp, 0x30\n" // Make room for all regs (except RIP). "mov [rsp + %c0], rbx\n" "mov [rsp + %c1], rbp\n" "mov [rsp + %c2], r12\n" "mov [rsp + %c3], r13\n" "mov [rsp + %c4], r14\n" "mov [rsp + %c5], r15\n" "mov [rdi], rsp\n" // rdi = old_rsp "mov rsp, rsi\n" // rsi = new_rsp "mov rbx, [rsp + %c0]\n" "mov rbp, [rsp + %c1]\n" "mov r12, [rsp + %c2]\n" "mov r13, [rsp + %c3]\n" "mov r14, [rsp + %c4]\n" "mov r15, [rsp + %c5]\n" "add rsp, 0x30\n" "call %c6\n" "ret" // This will conveniently move us to the RIP we put at this stack entry. : : "i"(offsetof(struct task_frame, rbx)), "i"(offsetof(struct task_frame, rbp)), "i"(offsetof(struct task_frame, r12)), "i"(offsetof(struct task_frame, r13)), "i"(offsetof(struct task_frame, r14)), "i"(offsetof(struct task_frame, r15)), "i"(irq_unlock) : "memory" ); } void arch_sched_switch(struct task* from, struct task* to) { struct arch_percpu* cpu = &percpu_get()->arch; struct arch_task_context* from_ctx = &from->context; struct arch_task_context* to_ctx = &to->context; cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE; if (from->user_stack) { cpu->fpu_save(from_ctx->fpu_region); from_ctx->ds = asm_read_ds(); from_ctx->es = asm_read_es(); from_ctx->fs = asm_read_fs(); from_ctx->gs = asm_read_gs(); from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE); from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE); } if (to->user_stack) { cpu->fpu_restore(to_ctx->fpu_region); asm_write_ds(to_ctx->ds); asm_write_es(to_ctx->es); asm_write_fs(to_ctx->fs); // If we have to change the GS segment we need to reload the MSR, otherwise we lose its value. if (to_ctx->gs != asm_read_gs()) { uint64_t percpu = asm_rdmsr(MSR_GS_BASE); asm_write_gs(to_ctx->gs); asm_wrmsr(MSR_GS_BASE, percpu); } asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base); asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base); } perform_switch(&from_ctx->rsp, to_ctx->rsp); } [[__naked]] void task_entry_thunk() { asm volatile( "mov rdi, rbx\n" "mov rsi, r12\n" "jmp %c0" : : "i"(task_entry) : "memory" ); } errno_t arch_task_init( struct arch_task_context* context, void* entry, uintptr_t arg0, uintptr_t stack_start, bool is_user ) { struct arch_percpu* cpu = &percpu_get()->arch; // Prepare a dummy stack with an entry point function to return to. struct task_frame* frame = (struct task_frame*)stack_start - 1; frame->rbx = (uint64_t)entry; frame->r12 = arg0; frame->rip = (uint64_t)task_entry_thunk; context->rsp = (uint64_t)frame; if (is_user) { // Allocate FPU space. const size_t fpu_size = cpu->fpu_size; void* fpu_base = vm_alloc(fpu_size); if (!fpu_base) return ENOMEM; for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) { phys_t page; errno_t s = mem_phys_alloc(1, 0, &page); if (s) { vm_free(fpu_base, fpu_size); return s; } pmap_map(&kernel_vas.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK); } context->fpu_region = fpu_base; context->ds = asm_read_ds(); context->es = asm_read_es(); context->fs = asm_read_fs(); context->gs = asm_read_gs(); context->fs_base = asm_rdmsr(MSR_FS_BASE); context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE); } return 0; } [[__naked]] void arch_sched_preempt_disable() { asm volatile( "inc qword ptr gs:%c0\n" "ret" : : "i"(offsetof(struct percpu, sched.preempt_level)) ); } [[__naked]] bool arch_sched_preempt_enable() { asm volatile( "dec qword ptr gs:%c0\n" "setz al\n" "movzx eax, al\n" "ret\n" : : "i"(offsetof(struct percpu, sched.preempt_level)) : "cc" ); } [[__naked]] void arch_sched_jump_to_context(struct arch_context* context) { asm volatile( "mov rsp, rdi\n" "jmp %c0\n" ::"i"(interrupt_return) ); } void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) { struct arch_context context = { .rip = ip, .rsp = sp, .rdi = arg, .rflags = 0x202, .cs = offsetof(struct gdt, user_code64) | CPL_USER, .ss = offsetof(struct gdt, user_data) | CPL_USER, }; uintptr_t percpu = asm_rdmsr(MSR_GS_BASE); const uint16_t zero = 0; asm volatile( "mov ds, %0\n" "mov es, %0\n" "mov fs, %0\n" "mov gs, %0\n" : : "r"(zero) ); asm_wrmsr(MSR_FS_BASE, 0); asm_wrmsr(MSR_GS_BASE, percpu); asm_wrmsr(MSR_KERNEL_GS_BASE, 0); arch_sched_jump_to_context(&context); }