marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

foo

Browse Source
This commit is contained in:
Marvin
2026-09-28 17:46:45 +02:00
committed by Marvin Friedrich
parent 09926f45ee
commit 4a20a5b0af
62 changed files with 1266 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/x86_64/apic.c
+32
View File
@@ -54,6 +54,10 @@ void lapic_init(struct local_apic* lapic) {
asm_wrmsr(0x1B, apic_msr);
lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
if (lapic->xapic_regs)
lapic->id >>= 24;
// Reset the TPR.
lapic_write_reg(lapic, APIC_REG_TPR, 0);
// Enable APIC bit in the SIVR.
@@ -84,6 +88,34 @@ void lapic_eoi(struct local_apic* lapic) {
lapic_write_reg(lapic, APIC_REG_EOI, 0);
}
void lapic_send_ipi(
struct local_apic* lapic,
uint32_t target,
uint8_t vector,
enum apic_delivery_mode delivery_mode,
enum apic_dest_mode dest_mode,
enum apic_level level,
enum apic_trigger_mode trigger_mode
) {
uint64_t icr = vector;
icr |= (uint64_t)delivery_mode << 8;
icr |= (uint64_t)dest_mode << 11;
icr |= (uint64_t)level << 14;
icr |= (uint64_t)trigger_mode << 15;
if (lapic->xapic_regs)
icr |= (uint64_t)(uint8_t)target << 56;
else
icr |= (uint64_t)target << 32;
lapic_write_reg(lapic, APIC_REG_ICR, icr);
if (lapic->xapic_regs) {
while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12))
asm volatile("pause");
}
}
constexpr uint16_t PIC1_COMMAND_PORT = 0x20;
constexpr uint16_t PIC1_DATA_PORT = 0x21;
constexpr uint16_t PIC2_COMMAND_PORT = 0xA0;
arch/x86_64/gdt.c
+3 -3
View File
@@ -1,13 +1,13 @@
#include <zinnia/percpu.h>
#include <stddef.h>
#include <zinnia/string.h>
#include <stddef.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/percpu.h>
#include <x86_64/tss.h>
static const struct gdt main_gdt = {
const struct gdt gdt_base = {
.null = {0},
.kernel_code32 = {.val = 0x00cf9b000000ffff},
.kernel_data32 = {.val = 0x00cf93000000ffff},
@@ -26,7 +26,7 @@ static const struct gdt main_gdt = {
void gdt_load() {
struct gdt* gdt = &percpu_ref(arch_data)->gdt;
memcpy(gdt, &main_gdt, sizeof(main_gdt));
memcpy(gdt, &gdt_base, sizeof(gdt_base));
struct tss* tss = &percpu_ref(arch_data)->tss;
tss->iopb = sizeof(struct tss);
arch/x86_64/hpet.c
+1 -1
View File
@@ -35,7 +35,7 @@ static void hpet_reset(struct clock* c) {
static uint64_t hpet_get_elapsed_ns(struct clock* c) {
struct hpet* hpet = containerof(c, struct hpet, clock);
uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
return (counter * (uint64_t)hpet->period / 1'000'000);
return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
}
static struct hpet hpet_clock = {
arch/x86_64/idt.c
+53 -11
View File
@@ -5,6 +5,9 @@
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/usercopy.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <bits/sched.h>
#include <stddef.h>
@@ -36,29 +39,68 @@ static_assert(sizeof(struct idtr) == 10);
void interrupt_handler(struct arch_context* ctx) {
struct percpu* cpu = percpu_get();
struct arch_percpu* arch_cpu = percpu_ref(arch_data);
bool old = irq_set_interrupted(true);
uint64_t isr = ctx->isr;
switch (isr) {
case IDT_PF:
uint64_t cr2;
asm volatile("mov %0, cr2" : "=r"(cr2));
panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
case IDT_IPI_RESCHED:
arch_sched_preempt_disable();
if (arch_sched_preempt_enable()) {
lapic_eoi(&arch_cpu->lapic);
sched_reschedule(&cpu->sched);
const struct vma_fault_info info = {
.ip = ctx->rip,
.addr = cr2,
.caused_by_user = (ctx->cs & CPL_USER) == CPL_USER,
.caused_by_write = ctx->error & (1 << 1),
.caused_by_fetch = ctx->error & (1 << 4),
.page_was_present = ctx->error & (1 << 0),
};
struct task* current = cpu->sched.current;
const bool irqs_were_on = ctx->rflags & RFLAGS_IF;
if (irqs_were_on)
irq_set_state(true);
const bool handled = current && current->space && vma_page_fault(current->space, &info);
if (irqs_were_on)
irq_set_state(false);
if (handled)
break;
if (current && current->usercopy_region) {
struct usercopy_region* region = current->usercopy_region;
if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) {
ctx->rip = (uintptr_t)region->fault_ip;
break;
}
}
panic(
"Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n",
info.caused_by_write ? "write to"
: info.caused_by_fetch ? "execute on"
: "read from",
info.page_was_present ? "present" : "non-present",
info.addr,
info.ip
);
case IDT_SPURIOUS:
break;
case IDT_IPI_RESCHED:
lapic_eoi(&arch_cpu->lapic);
arch_sched_preempt_disable();
if (arch_sched_preempt_enable())
sched_reschedule(&cpu->sched);
break;
default:
if (isr < 0x20)
panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu\n", isr);
struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
assert(line, "Unhandled interrupt on ISR %lu\n", isr);
// TODO: irq_raise(line);
irq_line_raise(line);
}
irq_set_interrupted(old);
}
#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
arch/x86_64/include/x86_64/apic.h
+11
View File
@@ -55,4 +55,15 @@ enum apic_regs {
void lapic_init(struct local_apic* lapic);
void lapic_eoi(struct local_apic* lapic);
void lapic_send_ipi(
struct local_apic* lapic,
uint32_t target,
uint8_t vector,
enum apic_delivery_mode delivery_mode,
enum apic_dest_mode dest_mode,
enum apic_level level,
enum apic_trigger_mode trigger_mode
);
void pic_disable();
arch/x86_64/include/x86_64/asm.h
+2 -2
View File
@@ -55,13 +55,13 @@ static inline void asm_fxrstor(const void* mem) {
// Saves the FPU state to a region of memory using XSAVE.
// Pointer must be 16-byte aligned.
static inline void asm_xsave(void* mem) {
asm volatile("xsave [%0]" ::"r"(mem) : "memory");
asm volatile("xsave [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
}
// Restores the FPU state from a region of memory using XRSTOR.
// Pointer must be 16-byte aligned.
static inline void asm_xrstor(const void* mem) {
asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
asm volatile("xrstor [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
}
static inline uint8_t asm_inb(uint16_t port) {
arch/x86_64/include/x86_64/gdt.h
+2 -2
View File
@@ -1,7 +1,7 @@
#pragma once
#include <stdint.h>
#include <zinnia/compiler.h>
#include <stdint.h>
union gdt_desc {
uint64_t val;
@@ -50,7 +50,7 @@ struct [[__packed]] gdtr {
struct gdt* base;
};
struct percpu;
extern const struct gdt gdt_base;
// Initializes a GDT on the local CPU.
void gdt_load();
arch/x86_64/include/x86_64/percpu.h
+4
View File
@@ -1,6 +1,7 @@
#pragma once
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/gdt.h>
@@ -18,3 +19,6 @@ struct arch_percpu {
};
extern struct arch_percpu arch_data;
// Initializes the calling CPU and makes `cpu` its per-CPU block.
void x86_64_cpu_setup(struct percpu* cpu);
arch/x86_64/include/x86_64/pmap.h
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include <zinnia/pmap.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
arch/x86_64/include/x86_64/smp.h
+6 -31
View File
@@ -1,23 +1,10 @@
#pragma once
#define SMP_GDTR_OFFSET 80
#define SMP_FARJMP_OFFSET 90
#define SMP_TEMP_STACK_OFFSET 98
#define SMP_TEMP_CR3_OFFSET 102
#define SMP_ENTRY_OFFSET 106
#define SMP_HHDM_OFFSET 114
#define SMP_KERNEL32_DS 16
#define SMP_KERNEL64_CS 24
#define SMP_KERNEL64_DS 32
#define SMP_INFO_SIZE 127
#ifndef __ASSEMBLER__
#include <zinnia/compiler.h>
#include <zinnia/utils.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/gdt.h>
#include <zinnia/compiler.h>
#include <zinnia/utils.h>
struct [[__packed]] smp_info {
struct gdt gdt;
@@ -28,22 +15,10 @@ struct [[__packed]] smp_info {
uint32_t temp_cr3;
uint64_t entry;
uint64_t hhdm_offset;
uint64_t percpu;
uint32_t lapic_id;
uint8_t booted;
_Atomic(uint8_t) booted;
};
static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr));
static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset));
static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack));
static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3));
static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry));
static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset));
static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32));
static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64));
static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64));
static_assert(SMP_INFO_SIZE == sizeof(struct smp_info));
// Initializes a LAPIC.
void x86_64_smp_init(uint32_t id);
#endif
// Discovers all APs and starts them.
void x86_64_smp_init();
arch/x86_64/irq.c
+5 -3
View File
@@ -8,11 +8,13 @@ bool arch_irq_get_state() {
return flags & RFLAGS_IF;
}
void arch_irq_set_state(bool state) {
bool arch_irq_set_state(bool state) {
bool old = arch_irq_get_state();
if (state)
asm volatile("sti");
asm volatile("sti" ::: "memory");
else
asm volatile("cli");
asm volatile("cli" ::: "memory");
return old;
}
void arch_irq_wait() {
arch/x86_64/mem.c
+1 -1
View File
@@ -32,5 +32,5 @@ uintptr_t arch_mem_mapping_addr() {
}
bool arch_mem_addr_is_user(uintptr_t addr) {
return addr < 0x0000'8000'0000'0000;
return addr < 0x0000'7FFF'FFFF'F000;
}
arch/x86_64/meson.build
-1
View File
@@ -20,7 +20,6 @@ kernel_sources += files(
'smp.c',
'sys.c',
'syscall.c',
'trampoline.S',
'tsc.c',
'usercopy.c',
)
arch/x86_64/percpu.c
+4 -2
View File
@@ -9,6 +9,7 @@
#include <x86_64/hpet.h>
#include <x86_64/idt.h>
#include <x86_64/percpu.h>
#include <x86_64/smp.h>
#include <x86_64/syscall.h>
#include <x86_64/tsc.h>
@@ -43,7 +44,7 @@ static_assert(sizeof(union ia32_star) == 8);
[[__percpu]]
struct arch_percpu arch_data = {0};
static void setup_cpu(struct percpu* cpu) {
void x86_64_cpu_setup(struct percpu* cpu) {
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
asm_wrmsr(MSR_FS_BASE, 0);
@@ -131,7 +132,8 @@ void arch_percpu_init() {
#endif
// Init the BSP.
setup_cpu(&percpu_bsp);
x86_64_cpu_setup(&percpu_bsp);
// Init any APs.
x86_64_smp_init();
}
arch/x86_64/pmap.c
+1 -17
View File
@@ -1,22 +1,6 @@
#include <zinnia/pmap.h>
#include <zinnia/types.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
#include <x86_64/pmap.h>
void pte_clear(pte_t* pte) {
*pte = 0;
arch/x86_64/sched.c
+27 -29
View File
@@ -25,30 +25,30 @@ struct task_frame {
static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) {
asm volatile(
"sub rsp, 0x30\n" // Make room for all regs (except RIP).
"mov [rsp + %c0], rbx\n"
"mov [rsp + %c1], rbp\n"
"mov [rsp + %c2], r12\n"
"mov [rsp + %c3], r13\n"
"mov [rsp + %c4], r14\n"
"mov [rsp + %c5], r15\n"
"mov [rsp + %c[rbx]], rbx\n"
"mov [rsp + %c[rbp]], rbp\n"
"mov [rsp + %c[r12]], r12\n"
"mov [rsp + %c[r13]], r13\n"
"mov [rsp + %c[r14]], r14\n"
"mov [rsp + %c[r15]], r15\n"
"mov [rdi], rsp\n" // rdi = old_rsp
"mov rsp, rsi\n" // rsi = new_rsp
"mov rbx, [rsp + %c0]\n"
"mov rbp, [rsp + %c1]\n"
"mov r12, [rsp + %c2]\n"
"mov r13, [rsp + %c3]\n"
"mov r14, [rsp + %c4]\n"
"mov r15, [rsp + %c5]\n"
"mov rbx, [rsp + %c[rbx]]\n"
"mov rbp, [rsp + %c[rbp]]\n"
"mov r12, [rsp + %c[r12]]\n"
"mov r13, [rsp + %c[r13]]\n"
"mov r14, [rsp + %c[r14]]\n"
"mov r15, [rsp + %c[r15]]\n"
"add rsp, 0x30\n"
"mov rax, rdx\n"
"ret" // This will conveniently move us to the RIP we put at this stack entry.
:
: "i"(offsetof(struct task_frame, rbx)),
"i"(offsetof(struct task_frame, rbp)),
"i"(offsetof(struct task_frame, r12)),
"i"(offsetof(struct task_frame, r13)),
"i"(offsetof(struct task_frame, r14)),
"i"(offsetof(struct task_frame, r15))
: [rbx] "i"(offsetof(struct task_frame, rbx)),
[rbp] "i"(offsetof(struct task_frame, rbp)),
[r12] "i"(offsetof(struct task_frame, r12)),
[r13] "i"(offsetof(struct task_frame, r13)),
[r14] "i"(offsetof(struct task_frame, r14)),
[r15] "i"(offsetof(struct task_frame, r15))
: "memory"
);
}
@@ -60,7 +60,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
if (from->user_stack) {
if (from_ctx->fpu_region != nullptr) {
cpu->fpu_save(from_ctx->fpu_region);
from_ctx->ds = asm_read_ds();
from_ctx->es = asm_read_es();
@@ -70,7 +70,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
if (to->user_stack) {
if (to_ctx->fpu_region != nullptr) {
cpu->fpu_restore(to_ctx->fpu_region);
asm_write_ds(to_ctx->ds);
asm_write_es(to_ctx->es);
@@ -124,16 +124,14 @@ errno_t arch_task_init(
uintptr_t fpu_base = vm_alloc(fpu_size);
if (!fpu_base)
return ENOMEM;
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
phys_t page;
errno_t s = mem_phys_alloc(1, 0, &page);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
pmap_map(&kernel_vma.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
phys_t pages;
errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size())
pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
context->fpu_region = (void*)fpu_base;
context->ds = asm_read_ds();
arch/x86_64/smp.c
+342 -7
View File
@@ -1,17 +1,352 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/clock.h>
#include <zinnia/compiler.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/string.h>
#include <zinnia/types.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>
#include <uacpi/acpi.h>
#include <uacpi/tables.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/percpu.h>
#include <x86_64/pmap.h>
#include <x86_64/smp.h>
extern uint8_t smp_trampoline_start[];
extern uint8_t smp_trampoline_data[];
extern uint8_t smp_trampoline_end[];
[[noreturn]]
static void ap_entry(phys_t info) {
// TODO
[[__naked, __used]]
static void smp_trampoline() {
asm volatile(
".pushsection .rodata\n"
".code16\n"
".global smp_trampoline_start\n"
"smp_trampoline_start:\n"
" cli\n"
" cld\n"
" jmp .Lmode16\n"
while (1) {
asm volatile("hlt");
}
".global smp_trampoline_data\n"
"smp_trampoline_data:\n"
" .skip %c[info_size]\n"
".set .Ldata_offset, (smp_trampoline_data - smp_trampoline_start)\n"
".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
".Lmode16:\n"
" mov bx, cs\n"
" shl ebx, 4\n"
" lidtd cs:(.Linvalid_idtr - smp_trampoline_start)\n"
" lgdtd cs:.Lgdtr_offset\n"
" lea eax, [ebx + (.Lmode32 - smp_trampoline_start)]\n"
" mov dword ptr cs:.Lfarjmp_offset, eax\n"
" mov eax, %c[cr0]\n"
" mov cr0, eax\n"
" jmp fword ptr cs:.Lfarjmp_offset\n"
".code32\n"
".Lmode32:\n"
" mov ax, %c[kernel32_ds]\n"
" mov ds, ax\n"
" mov es, ax\n"
" mov fs, ax\n"
" mov gs, ax\n"
" mov ss, ax\n"
" xor eax, eax\n"
" lldt ax\n"
" mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n"
" xor eax, eax\n"
" mov cr4, eax\n"
" or eax, %c[cr4_pae]\n"
" mov cr4, eax\n"
" mov ecx, %c[msr_efer]\n"
" mov eax, %c[efer_lme]\n"
" xor edx, edx\n"
" wrmsr\n"
" mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n"
" mov cr3, eax\n"
" mov eax, cr0\n"
" or eax, %c[cr0_pg]\n"
" mov cr0, eax\n"
" lea eax, [ebx + (.Lmode64 - smp_trampoline_start)]\n"
" push %c[kernel64_cs]\n"
" push eax\n"
" retf\n"
".code64\n"
".Lmode64:\n"
" mov ax, %c[kernel64_ds]\n"
" mov ds, ax\n"
" mov es, ax\n"
" mov fs, ax\n"
" mov gs, ax\n"
" mov ss, ax\n"
" mov ecx, %c[msr_efer]\n"
" rdmsr\n"
" or eax, %c[efer_nxe]\n"
" wrmsr\n"
" xor ebp, ebp\n"
" lea rdi, [rbx + .Ldata_offset]\n"
" mov rax, [rbx + .Lentry_offset]\n"
" add rsp, [rbx + .Lhhdm_offset]\n"
" jmp rax\n"
".Linvalid_idtr:\n"
" .word 0\n"
" .quad 0\n"
".global smp_trampoline_end\n"
"smp_trampoline_end:\n"
".popsection\n"
" ud2\n"
:
: [info_size] "i"(sizeof(struct smp_info)),
[gdtr] "i"(offsetof(struct smp_info, gdtr)),
[farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
[temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
[temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)),
[entry] "i"(offsetof(struct smp_info, entry)),
[hhdm] "i"(offsetof(struct smp_info, hhdm_offset)),
[kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)),
[kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
[kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
[cr0] "i"(CR0_PE | CR0_ET),
[cr0_pg] "i"(CR0_PG),
[cr4_pae] "i"(CR4_PAE),
[msr_efer] "i"(MSR_EFER),
[efer_lme] "i"(MSR_EFER_LME),
[efer_nxe] "i"(MSR_EFER_NXE)
);
}
void x86_64_smp_init(uint32_t id) {}
constexpr size_t AP_BOOT_STACK_SIZE = 0x8000;
constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000;
[[noreturn]]
static void ap_entry(phys_t info_addr) {
// We're still running on the temporary page table.
pmap_set(&kernel_vma.pmap);
struct smp_info* info = HHDM_PTR(info_addr);
struct percpu* cpu = (struct percpu*)info->percpu;
x86_64_cpu_setup(cpu);
irq_lock();
// This task keeps running on the boot stack, but is never scheduled again after the first yield.
sched_init(&cpu->sched);
// Let the BSP know that we're alive.
// `info` must not be touched after this point.
atomic_store_explicit(&info->booted, 1, memory_order_release);
log("smp: CPU %zu online\n", cpu->id);
sched_yield(&cpu->sched);
unreachable();
}
static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
phys_t stack;
if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
log("smp: Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
return false;
}
struct percpu* cpu = percpu_new();
if (!cpu) {
log("smp: Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
return false;
}
log("smp: Starting CPU %zu (LAPIC %u)\n", cpu->id, lapic_id);
memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start);
struct smp_info* info = HHDM_PTR(info_addr);
memcpy(&info->gdt, &gdt_base, sizeof(info->gdt));
info->gdtr.limit = sizeof(struct gdt) - 1;
info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt));
info->farjmp_offset = 0; // Filled in by the trampoline.
info->farjmp_segment = offsetof(struct gdt, kernel_code32);
info->temp_stack = stack + AP_BOOT_STACK_SIZE;
info->temp_cr3 = temp_cr3;
info->entry = (uint64_t)ap_entry;
info->hhdm_offset = mem_hhdm_base;
info->percpu = (uint64_t)cpu;
info->lapic_id = lapic_id;
atomic_init(&info->booted, 0);
struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE);
clock_spin_ns(10'000'000);
lapic_send_ipi(
lapic,
lapic_id,
trampoline >> 12,
APIC_DELIVERY_STARTUP,
APIC_DEST_PHYSICAL,
APIC_LEVEL_ASSERT,
APIC_TRIGGER_EDGE
);
clock_spin_ns(10'000'000);
for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) {
if (waited >= AP_BOOT_TIMEOUT_NS) {
// The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
log("smp: CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
return false;
}
clock_spin_ns(1'000'000);
}
return true;
}
static errno_t build_temp_table(phys_t* out) {
const size_t pgsz = arch_mem_page_size();
phys_t mem;
errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem);
if (status)
return status;
uint64_t* l4 = HHDM_PTR(mem);
uint64_t* l3 = HHDM_PTR(mem + pgsz);
const phys_t l2_base = mem + 2 * pgsz;
// Use 2 MiB pages.
l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
for (size_t i = 0; i < 4; i++) {
uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz);
l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
for (size_t j = 0; j < 512; j++)
l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE;
}
// Copy over the higher half from the kernel page table.
const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root);
for (size_t i = 256; i < 512; i++)
l4[i] = kernel_l4[i];
*out = mem;
return 0;
}
void x86_64_smp_init() {
const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page\n");
const size_t max_cpus = percpu_max_cpus();
if (max_cpus <= 1)
return;
uacpi_table table;
if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
log("smp: No MADT found, not starting any APs\n");
return;
}
phys_t trampoline;
if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
log("smp: Failed to allocate 20-bit memory for the AP trampoline\n");
uacpi_table_unref(&table);
return;
}
phys_t temp_table;
if (build_temp_table(&temp_table)) {
log("smp: Failed to allocate a temporary page table\n");
mem_phys_free(trampoline, 1);
uacpi_table_unref(&table);
return;
}
const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id;
bool all_booted = true;
size_t num_cpus = 1;
const struct acpi_madt* madt = table.ptr;
const uint8_t* entry = (const uint8_t*)madt->entries;
const uint8_t* end = (const uint8_t*)madt + madt->hdr.length;
while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) {
const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry;
// Don't trust malformed firmware tables.
if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
log("smp: MADT contains a malformed entry\n");
break;
}
uint32_t lapic_id = 0;
bool enabled = false;
switch (hdr->type) {
case ACPI_MADT_ENTRY_TYPE_LAPIC:
const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr;
lapic_id = lapic->id;
enabled = lapic->flags & ACPI_PIC_ENABLED;
break;
case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC:
const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr;
lapic_id = x2apic->id;
enabled = x2apic->flags & ACPI_PIC_ENABLED;
break;
}
if (enabled && lapic_id != bsp_id) {
if (start_ap(trampoline, temp_table, lapic_id))
num_cpus++;
else
all_booted = false;
}
entry += hdr->length;
}
uacpi_table_unref(&table);
// An AP which didn't respond might still be using these.
if (all_booted) {
mem_phys_free(temp_table, 6);
mem_phys_free(trampoline, 1);
}
log("smp: %zu CPU(s) online\n", num_cpus);
}
arch/x86_64/syscall.c
+16 -17
View File
@@ -12,17 +12,17 @@
void arch_syscall_stub() {
asm volatile(
" swapgs\n"
" mov gs:%c0, rsp\n" // user_stack
" mov rsp, gs:%c1\n" // kernel_stack
" mov gs:%c[user_stack], rsp\n"
" mov rsp, gs:%c[kernel_stack]\n"
" cld\n"
"1:\n"
" push %c3\n" // ss
" push gs:%c0\n"
" push r11\n" // rflags
" push %c2\n" // cs
" push rcx\n" // rip
" push 0x00\n" // arch_context::error
" push 0x00\n" // arch_context::isr
" push %c[user_data]\n" // ss
" push gs:%c[user_stack]\n"
" push r11\n" // rflags
" push %c[user_code64]\n" // cs
" push rcx\n" // rip
" push 0x00\n" // arch_context::error
" push 0x00\n" // arch_context::isr
" push rax\n"
" push rbx\n"
" push rcx\n"
@@ -39,8 +39,8 @@ void arch_syscall_stub() {
" push r14\n"
" push r15\n"
" xor rbp, rbp\n"
" mov rdi, rsp\n" // SysV ABI first argument.
" call %4\n" // Call syscall handler.
" mov rdi, rsp\n" // SysV ABI first argument.
" call syscall_dispatch\n" // Call syscall handler.
" cli\n"
" pop r15\n"
" pop r14\n"
@@ -59,14 +59,13 @@ void arch_syscall_stub() {
" pop rax\n"
" add rsp, 0x10\n" // Skip .error and .isr fields
"2:\n"
" mov rsp, gs:%c0\n" // user_stack
" mov rsp, gs:%c[user_stack]\n"
" swapgs\n"
" sysretq\n"
:
: "i"(offsetof(struct percpu, user_stack)), // 0
"i"(offsetof(struct percpu, kernel_stack)), // 1
"i"(offsetof(struct gdt, user_code64) | CPL_USER), // 2
"i"(offsetof(struct gdt, user_data) | CPL_USER), // 3
"r"(syscall_dispatch) // 4
: [user_stack] "i"(offsetof(struct percpu, user_stack)),
[kernel_stack] "i"(offsetof(struct percpu, kernel_stack)),
[user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER),
[user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER)
);
}
arch/x86_64/trampoline.S
-107
View File
@@ -1,107 +0,0 @@
#include <x86_64/smp.h>
#include <x86_64/defs.h>
.intel_syntax noprefix
.global smp_trampoline_start
.global smp_trampoline_end
.section .rodata
.code16
smp_trampoline_start:
cli
cld
jmp 1f
smp_trampoline_data:
.skip SMP_INFO_SIZE
.set data_offset, (smp_trampoline_data - smp_trampoline_start)
.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET)
.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET)
.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET)
.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET)
.set entry_offset, (data_offset + SMP_ENTRY_OFFSET)
.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET)
1:
mov bx, cs
shl ebx, 4
.set idtr_offset, (invalid_idtr - smp_trampoline_start)
lidtd cs:idtr_offset
lgdtd cs:gdtr_offset
.set mode32_offset, (mode32 - smp_trampoline_start)
lea eax, [ebx + mode32_offset]
mov dword ptr cs:farjmp_offset, eax
mov eax, CR0_PE | CR0_ET
mov cr0, eax
jmp fword ptr cs:farjmp_offset
.code32
mode32:
mov ax, SMP_KERNEL32_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
xor eax, eax
lldt ax
mov esp, dword ptr [ebx + temp_stack_offset]
xor eax, eax
mov cr4, eax
or eax, CR4_PAE
mov cr4, eax
mov ecx, MSR_EFER
mov eax, MSR_EFER_LME
xor edx, edx
wrmsr
mov eax, dword ptr [ebx + temp_cr3_offset]
mov cr3, eax
mov eax, cr0
or eax, CR0_PG
mov cr0, eax
.set mode64_offset, (mode64 - smp_trampoline_start)
lea eax, [ebx + mode64_offset]
push SMP_KERNEL64_CS
push eax
retf
.code64
mode64:
mov ax, SMP_KERNEL64_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
mov ecx, MSR_EFER
rdmsr
or eax, MSR_EFER_NXE
wrmsr
xor ebp, ebp
lea rdi, [rbx + data_offset]
mov rax, [rbx + entry_offset]
add rsp, [rbx + hhdm_offset]
jmp rax
invalid_idtr:
.word 0
.quad 0
smp_trampoline_end:
arch/x86_64/tsc.c
+3 -2
View File
@@ -53,8 +53,9 @@ void tsc_init() {
// Check for the TSC info leaf.
bool has_tsc_leaf = false;
struct cpuid tsc_leaf = {0};
if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) {
if (asm_cpuid(0, 0).eax >= 0x15) {
tsc_leaf = asm_cpuid(0x15, 0);
has_tsc_leaf = true;
}
if (clock_available()) {
@@ -69,7 +70,7 @@ void tsc_init() {
// since at this point we have at least the HPET timer.
if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
log("x86_64/tsc: Calibrating TSC using CPUID 0x15");
tsc_freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
} else {
log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information");
return;
arch/x86_64/usercopy.c
+14 -8
View File
@@ -1,7 +1,12 @@
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <arch/usercopy.h>
#include <x86_64/percpu.h>
void arch_usercopy_allow(bool state) {
if (!percpu_ref(arch_data)->can_smap)
return;
if (state) {
asm volatile("stac" ::: "memory");
} else {
@@ -24,7 +29,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
asm volatile(
// Setup regs for `rep movsb`.
" xchg rcx, rdx\n"
" mov rax, [rip + %0]\n"
" lea rax, [rip + %0]\n"
" xchg [rdx], rax\n"
".global x86_64_read_start\n"
@@ -41,7 +46,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
".global x86_64_read_fault\n"
"x86_64_read_fault:\n"
" xor rax, rax\n"
" xchg [rdx], rax\n"
" mov [rdx], rax\n"
" ret\n"
:
: "i"(&read_region)
@@ -64,7 +69,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
asm volatile(
// Setup regs for `rep movsb`.
" xchg rcx, rdx\n"
" mov rax, [rip + %0]\n"
" lea rax, [rip + %0]\n"
" mov [rdx], rax\n"
".global x86_64_write_start\n"
@@ -81,7 +86,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
".global x86_64_write_fault\n"
"x86_64_write_fault:\n"
" xor rax, rax\n"
" xchg [rdx], rax\n"
" mov [rdx], rax\n"
" ret\n"
:
: "i"(&write_region)
@@ -102,18 +107,19 @@ static const struct usercopy_region strlen_region = {
[[__naked]]
bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
asm volatile(
" mov rax, [rip + %0]\n"
" lea rax, [rip + %0]\n"
" xchg [rcx], rax\n"
".global x86_64_strlen_start\n"
" x86_64_strlen_start:\n"
" xor r8, r8\n"
".Lloop:\n"
" cmp r8, rsi\n"
" je .Lleave\n"
" cmp byte ptr [rdi + r8], 0\n"
" je .Lleave\n"
" inc r8\n"
" cmp rsi, r8\n"
" jne .Lloop\n"
" jmp .Lloop\n"
".Lleave:\n"
" mov [rdx], r8\n"
@@ -127,7 +133,7 @@ bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struc
".global x86_64_strlen_fault\n"
"x86_64_strlen_fault:\n"
" xor rax, rax\n"
" xchg [rcx], rax\n"
" mov [rcx], rax\n"
" ret\n"
:
: "i"(&strlen_region)