marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test

Browse Source
This commit is contained in:
Marvin
2026-07-20 01:53:04 +02:00
commit e71245c082
149 changed files with 8571 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/x86_64/apic.c
+102
View File
@@ -0,0 +1,102 @@
#include <zinnia/clock.h>
#include <zinnia/mmio.h>
#include <zinnia/percpu.h>
#include <assert.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
static uint64_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
if (lapic->xapic_regs.base) {
if (reg.offset == APIC_REG_ICR.offset) {
uint64_t lo, hi;
lo = mmio_load(&lapic->xapic_regs, APIC_REG_ICR);
hi = mmio_load(&lapic->xapic_regs, APIC_REG_ICR_HI);
return (hi << 32) | lo;
} else {
return mmio_load(&lapic->xapic_regs, reg);
}
} else {
return asm_rdmsr(0x800 + (reg.offset >> 4));
}
}
static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint64_t value) {
if (lapic->xapic_regs.base) {
if (reg.offset == APIC_REG_ICR.offset) {
mmio_store(&lapic->xapic_regs, APIC_REG_ICR_HI, value >> 32);
mmio_store(&lapic->xapic_regs, APIC_REG_ICR, (uint32_t)value);
} else {
mmio_store(&lapic->xapic_regs, reg, (uint32_t)value);
}
} else {
asm_wrmsr(0x800 + (reg.offset >> 4), value);
}
}
void lapic_init(struct local_apic* lapic) {
uint64_t apic_msr = asm_rdmsr(0x1B);
apic_msr |= 1 << 11; // Enable APIC.
struct cpuid cpuid = asm_cpuid(1, 0);
if (cpuid.ecx & CPUID_1C_X2APIC)
apic_msr |= 1 << 10;
else {
lapic->xapic_regs = mmio_new(apic_msr & 0xFFFF'F000, 0x1000);
assert(lapic->xapic_regs.base, "Failed to allocate virtual memory!\n");
}
asm_wrmsr(0x1B, apic_msr);
// Reset the TPR.
lapic_write_reg(lapic, APIC_REG_TPR, 0);
// Enable APIC bit in the SIVR.
lapic_write_reg(lapic, APIC_REG_SIVR, lapic_read_reg(lapic, APIC_REG_SIVR) | 0x100);
if (lapic->xapic_regs.base) {
lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
// Logical destination = LAPIC ID.
lapic_write_reg(lapic, APIC_REG_LDR, lapic_read_reg(lapic, APIC_REG_ID));
}
// TODO: Parse MADT and setup NMI sources.
// Tell the APIC timer to divide by 16.
lapic_write_reg(lapic, APIC_REG_DCR, 3);
// Set the timer to the highest value possible.
lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0xFFFF'FFFF);
// Sleep for 10 milliseconds.
clock_spin_ns(10'000'000);
lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
}
void lapic_eoi(struct local_apic* lapic) {
lapic_write_reg(lapic, APIC_REG_EOI, 0);
}
#define PIC1_COMMAND_PORT 0x20
#define PIC1_DATA_PORT 0x21
#define PIC2_COMMAND_PORT 0xA0
#define PIC2_DATA_PORT 0xA1
void pic_disable() {
// We initialize the PIC properly, but completely disable it and use the APIC in favor of it.
// Remap IRQs so they start at 0x20 since interrupts 0x00..0x1F are used by CPU exceptions.
asm_outb(PIC1_COMMAND_PORT, 0x11);
asm_outb(PIC1_DATA_PORT, 0x20);
asm_outb(PIC1_DATA_PORT, 0x04);
asm_outb(PIC1_DATA_PORT, 0x01);
asm_outb(PIC1_DATA_PORT, 0xFF);
// Same for the slave PIC.
asm_outb(PIC2_COMMAND_PORT, 0x11);
asm_outb(PIC2_DATA_PORT, 0x28);
asm_outb(PIC2_DATA_PORT, 0x02);
asm_outb(PIC2_DATA_PORT, 0x01);
asm_outb(PIC2_DATA_PORT, 0xFF);
}
arch/x86_64/archctl.c
+18
View File
@@ -0,0 +1,18 @@
#include <zinnia/errno.h>
#include <uapi/archctl.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
errno_t arch_archctl(uintptr_t op, uintptr_t arg) {
switch (op) {
case __ARCHCTL_SET_FSBASE:
asm_wrmsr(MSR_FS_BASE, arg);
return 0;
case __ARCHCTL_SET_GSBASE:
// Update the inactive GSBASE MSR, not ours.
asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
return 0;
default:
return EINVAL;
}
}
arch/x86_64/gdt.c
+76
View File
@@ -0,0 +1,76 @@
#include <zinnia/percpu.h>
#include <stddef.h>
#include <string.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/tss.h>
static const struct gdt main_gdt = {
.null = {.val = 0},
.kernel_code32 = {.val = 0x00cf9b000000ffff},
.kernel_data32 = {.val = 0x00cf93000000ffff},
.kernel_code64 = {.val = 0x00a09b0000000000},
.kernel_data64 = {.val = 0x0000930000000000},
.user_code32 = {.val = 0x00cffb000000ffff},
.user_data = {.val = 0x0000f30000000000},
.user_code64 = {.val = 0x00a0fb0000000000},
.tss = {
.limit0 = sizeof(struct tss) - 1,
.access = 0x89, // Present | DPL=0 | System | 64-bit TSS
.limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F,
.reserved = 0,
},
};
void gdt_load() {
struct gdt* gdt = &percpu_get()->arch.gdt;
memcpy(gdt, &main_gdt, sizeof(main_gdt));
struct tss* tss = &percpu_get()->arch.tss;
tss->iopb = sizeof(struct tss);
uintptr_t tss_addr = (uintptr_t)tss;
gdt->tss.base0 = tss_addr;
gdt->tss.base1 = tss_addr >> 16;
gdt->tss.base2 = tss_addr >> 24;
gdt->tss.base3 = tss_addr >> 32;
struct gdtr gdtr = {
.limit = sizeof(struct gdt) - 1,
.base = gdt,
};
asm volatile("lgdt %0" ::"m"(gdtr));
// Save the contents of MSR_GS_BASE, as they get cleared by a write to `gs`.
uintptr_t gs = asm_rdmsr(MSR_GS_BASE);
// Flush and reload the segment registers.
asm volatile(
"push %0\n"
"lea rax, [rip + 1f]\n"
"push rax\n"
"retfq\n"
"1:\n"
"mov ax, %1\n"
"mov ds, ax\n"
"mov es, ax\n"
"mov fs, ax\n"
"mov gs, ax\n"
"mov ss, ax\n"
:
: "i"(offsetof(struct gdt, kernel_code64)), "i"(offsetof(struct gdt, kernel_data64))
: "rax", "memory"
);
asm_wrmsr(MSR_GS_BASE, gs);
asm volatile(
"mov ax, %c0\n"
"ltr ax"
:
: "i"(offsetof(struct gdt, tss))
: "rax", "memory"
);
}
arch/x86_64/hpet.c
+69
View File
@@ -0,0 +1,69 @@
#include <zinnia/clock.h>
#include <zinnia/cmdline.h>
#include <zinnia/mmio.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <assert.h>
#include <uacpi/acpi.h>
#include <uacpi/status.h>
#include <uacpi/tables.h>
#include <uacpi/uacpi.h>
static bool use_hpet = true;
static void hpet_option(bool use) {
use_hpet = use;
}
CMDLINE_OPTION("hpet", hpet_option);
struct hpet {
struct clock clock;
struct mmio_view regs;
uint32_t period;
};
constexpr mmio_reg32 HPET_CAP = {.offset = 0};
constexpr mmio_reg32 HPET_CFG = {.offset = 0x10};
constexpr mmio_reg32 HPET_COUNTER = {.offset = 0xF0};
static void hpet_reset(struct clock* c) {
struct hpet* hpet = containerof(c, struct hpet, clock);
mmio_store(&hpet->regs, HPET_COUNTER, 0u);
}
static uint64_t hpet_get_elapsed_ns(struct clock* c) {
struct hpet* hpet = containerof(c, struct hpet, clock);
uint64_t counter = mmio_load(&hpet->regs, HPET_COUNTER);
return (counter * (uint64_t)hpet->period / 1'000'000);
}
static struct hpet hpet_clock = {
.clock = {
.name = "hpet",
.priority = 128,
.reset = hpet_reset,
.get_elapsed_ns = hpet_get_elapsed_ns,
},
};
static char table_buf[0x1000];
void hpet_init() {
if (!use_hpet)
return;
uacpi_setup_early_table_access(table_buf, sizeof(table_buf));
uacpi_table table = {0};
assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
struct acpi_hpet* hpet = table.ptr;
hpet_clock.regs = mmio_new(hpet->address.address, 0x1000);
uacpi_table_unref(&table);
printf("HPET regs at %p\n", hpet_clock.regs.base);
mmio_store(&hpet_clock.regs, HPET_CFG, mmio_load(&hpet_clock.regs, HPET_CFG) | 1);
uint64_t caps = mmio_load(&hpet_clock.regs, HPET_CAP);
hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");
}
arch/x86_64/idt.c
+209
View File
@@ -0,0 +1,209 @@
#include <zinnia/compiler.h>
#include <zinnia/irq.h>
#include <zinnia/panic.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <assert.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
struct [[__packed]] idt_entry {
uint16_t base0;
uint16_t selector;
uint8_t ist;
uint8_t attrib;
uint16_t base1;
uint32_t base2;
uint32_t reserved;
};
static_assert(sizeof(struct idt_entry) == 16);
struct [[__packed]] idtr {
uint16_t limit;
struct idt_entry* base;
};
static_assert(sizeof(struct idtr) == 10);
void interrupt_handler(struct arch_context* ctx) {
struct percpu* cpu = percpu_get();
bool old = irq_set_interrupted(true);
uint64_t isr = ctx->isr;
switch (isr) {
case IDT_PF:
uint64_t cr2;
asm volatile("mov %0, cr2" : "=r"(cr2));
panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
case IDT_IPI_RESCHED:
arch_sched_preempt_disable();
if (arch_sched_preempt_enable()) {
lapic_eoi(&cpu->arch.lapic);
sched_reschedule(&cpu->sched);
}
break;
case 0x20 ...(0x20 + ARRAY_SIZE(cpu->arch.irq_lines)):
struct irq_line* line = cpu->arch.irq_lines[isr - 0x20];
assert(line, "Unhandled interrupt on ISR %lu\n", isr);
// TODO: irq_raise(line);
break;
default:
panic("Unhandled exception on ISR %lu\n", isr);
}
irq_set_interrupted(old);
}
#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
[[__naked]]
void interrupt_return() {
asm volatile(
"pop r15\n"
"pop r14\n"
"pop r13\n"
"pop r12\n"
"pop r11\n"
"pop r10\n"
"pop r9\n"
"pop r8\n"
"pop rsi\n"
"pop rdi\n"
"pop rbp\n"
"pop rdx\n"
"pop rcx\n"
"pop rbx\n"
"pop rax\n"
// Change GS back if we came from user mode.
"cmp word ptr [rsp + %c0], %c1\n"
"je 2f\n"
"swapgs\n"
"2:\n"
// Skip .error and .isr fields.
"add rsp, 0x10\n"
"iretq\n"
:
: "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64))
);
}
[[__naked]]
void interrupt_stub_internal() {
asm volatile(
// Load the kernel GS base if we're coming from user space.
"cmp word ptr [rsp + %c0], %c1\n"
"je 2f\n"
"swapgs\n"
"2:\n"
"push rax\n"
"push rbx\n"
"push rcx\n"
"push rdx\n"
"push rbp\n"
"push rdi\n"
"push rsi\n"
"push r8\n"
"push r9\n"
"push r10\n"
"push r11\n"
"push r12\n"
"push r13\n"
"push r14\n"
"push r15\n"
"cld\n"
// Zero out the base pointer since we can't trust it.
"xor rbp, rbp\n"
// Load the frame as first argument.
"mov rdi, rsp\n"
"call %c2\n"
"jmp %c3\n"
:
: "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64)), "i"(interrupt_handler), "i"(interrupt_return)
);
}
// clang-format off
#define REPEAT_256(M) \
M(0) M(1) M(2) M(3) M(4) M(5) M(6) M(7) \
M(8) M(9) M(10) M(11) M(12) M(13) M(14) M(15) \
M(16) M(17) M(18) M(19) M(20) M(21) M(22) M(23) \
M(24) M(25) M(26) M(27) M(28) M(29) M(30) M(31) \
M(32) M(33) M(34) M(35) M(36) M(37) M(38) M(39) \
M(40) M(41) M(42) M(43) M(44) M(45) M(46) M(47) \
M(48) M(49) M(50) M(51) M(52) M(53) M(54) M(55) \
M(56) M(57) M(58) M(59) M(60) M(61) M(62) M(63) \
M(64) M(65) M(66) M(67) M(68) M(69) M(70) M(71) \
M(72) M(73) M(74) M(75) M(76) M(77) M(78) M(79) \
M(80) M(81) M(82) M(83) M(84) M(85) M(86) M(87) \
M(88) M(89) M(90) M(91) M(92) M(93) M(94) M(95) \
M(96) M(97) M(98) M(99) M(100) M(101) M(102) M(103) \
M(104) M(105) M(106) M(107) M(108) M(109) M(110) M(111) \
M(112) M(113) M(114) M(115) M(116) M(117) M(118) M(119) \
M(120) M(121) M(122) M(123) M(124) M(125) M(126) M(127) \
M(128) M(129) M(130) M(131) M(132) M(133) M(134) M(135) \
M(136) M(137) M(138) M(139) M(140) M(141) M(142) M(143) \
M(144) M(145) M(146) M(147) M(148) M(149) M(150) M(151) \
M(152) M(153) M(154) M(155) M(156) M(157) M(158) M(159) \
M(160) M(161) M(162) M(163) M(164) M(165) M(166) M(167) \
M(168) M(169) M(170) M(171) M(172) M(173) M(174) M(175) \
M(176) M(177) M(178) M(179) M(180) M(181) M(182) M(183) \
M(184) M(185) M(186) M(187) M(188) M(189) M(190) M(191) \
M(192) M(193) M(194) M(195) M(196) M(197) M(198) M(199) \
M(200) M(201) M(202) M(203) M(204) M(205) M(206) M(207) \
M(208) M(209) M(210) M(211) M(212) M(213) M(214) M(215) \
M(216) M(217) M(218) M(219) M(220) M(221) M(222) M(223) \
M(224) M(225) M(226) M(227) M(228) M(229) M(230) M(231) \
M(232) M(233) M(234) M(235) M(236) M(237) M(238) M(239) \
M(240) M(241) M(242) M(243) M(244) M(245) M(246) M(247) \
M(248) M(249) M(250) M(251) M(252) M(253) M(254) M(255)
// clang-format on
#define INTERRUPT_STUB(n) \
[[__naked]] \
static void interrupt_stub_##n() { \
asm volatile( \
".if (%c0 == 8 || (%c0 >= 10 && %c0 <= 14) || %c0 == 17 || %c0 == 21 || %c0 == 29 || %c0 == 30)\n" \
".else\n" \
"push 0\n" \
".endif\n" \
"push %c0\n" \
"jmp %c1\n" \
: \
: "i"(n), "i"(interrupt_stub_internal) \
); \
}
#define INTERRUPT_SETUP(n) \
do { \
uint64_t addr = (uint64_t)interrupt_stub_##n; \
idt[n].base0 = addr & 0xFFFF; \
idt[n].base1 = (addr >> 16) & 0xFFFF; \
idt[n].base2 = (addr >> 32) & 0xFFFF'FFFF; \
idt[n].ist = 0; \
idt[n].attrib = (1 << 7) | 0xE; \
idt[n].selector = offsetof(struct gdt, kernel_code64); \
idt[n].reserved = 0; \
} while (0);
REPEAT_256(INTERRUPT_STUB)
static struct idt_entry idt[256] = {};
void idt_init() {
REPEAT_256(INTERRUPT_SETUP)
}
void idt_load() {
struct idtr idtr = {
.limit = sizeof(idt) - 1,
.base = idt,
};
asm volatile("lidt %0" ::"m"(idtr));
}
arch/x86_64/include/bits/elf.h
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include <zinnia/elf.h>
#define ELF_ARCH_CLASS ELFCLASS64
#define ELF_ARCH_DATA ELFDATA2LSB
#define ELF_ARCH_MACHINE EM_X86_64
#define ELF_R_SYM ELF64_R_SYM
#define ELF_R_TYPE ELF64_R_TYPE
#define ELF_ST_BIND ELF64_ST_BIND
#define ELF_ST_TYPE ELF64_ST_TYPE
#define elf_ehdr elf64_ehdr
#define elf_phdr elf64_phdr
#define elf_dyn elf64_dyn
#define elf_addr elf64_addr
#define elf_off elf64_off
#define elf_nhdr elf64_nhdr
#define elf_auxv elf64_auxv
#define elf_sym elf64_sym
#define elf_rela elf64_rela
#define elf_rel elf64_rel
enum {
R_NONE = 0,
R_COPY = 5,
R_GLOB_DAT = 6,
R_JUMP_SLOT = 7,
R_RELATIVE = 8,
R_IRELATIVE = 37,
};
arch/x86_64/include/bits/irq.h
+21
View File
@@ -0,0 +1,21 @@
#pragma once
#include <stdint.h>
#include <x86_64/defs.h>
static inline bool arch_irq_get_state() {
uint64_t flags;
asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
return flags & RFLAGS_IF;
}
static inline void arch_irq_set_state(bool state) {
if (state)
asm volatile("sti");
else
asm volatile("cli");
}
static inline void arch_irq_wait() {
asm volatile("hlt");
}
arch/x86_64/include/bits/mem.h
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <zinnia/types.h>
static inline size_t arch_mem_page_size() {
return 1 << 12;
}
static inline size_t arch_mem_page_bits() {
return 12;
}
static inline size_t arch_mem_level_bits() {
return 9;
}
static inline size_t arch_mem_num_levels() {
return 4;
}
static inline uintptr_t arch_mem_hhdm_addr() {
return 0xFFFF'8000'0000'0000;
}
static inline uintptr_t arch_vm_pfndb_addr() {
return 0xFFFF'A000'0000'0000;
}
static inline uintptr_t arch_mem_mapping_addr() {
return 0xFFFF'C000'0000'0000;
}
arch/x86_64/include/bits/mmu.h
+5
View File
@@ -0,0 +1,5 @@
#pragma once
#include <stdint.h>
typedef uint64_t pte_t;
arch/x86_64/include/bits/percpu.h
+27
View File
@@ -0,0 +1,27 @@
#pragma once
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/gdt.h>
#include <x86_64/tss.h>
#include <zinnia/irq.h>
static inline struct percpu* arch_percpu_get() {
struct percpu* result;
asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
return result;
}
struct arch_percpu {
struct gdt gdt;
struct tss tss;
struct local_apic lapic;
struct irq_line* irq_lines[128];
size_t fpu_size;
void (*fpu_save)(void*);
void (*fpu_restore)(const void*);
bool can_smap;
};
void arch_percpu_bsp_init();
void arch_percpu_init();
arch/x86_64/include/bits/sched.h
+48
View File
@@ -0,0 +1,48 @@
#pragma once
#include <bits/types.h>
#include <stdint.h>
struct arch_task_context {
uint64_t rsp;
uint8_t* fpu_region;
uint16_t ds, es, fs, gs;
uint64_t fs_base, gs_base;
};
struct arch_context {
uint64_t r15;
uint64_t r14;
uint64_t r13;
uint64_t r12;
uint64_t r11;
uint64_t r10;
uint64_t r9;
uint64_t r8;
uint64_t rsi;
uint64_t rdi;
uint64_t rbp;
uint64_t rdx;
uint64_t rcx;
uint64_t rbx;
uint64_t rax;
uint64_t isr; // Pushed onto the stack by the interrupt handler stubs.
uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code.
uint64_t rip; // The rest is pushed onto the stack by the CPU during an interrupt.
uint64_t cs;
uint64_t rflags;
uint64_t rsp;
uint64_t ss;
};
#define ARCH_CTX_NUM rax
#define ARCH_CTX_A0 rdi
#define ARCH_CTX_A1 rsi
#define ARCH_CTX_A2 rdx
#define ARCH_CTX_A3 r9
#define ARCH_CTX_A4 r8
#define ARCH_CTX_A5 r10
#define ARCH_CTX_A6 r12
#define ARCH_CTX_A7 r13
#define ARCH_CTX_RET0 rax
#define ARCH_CTX_RET1 rdx
arch/x86_64/include/bits/types.h
+6
View File
@@ -0,0 +1,6 @@
#pragma once
#include <stdint.h>
typedef uint64_t phys_t;
typedef uint64_t reg_t;
arch/x86_64/include/x86_64/apic.h
+57
View File
@@ -0,0 +1,57 @@
#pragma once
#include <stdint.h>
#include <zinnia/mmio.h>
struct local_apic {
uint32_t id;
uint32_t ticks_per_10ms;
struct mmio_view xapic_regs;
};
enum apic_delivery_mode {
APIC_DELIVERY_FIXED = 0b000,
APIC_DELIVERY_LOWESTPRIO = 0b001,
APIC_DELIVERY_SMI = 0b010,
APIC_DELIVERY_NMI = 0b100,
APIC_DELIVERY_INIT = 0b101,
APIC_DELIVERY_STARTUP = 0b110,
};
enum apic_dest_mode {
APIC_DEST_PHYSICAL = 0,
APIC_DEST_LOGICAL = 1,
};
enum apic_delivery_status {
APIC_DELIVERY_STATUS_IDLE = 0,
APIC_DELIVERY_STATUS_PENDING = 1,
};
enum apic_level {
APIC_LEVEL_DEASSERT = 0,
APIC_LEVEL_ASSERT = 1,
};
enum apic_trigger_mode {
APIC_TRIGGER_EDGE = 0,
APIC_TRIGGER_LEVEL = 1,
};
constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
constexpr mmio_reg32 APIC_REG_ICR = {.offset = 0x300};
constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
void lapic_init(struct local_apic* lapic);
void lapic_eoi(struct local_apic* lapic);
void pic_disable();
arch/x86_64/include/x86_64/asm.h
+135
View File
@@ -0,0 +1,135 @@
#pragma once
#include <stdint.h>
struct cpuid {
uint32_t eax, ebx, ecx, edx;
};
static inline struct cpuid asm_cpuid(uint32_t leaf, uint32_t subleaf) {
struct cpuid out = {0};
asm volatile("cpuid" : "=a"(out.eax), "=b"(out.ebx), "=c"(out.ecx), "=d"(out.edx) : "0"(leaf), "2"(subleaf));
return out;
}
static inline uint64_t asm_rdmsr(uint32_t msr) {
uint32_t eax;
uint32_t edx;
asm volatile("rdmsr" : "=a"(eax), "=d"(edx) : "c"(msr) : "memory");
return ((uint64_t)edx << 32) | eax;
}
static inline uint64_t asm_rdtsc() {
uint32_t eax;
uint32_t edx;
asm volatile("rdtsc" : "=a"(eax), "=d"(edx) : : "memory");
return ((uint64_t)edx << 32) | eax;
}
// Writes a 64-bit value to a given MSR.
static inline void asm_wrmsr(uint32_t msr, uint64_t val) {
uint32_t eax = (uint32_t)val;
uint32_t edx = val >> 32;
asm volatile("wrmsr" : : "a"(eax), "d"(edx), "c"(msr) : "memory");
}
// Writes a 64-bit value to a control register using XSETBV.
static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
uint32_t eax = val;
uint32_t edx = val >> 32;
asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory");
}
// Saves the FPU state to a 512-byte region of memory using FXSAVE.
// Pointer must be 16-byte aligned.
static inline void asm_fxsave(void* mem) {
asm volatile("fxsave [%0]" ::"r"(mem) : "memory");
}
// Restores the FPU state from a 512-byte region of memory using FXRSTOR.
// Pointer must be 16-byte aligned.
static inline void asm_fxrstor(const void* mem) {
asm volatile("fxrstor [%0]" ::"r"(mem) : "memory");
}
// Saves the FPU state to a region of memory using XSAVE.
// Pointer must be 16-byte aligned.
static inline void asm_xsave(void* mem) {
asm volatile("xsave [%0]" ::"r"(mem) : "memory");
}
// Restores the FPU state from a region of memory using XRSTOR.
// Pointer must be 16-byte aligned.
static inline void asm_xrstor(const void* mem) {
asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
}
static inline uint8_t asm_inb(uint16_t port) {
uint8_t result;
asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
return result;
}
static inline uint16_t asm_inw(uint16_t port) {
uint16_t result;
asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
return result;
}
static inline uint32_t asm_inl(uint16_t port) {
uint32_t result;
asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
return result;
}
static inline void asm_outb(uint16_t port, uint8_t value) {
asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
}
static inline void asm_outw(uint16_t port, uint16_t value) {
asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
}
static inline void asm_outl(uint16_t port, uint32_t value) {
asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
}
static inline uint16_t asm_read_ds() {
uint16_t value = 0;
asm volatile("mov %0, ds" : "=r"(value));
return value;
}
static inline uint16_t asm_read_es() {
uint16_t value = 0;
asm volatile("mov %0, es" : "=r"(value));
return value;
}
static inline uint16_t asm_read_fs() {
uint16_t value = 0;
asm volatile("mov %0, fs" : "=r"(value));
return value;
}
static inline uint16_t asm_read_gs() {
uint16_t value = 0;
asm volatile("mov %0, gs" : "=r"(value));
return value;
}
static inline void asm_write_ds(uint16_t value) {
asm volatile("mov ds, %0" ::"r"(value));
}
static inline void asm_write_es(uint16_t value) {
asm volatile("mov es, %0" ::"r"(value));
}
static inline void asm_write_fs(uint16_t value) {
asm volatile("mov fs, %0" ::"r"(value));
}
static inline void asm_write_gs(uint16_t value) {
asm volatile("mov gs, %0" ::"r"(value));
}
arch/x86_64/include/x86_64/defs.h
+201
View File
@@ -0,0 +1,201 @@
#pragma once
#define CPUID_1C_SSE3 (1 << 0)
#define CPUID_1C_PCLMUL (1 << 1)
#define CPUID_1C_DTES64 (1 << 2)
#define CPUID_1C_MONITOR (1 << 3)
#define CPUID_1C_DS_CPL (1 << 4)
#define CPUID_1C_VMX (1 << 5)
#define CPUID_1C_SMX (1 << 6)
#define CPUID_1C_EST (1 << 7)
#define CPUID_1C_TM2 (1 << 8)
#define CPUID_1C_SSSE3 (1 << 9)
#define CPUID_1C_CID (1 << 10)
#define CPUID_1C_SDBG (1 << 11)
#define CPUID_1C_FMA (1 << 12)
#define CPUID_1C_CX16 (1 << 13)
#define CPUID_1C_XTPR (1 << 14)
#define CPUID_1C_PDCM (1 << 15)
#define CPUID_1C_PCID (1 << 17)
#define CPUID_1C_DCA (1 << 18)
#define CPUID_1C_SSE4_1 (1 << 19)
#define CPUID_1C_SSE4_2 (1 << 20)
#define CPUID_1C_X2APIC (1 << 21)
#define CPUID_1C_MOVBE (1 << 22)
#define CPUID_1C_POPCNT (1 << 23)
#define CPUID_1C_TSC (1 << 24)
#define CPUID_1C_AES (1 << 25)
#define CPUID_1C_XSAVE (1 << 26)
#define CPUID_1C_OSXSAVE (1 << 27)
#define CPUID_1C_AVX (1 << 28)
#define CPUID_1C_F16C (1 << 29)
#define CPUID_1C_RDRAND (1 << 30)
#define CPUID_1C_HYPERVISOR (1 << 31)
#define CPUID_1D_FPU (1 << 0)
#define CPUID_1D_VME (1 << 1)
#define CPUID_1D_DE (1 << 2)
#define CPUID_1D_PSE (1 << 3)
#define CPUID_1D_TSC (1 << 4)
#define CPUID_1D_MSR (1 << 5)
#define CPUID_1D_PAE (1 << 6)
#define CPUID_1D_MCE (1 << 7)
#define CPUID_1D_CX8 (1 << 8)
#define CPUID_1D_APIC (1 << 9)
#define CPUID_1D_SEP (1 << 11)
#define CPUID_1D_MTRR (1 << 12)
#define CPUID_1D_PGE (1 << 13)
#define CPUID_1D_MCA (1 << 14)
#define CPUID_1D_CMOV (1 << 15)
#define CPUID_1D_PAT (1 << 16)
#define CPUID_1D_PSE36 (1 << 17)
#define CPUID_1D_PSN (1 << 18)
#define CPUID_1D_CLFLUSH (1 << 19)
#define CPUID_1D_DS (1 << 21)
#define CPUID_1D_ACPI (1 << 22)
#define CPUID_1D_MMX (1 << 23)
#define CPUID_1D_FXSR (1 << 24)
#define CPUID_1D_SSE (1 << 25)
#define CPUID_1D_SSE2 (1 << 26)
#define CPUID_1D_SS (1 << 27)
#define CPUID_1D_HTT (1 << 28)
#define CPUID_1D_TM (1 << 29)
#define CPUID_1D_IA64 (1 << 30)
#define CPUID_1D_PBE (1 << 31)
#define CPUID_7B_FSGSBASE (1 << 0)
#define CPUID_7B_SGX (1 << 2)
#define CPUID_7B_BMI (1 << 3)
#define CPUID_7B_HLE (1 << 4)
#define CPUID_7B_AVX2 (1 << 5)
#define CPUID_7B_SMEP (1 << 7)
#define CPUID_7B_BMI2 (1 << 8)
#define CPUID_7B_RTM (1 << 11)
#define CPUID_7B_AVX512F (1 << 16)
#define CPUID_7B_AVX512DQ (1 << 17)
#define CPUID_7B_RDSEED (1 << 18)
#define CPUID_7B_ADX (1 << 19)
#define CPUID_7B_SMAP (1 << 20)
#define CPUID_7B_AVX512IFMA (1 << 21)
#define CPUID_7B_CLFLUSHOPT (1 << 23)
#define CPUID_7B_CLWB (1 << 24)
#define CPUID_7B_AVX512CD (1 << 28)
#define CPUID_7B_SHA (1 << 29)
#define CPUID_7B_AVX512BW (1 << 30)
#define CPUID_7B_AVX512VL (1 << 31)
#define CPUID_7C_AVX512VBMI (1 << 1)
#define CPUID_7C_UMIP (1 << 2)
#define CPUID_7C_PKU (1 << 3)
#define CPUID_7C_OSPKE (1 << 4)
#define CPUID_7C_WAITPKG (1 << 5)
#define CPUID_7C_AVX512VBMI2 (1 << 6)
#define CPUID_7C_SHSTK (1 << 7)
#define CPUID_7C_GFNI (1 << 8)
#define CPUID_7C_VAES (1 << 9)
#define CPUID_7C_VPCLMULQDQ (1 << 10)
#define CPUID_7C_AVX512VNNI (1 << 11)
#define CPUID_7C_AVX512BITALG (1 << 12)
#define CPUID_7C_TME_EN (1 << 13)
#define CPUID_7C_AVX512VPOPCNTDQ (1 << 14)
#define CPUID_7C_RDPID (1 << 22)
#define CPUID_7C_KL (1 << 23)
#define CPUID_7C_CLDEMOTE (1 << 25)
#define CPUID_7C_MOVDIRI (1 << 27)
#define CPUID_7C_MOVDIR64B (1 << 28)
#define CPUID_7C_ENQCMD (1 << 29)
#define CPUID_7D_UINTR (1 << 5)
#define CPUID_7D_AVX512VP2INTERSECT (1 << 8)
#define CPUID_7D_SERIALIZE (1 << 14)
#define CPUID_7D_TSXLDTRK (1 << 16)
#define CPUID_7D_PCONFIG (1 << 18)
#define CPUID_7D_IBT (1 << 20)
#define CPUID_7D_AMX_BF16 (1 << 22)
#define CPUID_7D_AVX512FP16 (1 << 23)
#define CPUID_7D_AMX_TILE (1 << 24)
#define CPUID_7D_AMX_INT8 (1 << 25)
#define RFLAGS_CF (1u << 0)
#define RFLAGS_PF (1u << 2)
#define RFLAGS_AF (1u << 4)
#define RFLAGS_ZF (1u << 6)
#define RFLAGS_SF (1u << 7)
#define RFLAGS_TF (1u << 8)
#define RFLAGS_IF (1u << 9)
#define RFLAGS_DF (1u << 10)
#define RFLAGS_OF (1u << 11)
#define RFLAGS_IOPL (3u << 12)
#define RFLAGS_NT (1u << 14)
#define RFLAGS_RF (1u << 16)
#define RFLAGS_VM (1u << 17)
#define RFLAGS_AC (1u << 18)
#define RFLAGS_VIF (1u << 19)
#define RFLAGS_VIP (1u << 20)
#define RFLAGS_ID (1u << 21)
#define XCR0_X87 (1 << 0)
#define XCR0_SSE (1 << 1)
#define XCR0_AVX (1 << 2)
#define XCR0_AXV512_OPMASK (1 << 5)
#define XCR0_AVX512_ZMM_HI256 (1 << 6)
#define XCR0_AVX512_HI16_ZMM (1 << 7)
#define CR0_PE (1 << 0)
#define CR0_MP (1 << 1)
#define CR0_EM (1 << 2)
#define CR0_TS (1 << 3)
#define CR0_ET (1 << 4)
#define CR0_NE (1 << 5)
#define CR0_WP (1 << 16)
#define CR0_AM (1 << 18)
#define CR0_NW (1 << 29)
#define CR0_CD (1 << 30)
#define CR0_PG (1 << 31)
#define CR4_VME (1 << 0)
#define CR4_PVI (1 << 1)
#define CR4_TSD (1 << 2)
#define CR4_DE (1 << 3)
#define CR4_PSE (1 << 4)
#define CR4_PAE (1 << 5)
#define CR4_MCE (1 << 6)
#define CR4_PGE (1 << 7)
#define CR4_PCE (1 << 8)
#define CR4_OSFXSR (1 << 9)
#define CR4_OSXMMEXCPT (1 << 10)
#define CR4_UMIP (1 << 11)
#define CR4_VMXE (1 << 13)
#define CR4_SMXE (1 << 14)
#define CR4_FSGSBASE (1 << 16)
#define CR4_PCIDE (1 << 17)
#define CR4_OSXSAVE (1 << 18)
#define CR4_SMEP (1 << 20)
#define CR4_SMAP (1 << 21)
#define CR4_PKE (1 << 22)
#define CR4_CET (1 << 23)
#define CR4_PKS (1 << 24)
#define MSR_EFER_SCE (1 << 0)
#define MSR_EFER_LME (1 << 8)
#define MSR_EFER_LMA (1 << 10)
#define MSR_EFER_NXE (1 << 11)
#define MSR_EFER_SVME (1 << 12)
#define MSR_EFER_LMSLE (1 << 13)
#define MSR_EFER_FFXSR (1 << 14)
#define MSR_EFER_TCE (1 << 15)
#define MSR_EFER (0xC0000080)
#define MSR_STAR (0xC0000081)
#define MSR_LSTAR (0xC0000082)
#define MSR_CSTAR (0xC0000083)
#define MSR_SFMASK (0xC0000084)
#define MSR_FS_BASE (0xC0000100)
#define MSR_GS_BASE (0xC0000101)
#define MSR_KERNEL_GS_BASE (0xC0000102)
#define CPL_USER (0b11)
#define CPL_KERNEL (0b00)
#define IDT_PF (0xE)
#define IDT_IPI_RESCHED (0xFF)
arch/x86_64/include/x86_64/gdt.h
+56
View File
@@ -0,0 +1,56 @@
#pragma once
#include <stdint.h>
#include <zinnia/compiler.h>
union gdt_desc {
uint64_t val;
struct {
uint16_t limit0;
uint16_t base0;
uint8_t base1;
uint8_t access;
uint8_t limit1_flags;
uint8_t base2;
};
};
static_assert(sizeof(union gdt_desc) == 0x8);
union gdt_long_desc {
uint64_t val[2];
struct {
uint16_t limit0;
uint16_t base0;
uint8_t base1;
uint8_t access;
uint8_t limit1_flags;
uint8_t base2;
uint32_t base3;
uint32_t reserved;
};
};
static_assert(sizeof(union gdt_long_desc) == 0x10);
struct [[__packed]] gdt {
union gdt_desc null;
union gdt_desc kernel_code32;
union gdt_desc kernel_data32;
union gdt_desc kernel_code64;
union gdt_desc kernel_data64;
union gdt_desc user_code32;
union gdt_desc user_data;
union gdt_desc user_code64;
union gdt_long_desc tss;
};
struct [[__packed]] gdtr {
uint16_t limit;
struct gdt* base;
};
struct percpu;
// Initializes a GDT on the local CPU.
void gdt_load();
arch/x86_64/include/x86_64/hpet.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
void hpet_init();
arch/x86_64/include/x86_64/idt.h
+11
View File
@@ -0,0 +1,11 @@
#pragma once
#include <zinnia/init.h>
[[__init]]
void idt_init();
// Loads the IDT on this CPU.
void idt_load();
void interrupt_return();
arch/x86_64/include/x86_64/smp.h
+49
View File
@@ -0,0 +1,49 @@
#pragma once
#define SMP_GDTR_OFFSET 80
#define SMP_FARJMP_OFFSET 90
#define SMP_TEMP_STACK_OFFSET 98
#define SMP_TEMP_CR3_OFFSET 102
#define SMP_ENTRY_OFFSET 106
#define SMP_HHDM_OFFSET 114
#define SMP_KERNEL32_DS 16
#define SMP_KERNEL64_CS 24
#define SMP_KERNEL64_DS 32
#define SMP_INFO_SIZE 127
#ifndef __ASSEMBLER__
#include <stddef.h>
#include <stdint.h>
#include <x86_64/gdt.h>
#include <zinnia/compiler.h>
#include <zinnia/utils.h>
struct [[__packed]] smp_info {
struct gdt gdt;
struct gdtr gdtr;
uint32_t farjmp_offset;
uint32_t farjmp_segment;
uint32_t temp_stack;
uint32_t temp_cr3;
uint64_t entry;
uint64_t hhdm_offset;
uint32_t lapic_id;
uint8_t booted;
};
static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr));
static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset));
static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack));
static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3));
static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry));
static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset));
static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32));
static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64));
static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64));
static_assert(SMP_INFO_SIZE == sizeof(struct smp_info));
// Initializes a LAPIC.
void x86_64_smp_init(uint32_t id);
#endif
arch/x86_64/include/x86_64/syscall.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
void arch_syscall_stub();
arch/x86_64/include/x86_64/tsc.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
void tsc_init();
arch/x86_64/include/x86_64/tss.h
+26
View File
@@ -0,0 +1,26 @@
#pragma once
#include <stdint.h>
#include <zinnia/compiler.h>
struct [[__packed]] tss {
uint32_t reserved0;
uint64_t rsp0;
uint64_t rsp1;
uint64_t rsp2;
uint32_t reserved1;
uint32_t reserved2;
uint64_t ist1;
uint64_t ist2;
uint64_t ist3;
uint64_t ist4;
uint64_t ist5;
uint64_t ist6;
uint64_t ist7;
uint32_t reserved3;
uint32_t reserved4;
uint16_t reserved5;
uint16_t iopb;
};
static_assert(sizeof(struct tss) == 0x68);
arch/x86_64/meson.build
+23
View File
@@ -0,0 +1,23 @@
kernel_c_args += [
'-mgeneral-regs-only',
'-mno-red-zone',
'-mcmodel=kernel',
]
kernel_sources += files(
'apic.c',
'archctl.c',
'gdt.c',
'hpet.c',
'idt.c',
'percpu.c',
'pmap.c',
'sched.c',
'serial.c',
'smp.c',
'sys.c',
'syscall.c',
'trampoline.S',
'tsc.c',
'usercopy.c',
)
arch/x86_64/percpu.c
+116
View File
@@ -0,0 +1,116 @@
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/hpet.h>
#include <x86_64/idt.h>
#include <x86_64/syscall.h>
#include <x86_64/tsc.h>
void arch_percpu_bsp_init() {
asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
pic_disable();
gdt_load();
idt_init();
idt_load();
}
union ia32_star {
uint64_t value;
struct {
uint32_t rsvd;
uint16_t kernel_cs_ss;
uint16_t user_cs_ss;
};
};
static void setup_cpu(struct percpu* cpu) {
asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
gdt_load();
idt_load();
// Syscall extension.
asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
union ia32_star star = {0};
star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
star.user_cs_ss = offsetof(struct gdt, user_code32);
asm_wrmsr(MSR_STAR, star.value);
asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
uint64_t cr0, cr4;
asm volatile("mov %0, cr0" : "=r"(cr0));
asm volatile("mov %0, cr4" : "=r"(cr4));
struct cpuid cpuid1 = asm_cpuid(1, 0);
struct cpuid cpuid7 = asm_cpuid(7, 0);
struct cpuid cpuid13 = asm_cpuid(13, 0);
// Enable SSE.
cr0 &= ~CR0_EM;
cr0 |= CR0_MP;
cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
if (cpuid1.ecx & CPUID_1C_XSAVE) {
cr4 |= CR4_OSXSAVE;
asm volatile("mov cr4, %0" ::"r"(cr4));
uint64_t xcr0 = XCR0_X87 | XCR0_SSE;
// AVX
if (cpuid1.ecx & CPUID_1C_AVX)
xcr0 |= XCR0_AVX;
// AVX-512
if (cpuid7.ebx & CPUID_7B_AVX512F)
xcr0 |= XCR0_AXV512_OPMASK | XCR0_AVX512_ZMM_HI256 | XCR0_AVX512_HI16_ZMM;
asm_wrxcr(0, xcr0);
cpu->arch.fpu_size = cpuid13.ecx;
cpu->arch.fpu_save = asm_xsave;
cpu->arch.fpu_restore = asm_xrstor;
} else {
cpu->arch.fpu_size = 512;
cpu->arch.fpu_save = asm_fxsave;
cpu->arch.fpu_restore = asm_fxrstor;
}
if (cpuid7.ecx & CPUID_7C_UMIP)
cr4 |= CR4_UMIP;
if (cpuid7.ebx & CPUID_7B_SMEP)
cr4 |= CR4_SMEP;
if (cpuid7.ebx & CPUID_7B_SMAP) {
cr4 |= CR4_SMAP;
cpu->arch.can_smap = true;
}
if (cpuid7.ebx & CPUID_7B_FSGSBASE)
cr4 |= CR4_FSGSBASE;
asm volatile("mov %0, cr0" : "=r"(cr0));
asm volatile("mov %0, cr4" : "=r"(cr4));
lapic_init(&cpu->arch.lapic);
cpu->online = true;
}
void arch_percpu_init() {
hpet_init();
tsc_init();
// Init the BSP.
setup_cpu(&percpu_bsp);
// Init any APs.
}
arch/x86_64/pmap.c
+56
View File
@@ -0,0 +1,56 @@
#include <zinnia/pmap.h>
#include <zinnia/types.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
void pte_clear(pte_t* pte) {
*pte = 0;
}
pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
if (flags & PTE_USER)
result |= ARCH_FLAG_USER_MODE;
if (flags & PTE_DIR)
result |= ARCH_FLAG_READ_WRITE;
else {
if (flags & PTE_WRITE)
result |= ARCH_FLAG_READ_WRITE;
if (!(flags & PTE_EXEC))
result |= ARCH_FLAG_EXECUTE_DISABLE;
}
return result;
}
bool pte_is_present(pte_t* pte) {
return *pte & ARCH_FLAG_PRESENT;
}
bool pte_is_dir(pte_t* pte) {
return *pte & ARCH_FLAG_SIZE;
}
phys_t pte_address(pte_t* pte) {
return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
}
void pmap_set(struct pmap* pt) {
asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
}
arch/x86_64/sched.c
+207
View File
@@ -0,0 +1,207 @@
#include <zinnia/alloc.h>
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/sched.h>
#include <zinnia/vas.h>
#include <zinnia/virt.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/idt.h>
// The task frame consists of registers that the C ABI marks as callee-saved.
// If we don't save them, these registers are lost during a context switch.
struct task_frame {
uint64_t rbx, rbp, r12, r13, r14, r15, rip;
};
[[__naked]]
static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) {
asm volatile(
"sub rsp, 0x30\n" // Make room for all regs (except RIP).
"mov [rsp + %c0], rbx\n"
"mov [rsp + %c1], rbp\n"
"mov [rsp + %c2], r12\n"
"mov [rsp + %c3], r13\n"
"mov [rsp + %c4], r14\n"
"mov [rsp + %c5], r15\n"
"mov [rdi], rsp\n" // rdi = old_rsp
"mov rsp, rsi\n" // rsi = new_rsp
"mov rbx, [rsp + %c0]\n"
"mov rbp, [rsp + %c1]\n"
"mov r12, [rsp + %c2]\n"
"mov r13, [rsp + %c3]\n"
"mov r14, [rsp + %c4]\n"
"mov r15, [rsp + %c5]\n"
"add rsp, 0x30\n"
"call %c6\n"
"ret" // This will conveniently move us to the RIP we put at this stack entry.
:
: "i"(offsetof(struct task_frame, rbx)),
"i"(offsetof(struct task_frame, rbp)),
"i"(offsetof(struct task_frame, r12)),
"i"(offsetof(struct task_frame, r13)),
"i"(offsetof(struct task_frame, r14)),
"i"(offsetof(struct task_frame, r15)),
"i"(irq_unlock)
: "memory"
);
}
void arch_sched_switch(struct task* from, struct task* to) {
struct arch_percpu* cpu = &percpu_get()->arch;
struct arch_task_context* from_ctx = &from->context;
struct arch_task_context* to_ctx = &to->context;
cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
if (from->user_stack) {
cpu->fpu_save(from_ctx->fpu_region);
from_ctx->ds = asm_read_ds();
from_ctx->es = asm_read_es();
from_ctx->fs = asm_read_fs();
from_ctx->gs = asm_read_gs();
from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE);
from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
if (to->user_stack) {
cpu->fpu_restore(to_ctx->fpu_region);
asm_write_ds(to_ctx->ds);
asm_write_es(to_ctx->es);
asm_write_fs(to_ctx->fs);
// If we have to change the GS segment we need to reload the MSR, otherwise we lose its value.
if (to_ctx->gs != asm_read_gs()) {
uint64_t percpu = asm_rdmsr(MSR_GS_BASE);
asm_write_gs(to_ctx->gs);
asm_wrmsr(MSR_GS_BASE, percpu);
}
asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base);
asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base);
}
perform_switch(&from_ctx->rsp, to_ctx->rsp);
}
[[__naked]]
void task_entry_thunk() {
asm volatile(
"mov rdi, rbx\n"
"mov rsi, r12\n"
"jmp %c0"
:
: "i"(task_entry)
: "memory"
);
}
errno_t arch_task_init(
struct arch_task_context* context,
void* entry,
uintptr_t arg0,
uintptr_t stack_start,
bool is_user
) {
struct arch_percpu* cpu = &percpu_get()->arch;
// Prepare a dummy stack with an entry point function to return to.
struct task_frame* frame = (struct task_frame*)stack_start - 1;
frame->rbx = (uint64_t)entry;
frame->r12 = arg0;
frame->rip = (uint64_t)task_entry_thunk;
context->rsp = (uint64_t)frame;
if (is_user) {
// Allocate FPU space.
const size_t fpu_size = cpu->fpu_size;
void* fpu_base = vm_alloc(fpu_size);
if (!fpu_base)
return ENOMEM;
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
phys_t page;
errno_t s = mem_phys_alloc(1, 0, &page);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
pmap_map(&kernel_vas.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
}
context->fpu_region = fpu_base;
context->ds = asm_read_ds();
context->es = asm_read_es();
context->fs = asm_read_fs();
context->gs = asm_read_gs();
context->fs_base = asm_rdmsr(MSR_FS_BASE);
context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
return 0;
}
[[__naked]]
void arch_sched_preempt_disable() {
asm volatile(
"inc qword ptr gs:%c0\n"
"ret"
:
: "i"(offsetof(struct percpu, sched.preempt_level))
);
}
[[__naked]]
bool arch_sched_preempt_enable() {
asm volatile(
"dec qword ptr gs:%c0\n"
"setz al\n"
"movzx eax, al\n"
"ret\n"
:
: "i"(offsetof(struct percpu, sched.preempt_level))
: "cc"
);
}
[[__naked]]
void arch_sched_jump_to_context(struct arch_context* context) {
asm volatile(
"mov rsp, rdi\n"
"jmp %c0\n" ::"i"(interrupt_return)
);
}
void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
struct arch_context context = {
.rip = ip,
.rsp = sp,
.rdi = arg,
.rflags = 0x202,
.cs = offsetof(struct gdt, user_code64) | CPL_USER,
.ss = offsetof(struct gdt, user_data) | CPL_USER,
};
uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
const uint16_t zero = 0;
asm volatile(
"mov ds, %0\n"
"mov es, %0\n"
"mov fs, %0\n"
"mov gs, %0\n"
:
: "r"(zero)
);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_GS_BASE, percpu);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
arch_sched_jump_to_context(&context);
}
arch/x86_64/serial.c
+47
View File
@@ -0,0 +1,47 @@
#include <zinnia/console.h>
#include <stdint.h>
#include <x86_64/asm.h>
constexpr uint16_t COM1_BASE = 0x3F8;
static bool is_ready() {
return asm_inb(COM1_BASE + 5) & 0x20;
}
static void serial_write(struct console* con, const char* buf, size_t count) {
for (size_t i = 0; i < count; i++) {
while (!is_ready())
asm volatile("pause");
asm_outb(COM1_BASE, buf[i]);
}
}
bool serial_setup(struct console* con) {
asm_outb(COM1_BASE + 1, 0x00); // Disable interrupts
asm_outb(COM1_BASE + 3, 0x80); // Enable DLAB (set baud rate divisor)
asm_outb(COM1_BASE, 0x01); // Set divisor low byte (115200 baud if 1)
asm_outb(COM1_BASE + 1, 0x00); // Set divisor high byte
asm_outb(COM1_BASE + 3, 0x03); // 8 bits, no parity, one stop bit (8n1)
asm_outb(COM1_BASE + 2, 0xC7); // Enable FIFO, clear them, with 14-byte threshold
asm_outb(COM1_BASE + 4, 0x0B); // IRQs enabled, RTS/DSR set
asm_outb(COM1_BASE + 4, 0x1E); // Set to loopback mode.
asm_outb(COM1_BASE, 0xAE); // Send a test byte.
// If we don't get the same value back, this serial port doesn't work.
if (asm_inb(COM1_BASE) != 0xAE) {
return false;
}
asm_outb(COM1_BASE + 4, 0x0F); // Disable loopback mode.
return true;
}
static struct console serial_con = {
.name = "com1",
.write = serial_write,
.init = serial_setup,
};
DEFINE_CONSOLE(&serial_con);
arch/x86_64/smp.c
+17
View File
@@ -0,0 +1,17 @@
#include <zinnia/compiler.h>
#include <zinnia/types.h>
#include <x86_64/smp.h>
extern uint8_t smp_trampoline_start[];
extern uint8_t smp_trampoline_end[];
[[noreturn]]
static void ap_entry(phys_t info) {
// TODO
while (1) {
asm volatile("hlt");
}
}
void x86_64_smp_init(uint32_t id) {}
arch/x86_64/sys.c
+21
View File
@@ -0,0 +1,21 @@
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/init.h>
#include <zinnia/percpu.h>
#include <uapi/archctl.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
[[__init, __naked, __used]]
void _start() {
asm volatile(
"lea rsp, [rip + %0]\n"
"jmp %1"
:
: "i"(__ld_stack_top), "r"(kernel_entry)
);
}
void arch_panic() {}
arch/x86_64/syscall.c
+68
View File
@@ -0,0 +1,68 @@
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <zinnia/syscall.h>
#include <bits/sched.h>
#include <stddef.h>
#include <x86_64/defs.h>
[[__naked]]
void arch_syscall_stub() {
asm volatile(
"swapgs\n"
"mov gs:%c0, rsp\n"
"mov rsp, gs:%c1\n"
"cld\n"
// We're pretending to be an interrupt, so fill the bottom fields of `Context`.
"push %c3\n" // SS and CS are not changed during SYSCALL. Use `gdt::user_data | CPL_USER`.
"push gs:%c0\n"
"push r11\n" // RFLAGS is moved into r11 by the CPU.
"push %c2\n" // Same as SS. Use `gdt::user_code64 | CPL_USER`
"push rcx\n" // RIP is moved into rcx by the CPU.
"push 0x00\n" // arch_context::error field
"push 0x00\n" // arch_context::isr field
"push rax\n"
"push rbx\n"
"push rcx\n"
"push rdx\n"
"push rbp\n"
"push rdi\n"
"push rsi\n"
"push r8\n"
"push r9\n"
"push r10\n"
"push r11\n"
"push r12\n"
"push r13\n"
"push r14\n"
"push r15\n"
"xor rbp, rbp\n"
"mov rdi, rsp\n" // Put the trap frame struct as first argument.
"call %c4\n" // Call syscall handler
"cli\n"
"pop r15\n"
"pop r14\n"
"pop r13\n"
"pop r12\n"
"pop r11\n"
"pop r10\n"
"pop r9\n"
"pop r8\n"
"pop rsi\n"
"pop rdi\n"
"pop rbp\n"
"pop rdx\n"
"pop rcx\n"
"pop rbx\n"
"pop rax\n"
"add rsp, 0x10\n" // Skip .error and .isr fields (2 * sizeof(uint64_t))
"mov rsp, gs:%c0\n" // Load user stack from `percpu::user_stack`.
"swapgs\n"
"sysretq\n" // Return to user mode.
:
: "i"(offsetof(struct percpu, user_stack)),
"i"(offsetof(struct percpu, kernel_stack)),
"i"(offsetof(struct gdt, user_code64) | CPL_USER),
"i"(offsetof(struct gdt, user_data) | CPL_USER),
"i"(syscall_dispatch)
);
}
arch/x86_64/trampoline.S
+107
View File
@@ -0,0 +1,107 @@
#include <x86_64/smp.h>
#include <x86_64/defs.h>
.intel_syntax noprefix
.global smp_trampoline_start
.global smp_trampoline_end
.section .rodata
.code16
smp_trampoline_start:
cli
cld
jmp 1f
smp_trampoline_data:
.skip SMP_INFO_SIZE
.set data_offset, (smp_trampoline_data - smp_trampoline_start)
.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET)
.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET)
.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET)
.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET)
.set entry_offset, (data_offset + SMP_ENTRY_OFFSET)
.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET)
1:
mov bx, cs
shl ebx, 4
.set idtr_offset, (invalid_idtr - smp_trampoline_start)
lidtd cs:idtr_offset
lgdtd cs:gdtr_offset
.set mode32_offset, (mode32 - smp_trampoline_start)
lea eax, [ebx + mode32_offset]
mov dword ptr cs:farjmp_offset, eax
mov eax, CR0_PE | CR0_ET
mov cr0, eax
jmp fword ptr cs:farjmp_offset
.code32
mode32:
mov ax, SMP_KERNEL32_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
xor eax, eax
lldt ax
mov esp, dword ptr [ebx + temp_stack_offset]
xor eax, eax
mov cr4, eax
or eax, CR4_PAE
mov cr4, eax
mov ecx, MSR_EFER
mov eax, MSR_EFER_LME
xor edx, edx
wrmsr
mov eax, dword ptr [ebx + temp_cr3_offset]
mov cr3, eax
mov eax, cr0
or eax, CR0_PG
mov cr0, eax
.set mode64_offset, (mode64 - smp_trampoline_start)
lea eax, [ebx + mode64_offset]
push SMP_KERNEL64_CS
push eax
retf
.code64
mode64:
mov ax, SMP_KERNEL64_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
mov ecx, MSR_EFER
rdmsr
or eax, MSR_EFER_NXE
wrmsr
xor ebp, ebp
lea rdi, [rbx + data_offset]
mov rax, [rbx + entry_offset]
add rsp, [rbx + hhdm_offset]
jmp rax
invalid_idtr:
.word 0
.quad 0
smp_trampoline_end:
arch/x86_64/tsc.c
+54
View File
@@ -0,0 +1,54 @@
#include <zinnia/clock.h>
#include <zinnia/cmdline.h>
#include <zinnia/print.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
static bool use_tsc = true;
void tsc_init() {
if (!use_tsc)
return;
if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
printf("No invariant TSC detected!\n");
return;
}
// Check for the TSC info leaf.
bool has_tsc_leaf = false;
struct cpuid tsc_leaf = {0};
if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) {
tsc_leaf = asm_cpuid(0x15, 0);
}
uint64_t freq = 0;
if (clock_available()) {
// Calibrate using existing clock.
uint64_t t1 = asm_rdtsc();
clock_spin_ns(10'000'000);
uint64_t t2 = asm_rdtsc();
freq = (t2 - t1) * 100;
} else if (has_tsc_leaf) {
// If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency.
// On a normal system, this should usually never be called and is a last resort
// since at this point we have at least the HPET timer.
if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
printf("Calibrating TSC using CPUID 0x15");
freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
} else {
printf("Unable to calibrate TSC using CPUID frequency information");
return;
}
} else {
printf("Unable to calibrate the TSC without a clock or static frequency info!\n");
return;
}
printf("Timer frequency is %lu MHz\n", freq / 1'000'000);
}
static void tsc_option(bool opt) {
use_tsc = opt;
}
CMDLINE_OPTION("tsc", tsc_option);
arch/x86_64/usercopy.c
+137
View File
@@ -0,0 +1,137 @@
#include <zinnia/bits/usercopy.h>
#include <zinnia/compiler.h>
extern void x86_64_read_start();
extern void x86_64_read_end();
extern void x86_64_read_fault();
static const struct usercopy_region read_region = {
.start_ip = x86_64_read_start,
.end_ip = x86_64_read_end,
.fault_ip = x86_64_read_fault,
};
bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) {
asm volatile(
"stac\n"
// Setup regs for `rep movsb`.
"xchg rcx, rdx\n"
"mov rax, [rip + %0]\n"
"mov [rdx], rax\n"
".global x86_64_read_start\n"
"x86_64_read_start:\n"
"rep movsb\n"
".global x86_64_read_end\n"
"x86_64_read_end:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"mov rax, 1\n"
"clac\n"
"ret\n"
".global x86_64_read_fault\n"
"x86_64_read_fault:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"clac\n"
"ret\n"
:
: "i"(&read_region)
: "memory"
);
return false;
}
extern void x86_64_write_start();
extern void x86_64_write_end();
extern void x86_64_write_fault();
static const struct usercopy_region write_region = {
.start_ip = x86_64_write_start,
.end_ip = x86_64_write_end,
.fault_ip = x86_64_write_fault,
};
bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) {
asm volatile(
"stac\n"
// Setup regs for `rep movsb`.
"xchg rcx, rdx\n"
"mov rax, [rip + %0]\n"
"mov [rdx], rax\n"
".global x86_64_write_start\n"
"x86_64_write_start:\n"
"rep movsb\n"
".global x86_64_write_end\n"
"x86_64_write_end:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"mov rax, 1\n"
"clac\n"
"ret\n"
".global x86_64_write_fault\n"
"x86_64_write_fault:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"clac\n"
"ret\n"
:
: "i"(&read_region)
: "memory"
);
return false;
}
extern void x86_64_strlen_start();
extern void x86_64_strlen_end();
extern void x86_64_strlen_fault();
static const struct usercopy_region strlen_region = {
.start_ip = x86_64_strlen_start,
.end_ip = x86_64_strlen_end,
.fault_ip = x86_64_strlen_fault,
};
[[__naked]]
bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
asm volatile(
"stac\n"
"mov rax, [rip + %0]\n"
"mov [rcx], rax\n"
".global x86_64_strlen_start\n"
"x86_64_strlen_start:\n"
"xor r8, r8\n"
".Lloop:\n"
"cmp byte ptr [rdi + r8], 0\n"
"je .Lleave\n"
"inc r8\n"
"cmp rsi, r8\n"
"jne .Lloop\n"
".Lleave:\n"
"mov [rdx], r8\n"
".global x86_64_strlen_end\n"
"x86_64_strlen_end:\n"
"xor rax, rax\n"
"mov [rcx], rax\n"
"mov rax, 1\n"
"clac\n"
"ret\n"
".global x86_64_strlen_fault\n"
"x86_64_strlen_fault:\n"
"xor rax, rax\n"
"mov [rcx], rax\n"
"clac\n"
"ret\n"
:
: "i"(&strlen_region)
: "memory"
);
}