marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stuff

Browse Source
This commit is contained in:
Marvin
2026-07-15 22:09:13 +02:00
commit cf33867e68
152 changed files with 9663 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/arch/x86_64/apic.c
+103
View File
@@ -0,0 +1,103 @@
#include <zinnia/mem.h>
#include <kernel/assert.h>
#include <kernel/clock.h>
#include <kernel/mmio.h>
#include <kernel/percpu.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
if (lapic->xapic_regs) {
if (reg == APIC_REG_ICR) {
uint64_t lo, hi;
lo = lapic->xapic_regs[APIC_REG_ICR / sizeof(uint32_t)];
hi = lapic->xapic_regs[APIC_REG_ICR_HI / sizeof(uint32_t)];
return (hi << 32) | lo;
} else {
return lapic->xapic_regs[reg / sizeof(uint32_t)];
}
} else {
return asm_rdmsr(0x800 + (reg >> 4));
}
}
static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
if (lapic->xapic_regs) {
if (reg == APIC_REG_ICR) {
lapic->xapic_regs[APIC_REG_ICR_HI / sizeof(uint32_t)] = value >> 32;
lapic->xapic_regs[APIC_REG_ICR / sizeof(uint32_t)] = value & 0xFFFF'FFFF;
} else {
lapic->xapic_regs[reg / sizeof(uint32_t)] = value & 0xFFFF'FFFF;
}
} else {
asm_wrmsr(0x800 + (reg >> 4), value);
}
}
void lapic_init(struct local_apic* lapic) {
uint64_t apic_msr = asm_rdmsr(0x1B);
apic_msr |= 1 << 11; // Enable APIC.
struct cpuid cpuid = asm_cpuid(1, 0);
if (cpuid.ecx & CPUID_1C_X2APIC)
apic_msr |= 1 << 10;
else {
lapic->xapic_regs = mmio_new(apic_msr & 0xFFFF'F000, 0x1000);
ASSERT(lapic->xapic_regs, "Failed to allocate virtual memory!\n");
}
asm_wrmsr(0x1B, apic_msr);
// Reset the TPR.
lapic_write_reg(lapic, APIC_REG_TPR, 0);
// Enable APIC bit in the SIVR.
lapic_write_reg(lapic, APIC_REG_SIVR, lapic_read_reg(lapic, APIC_REG_SIVR) | 0x100);
if (lapic->xapic_regs) {
lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
// Logical destination = LAPIC ID.
lapic_write_reg(lapic, APIC_REG_LDR, lapic_read_reg(lapic, APIC_REG_ID));
}
// TODO: Parse MADT and setup NMI sources.
// Tell the APIC timer to divide by 16.
lapic_write_reg(lapic, APIC_REG_DCR, 3);
// Set the timer to the highest value possible.
lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0xFFFF'FFFF);
// Sleep for 10 milliseconds.
clock_spin_ns(10'000'000);
lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
}
void lapic_eoi(struct local_apic* lapic) {
lapic_write_reg(lapic, APIC_REG_EOI, 0);
}
#define PIC1_COMMAND_PORT 0x20
#define PIC1_DATA_PORT 0x21
#define PIC2_COMMAND_PORT 0xA0
#define PIC2_DATA_PORT 0xA1
void pic_disable() {
// Note: We initialize the PIC properly, but completely disable it and use the APIC in favor of it.
// Remap IRQs so they start at 0x20 since interrupts 0x00..0x1F are used by CPU exceptions.
asm_outb(PIC1_COMMAND_PORT, 0x11); // ICW1: Begin initialization and set cascade mode.
asm_outb(PIC1_DATA_PORT, 0x20); // ICW2: Set where interrupts should be mapped to (0x20-0x27).
asm_outb(PIC1_DATA_PORT, 0x04); // ICW3: Connect IRQ2 (0x04) to the slave PIC.
asm_outb(PIC1_DATA_PORT, 0x01); // ICW4: Set the PIC to operate in 8086/88 mode.
asm_outb(PIC1_DATA_PORT, 0xFF); // Mask all interrupts.
// Same for the slave PIC.
asm_outb(PIC2_COMMAND_PORT, 0x11); // ICW1: Begin initialization.
asm_outb(PIC2_DATA_PORT, 0x28); // ICW2: Set where interrupts should be mapped to (0x28-0x2F).
asm_outb(PIC2_DATA_PORT, 0x02); // ICW3: Connect to master PIC at IRQ2.
asm_outb(PIC2_DATA_PORT, 0x01); // ICW4: Set the PIC to operate in 8086/88 mode.
asm_outb(PIC2_DATA_PORT, 0xFF); // Mask all interrupts.
}
kernel/arch/x86_64/gdt.c
+77
View File
@@ -0,0 +1,77 @@
#include <kernel/percpu.h>
#include <kernel/syscalls.h>
#include <stddef.h>
#include <string.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/tss.h>
static const struct gdt main_gdt = {
.null = {.val = 0},
.kernel_code32 = {.val = 0x00cf9b000000ffff},
.kernel_data32 = {.val = 0x00cf93000000ffff},
.kernel_code64 = {.val = 0x00a09b0000000000},
.kernel_data64 = {.val = 0x0000930000000000},
.user_code32 = {.val = 0x00cffb000000ffff},
.user_data = {.val = 0x0000f30000000000},
.user_code64 = {.val = 0x00a0fb0000000000},
.tss = {
.limit0 = sizeof(struct tss) - 1,
.access = 0x89, // Present | DPL=0 | System | 64-bit TSS
.limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F,
.reserved = 0,
},
};
void gdt_load() {
struct gdt* gdt = &percpu_get()->arch.gdt;
memcpy(gdt, &main_gdt, sizeof(main_gdt));
struct tss* tss = &percpu_get()->arch.tss;
tss->iopb = sizeof(struct tss);
uintptr_t tss_addr = (uintptr_t)tss;
gdt->tss.base0 = tss_addr;
gdt->tss.base1 = tss_addr >> 16;
gdt->tss.base2 = tss_addr >> 24;
gdt->tss.base3 = tss_addr >> 32;
struct gdtr gdtr = {
.limit = sizeof(struct gdt) - 1,
.base = gdt,
};
asm volatile("lgdt %0" ::"m"(gdtr));
// Save the contents of MSR_GS_BASE, as they get cleared by a write to `gs`.
uintptr_t gs = asm_rdmsr(MSR_GS_BASE);
// Flush and reload the segment registers.
asm volatile(
"push %0\n"
"lea rax, [rip + 1f]\n"
"push rax\n"
"retfq\n"
"1:\n"
"mov ax, %1\n"
"mov ds, ax\n"
"mov es, ax\n"
"mov fs, ax\n"
"mov gs, ax\n"
"mov ss, ax\n"
:
: "i"(offsetof(struct gdt, kernel_code64)), "i"(offsetof(struct gdt, kernel_data64))
: "rax", "memory"
);
asm_wrmsr(MSR_GS_BASE, gs);
asm volatile(
"mov ax, %c0\n"
"ltr ax"
:
: "i"(offsetof(struct gdt, tss))
: "rax", "memory"
);
}
kernel/arch/x86_64/hpet.c
+69
View File
@@ -0,0 +1,69 @@
#include <common/utils.h>
#include <kernel/assert.h>
#include <kernel/clock.h>
#include <kernel/cmdline.h>
#include <kernel/mmio.h>
#include <kernel/print.h>
#include <uacpi/acpi.h>
#include <uacpi/status.h>
#include <uacpi/tables.h>
#include <uacpi/uacpi.h>
static bool use_hpet = true;
static void hpet_option(bool use) {
use_hpet = use;
}
CMDLINE_OPTION("hpet", hpet_option);
struct hpet {
struct clock clock;
volatile uint64_t* regs;
uint32_t period;
};
#define HPET_CAP (0x0 / sizeof(uint64_t))
#define HPET_CFG (0x10 / sizeof(uint64_t))
#define HPET_COUNTER (0xF0 / sizeof(uint64_t))
static void hpet_reset(struct clock* c) {
struct hpet* hpet = CONTAINER_OF(c, struct hpet, clock);
hpet->regs[HPET_COUNTER] = 0;
}
static uint64_t hpet_get_elapsed_ns(struct clock* c) {
struct hpet* hpet = CONTAINER_OF(c, struct hpet, clock);
uint64_t counter = hpet->regs[HPET_COUNTER];
return (counter * (uint64_t)hpet->period / 1'000'000);
}
static struct hpet hpet_clock = {
.clock = {
.name = "hpet",
.priority = 128,
.reset = hpet_reset,
.get_elapsed_ns = hpet_get_elapsed_ns,
},
};
static char table_buf[0x1000];
void hpet_init() {
if (!use_hpet)
return;
uacpi_setup_early_table_access(table_buf, sizeof(table_buf));
uacpi_table table = {0};
ASSERT(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
struct acpi_hpet* hpet = table.ptr;
hpet_clock.regs = mmio_new(hpet->address.address, 0x1000);
uacpi_table_unref(&table);
kprintf("HPET regs at %p\n", hpet_clock.regs);
hpet_clock.regs[HPET_CFG] = hpet_clock.regs[HPET_CFG] | 1;
uint64_t caps = hpet_clock.regs[HPET_CAP];
hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
ASSERT(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");
}
kernel/arch/x86_64/idt.c
+209
View File
@@ -0,0 +1,209 @@
#include <common/compiler.h>
#include <kernel/assert.h>
#include <kernel/irq.h>
#include <kernel/panic.h>
#include <kernel/percpu.h>
#include <kernel/print.h>
#include <kernel/sched.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
struct [[__packed]] idt_entry {
uint16_t base0;
uint16_t selector;
uint8_t ist;
uint8_t attrib;
uint16_t base1;
uint32_t base2;
uint32_t reserved;
};
static_assert(sizeof(struct idt_entry) == 16);
struct [[__packed]] idtr {
uint16_t limit;
struct idt_entry* base;
};
static_assert(sizeof(struct idtr) == 10);
void interrupt_handler(struct arch_context* ctx) {
struct percpu* cpu = percpu_get();
bool old = irq_set_interrupted(true);
uint64_t isr = ctx->isr;
switch (isr) {
case IDT_PF:
uint64_t cr2;
asm volatile("mov %0, cr2" : "=r"(cr2));
panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
case IDT_IPI_RESCHED:
arch_sched_preempt_disable();
if (arch_sched_preempt_enable()) {
lapic_eoi(&cpu->arch.lapic);
sched_reschedule(&cpu->sched);
}
break;
case 0x20 ...(0x20 + ARRAY_SIZE(cpu->arch.irq_lines)):
struct irq_line* line = cpu->arch.irq_lines[isr - 0x20];
ASSERT(line, "Unhandled interrupt on ISR %lu\n", isr);
// TODO: irq_raise(line);
break;
default:
panic("Unhandled exception on ISR %lu\n", isr);
}
irq_set_interrupted(old);
}
#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
[[__naked]]
void interrupt_return() {
asm volatile(
"pop r15\n"
"pop r14\n"
"pop r13\n"
"pop r12\n"
"pop r11\n"
"pop r10\n"
"pop r9\n"
"pop r8\n"
"pop rsi\n"
"pop rdi\n"
"pop rbp\n"
"pop rdx\n"
"pop rcx\n"
"pop rbx\n"
"pop rax\n"
// Change GS back if we came from user mode.
"cmp word ptr [rsp + %c0], %c1\n"
"je 2f\n"
"swapgs\n"
"2:\n"
// Skip .error and .isr fields.
"add rsp, 0x10\n"
"iretq\n"
:
: "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64))
);
}
[[__naked]]
void interrupt_stub_internal() {
asm volatile(
// Load the kernel GS base if we're coming from user space.
"cmp word ptr [rsp + %c0], %c1\n"
"je 2f\n"
"swapgs\n"
"2:\n"
"push rax\n"
"push rbx\n"
"push rcx\n"
"push rdx\n"
"push rbp\n"
"push rdi\n"
"push rsi\n"
"push r8\n"
"push r9\n"
"push r10\n"
"push r11\n"
"push r12\n"
"push r13\n"
"push r14\n"
"push r15\n"
"cld\n"
// Zero out the base pointer since we can't trust it.
"xor rbp, rbp\n"
// Load the frame as first argument.
"mov rdi, rsp\n"
"call %c2\n"
"jmp %c3\n"
:
: "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64)), "i"(interrupt_handler), "i"(interrupt_return)
);
}
// clang-format off
#define REPEAT_256(M) \
M(0) M(1) M(2) M(3) M(4) M(5) M(6) M(7) \
M(8) M(9) M(10) M(11) M(12) M(13) M(14) M(15) \
M(16) M(17) M(18) M(19) M(20) M(21) M(22) M(23) \
M(24) M(25) M(26) M(27) M(28) M(29) M(30) M(31) \
M(32) M(33) M(34) M(35) M(36) M(37) M(38) M(39) \
M(40) M(41) M(42) M(43) M(44) M(45) M(46) M(47) \
M(48) M(49) M(50) M(51) M(52) M(53) M(54) M(55) \
M(56) M(57) M(58) M(59) M(60) M(61) M(62) M(63) \
M(64) M(65) M(66) M(67) M(68) M(69) M(70) M(71) \
M(72) M(73) M(74) M(75) M(76) M(77) M(78) M(79) \
M(80) M(81) M(82) M(83) M(84) M(85) M(86) M(87) \
M(88) M(89) M(90) M(91) M(92) M(93) M(94) M(95) \
M(96) M(97) M(98) M(99) M(100) M(101) M(102) M(103) \
M(104) M(105) M(106) M(107) M(108) M(109) M(110) M(111) \
M(112) M(113) M(114) M(115) M(116) M(117) M(118) M(119) \
M(120) M(121) M(122) M(123) M(124) M(125) M(126) M(127) \
M(128) M(129) M(130) M(131) M(132) M(133) M(134) M(135) \
M(136) M(137) M(138) M(139) M(140) M(141) M(142) M(143) \
M(144) M(145) M(146) M(147) M(148) M(149) M(150) M(151) \
M(152) M(153) M(154) M(155) M(156) M(157) M(158) M(159) \
M(160) M(161) M(162) M(163) M(164) M(165) M(166) M(167) \
M(168) M(169) M(170) M(171) M(172) M(173) M(174) M(175) \
M(176) M(177) M(178) M(179) M(180) M(181) M(182) M(183) \
M(184) M(185) M(186) M(187) M(188) M(189) M(190) M(191) \
M(192) M(193) M(194) M(195) M(196) M(197) M(198) M(199) \
M(200) M(201) M(202) M(203) M(204) M(205) M(206) M(207) \
M(208) M(209) M(210) M(211) M(212) M(213) M(214) M(215) \
M(216) M(217) M(218) M(219) M(220) M(221) M(222) M(223) \
M(224) M(225) M(226) M(227) M(228) M(229) M(230) M(231) \
M(232) M(233) M(234) M(235) M(236) M(237) M(238) M(239) \
M(240) M(241) M(242) M(243) M(244) M(245) M(246) M(247) \
M(248) M(249) M(250) M(251) M(252) M(253) M(254) M(255)
// clang-format on
#define INTERRUPT_STUB(n) \
[[__naked]] \
static void interrupt_stub_##n() { \
asm volatile( \
".if (%c0 == 8 || (%c0 >= 10 && %c0 <= 14) || %c0 == 17 || %c0 == 21 || %c0 == 29 || %c0 == 30)\n" \
".else\n" \
"push 0\n" \
".endif\n" \
"push %c0\n" \
"jmp %c1\n" \
: \
: "i"(n), "i"(interrupt_stub_internal) \
); \
}
#define INTERRUPT_SETUP(n) \
do { \
uint64_t addr = (uint64_t)interrupt_stub_##n; \
idt[n].base0 = addr & 0xFFFF; \
idt[n].base1 = (addr >> 16) & 0xFFFF; \
idt[n].base2 = (addr >> 32) & 0xFFFF'FFFF; \
idt[n].ist = 0; \
idt[n].attrib = (1 << 7) | 0xE; \
idt[n].selector = offsetof(struct gdt, kernel_code64); \
idt[n].reserved = 0; \
} while (0);
REPEAT_256(INTERRUPT_STUB)
static struct idt_entry idt[256] = {};
void idt_init() {
REPEAT_256(INTERRUPT_SETUP)
}
void idt_load() {
struct idtr idtr = {
.limit = sizeof(idt) - 1,
.base = idt,
};
asm volatile("lidt %0" ::"m"(idtr));
}
kernel/arch/x86_64/include/bits/irq.h
+21
View File
@@ -0,0 +1,21 @@
#pragma once
#include <stdint.h>
#include <x86_64/defs.h>
static inline bool arch_irq_get_state() {
uint64_t flags;
asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
return flags & RFLAGS_IF;
}
static inline void arch_irq_set_state(bool state) {
if (state)
asm volatile("sti");
else
asm volatile("cli");
}
static inline void arch_irq_wait() {
asm volatile("hlt");
}
kernel/arch/x86_64/include/bits/mem.h
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include <kernel/types.h>
#include <stddef.h>
#include <stdint.h>
static inline size_t arch_mem_page_size() {
return 1 << 12;
}
static inline size_t arch_mem_page_bits() {
return 12;
}
static inline size_t arch_mem_level_bits() {
return 9;
}
static inline size_t arch_mem_num_levels() {
return 4;
}
static inline uintptr_t arch_mem_hhdm_addr() {
return 0xFFFF'8000'0000'0000;
}
static inline uintptr_t arch_vm_pfndb_addr() {
return 0xFFFF'A000'0000'0000;
}
static inline uintptr_t arch_mem_mapping_addr() {
return 0xFFFF'C000'0000'0000;
}
kernel/arch/x86_64/include/bits/mmu.h
+5
View File
@@ -0,0 +1,5 @@
#pragma once
#include <stdint.h>
typedef uint64_t pte_t;
kernel/arch/x86_64/include/bits/percpu.h
+27
View File
@@ -0,0 +1,27 @@
#pragma once
#include <kernel/irq.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/gdt.h>
#include <x86_64/tss.h>
static inline struct percpu* arch_percpu_get() {
struct percpu* result;
asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
return result;
}
struct arch_percpu {
struct gdt gdt;
struct tss tss;
struct local_apic lapic;
struct irq_line* irq_lines[128];
size_t fpu_size;
void (*fpu_save)(void*);
void (*fpu_restore)(const void*);
bool can_smap;
};
void arch_percpu_bsp_init();
void arch_percpu_init();
kernel/arch/x86_64/include/bits/sched.h
+48
View File
@@ -0,0 +1,48 @@
#pragma once
#include <bits/types.h>
#include <stdint.h>
struct arch_task_context {
uint64_t rsp;
uint8_t* fpu_region;
uint16_t ds, es, fs, gs;
uint64_t fs_base, gs_base;
};
struct arch_context {
uint64_t r15;
uint64_t r14;
uint64_t r13;
uint64_t r12;
uint64_t r11;
uint64_t r10;
uint64_t r9;
uint64_t r8;
uint64_t rsi;
uint64_t rdi;
uint64_t rbp;
uint64_t rdx;
uint64_t rcx;
uint64_t rbx;
uint64_t rax;
uint64_t isr; // Pushed onto the stack by the interrupt handler stubs.
uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code.
uint64_t rip; // The rest is pushed onto the stack by the CPU during an interrupt.
uint64_t cs;
uint64_t rflags;
uint64_t rsp;
uint64_t ss;
};
#define ARCH_CTX_NUM rax
#define ARCH_CTX_A0 rdi
#define ARCH_CTX_A1 rsi
#define ARCH_CTX_A2 rdx
#define ARCH_CTX_A3 r9
#define ARCH_CTX_A4 r8
#define ARCH_CTX_A5 r10
#define ARCH_CTX_A6 r12
#define ARCH_CTX_A7 r13
#define ARCH_CTX_RET0 rax
#define ARCH_CTX_RET1 rdx
kernel/arch/x86_64/include/bits/types.h
+6
View File
@@ -0,0 +1,6 @@
#pragma once
#include <stdint.h>
typedef uint64_t phys_t;
typedef uint64_t reg_t;
kernel/arch/x86_64/include/x86_64/apic.h
+58
View File
@@ -0,0 +1,58 @@
#pragma once
#include <stdint.h>
struct local_apic {
uint32_t id;
uint32_t ticks_per_10ms;
volatile uint32_t* xapic_regs;
};
enum apic_delivery_mode {
APIC_DELIVERY_FIXED = 0b000,
APIC_DELIVERY_LOWESTPRIO = 0b001,
APIC_DELIVERY_SMI = 0b010,
APIC_DELIVERY_NMI = 0b100,
APIC_DELIVERY_INIT = 0b101,
APIC_DELIVERY_STARTUP = 0b110,
};
enum apic_dest_mode {
APIC_DEST_PHYSICAL = 0,
APIC_DEST_LOGICAL = 1,
};
enum apic_delivery_status {
APIC_DELIVERY_STATUS_IDLE = 0,
APIC_DELIVERY_STATUS_PENDING = 1,
};
enum apic_level {
APIC_LEVEL_DEASSERT = 0,
APIC_LEVEL_ASSERT = 1,
};
enum apic_trigger_mode {
APIC_TRIGGER_EDGE = 0,
APIC_TRIGGER_LEVEL = 1,
};
enum apic_regs {
APIC_REG_ID = 0x20,
APIC_REG_TPR = 0x80,
APIC_REG_EOI = 0xB0,
APIC_REG_LDR = 0xD0,
APIC_REG_DFR = 0xE0,
APIC_REG_SIVR = 0xF0,
APIC_REG_ESR = 0x280,
APIC_REG_ICR = 0x300,
APIC_REG_ICR_HI = 0x310,
APIC_REG_LVT_TR = 0x320,
APIC_REG_ICR_TIMER = 0x380,
APIC_REG_CCR = 0x390,
APIC_REG_DCR = 0x3E0,
};
void lapic_init(struct local_apic* lapic);
void lapic_eoi(struct local_apic* lapic);
void pic_disable();
kernel/arch/x86_64/include/x86_64/asm.h
+135
View File
@@ -0,0 +1,135 @@
#pragma once
#include <stdint.h>
struct cpuid {
uint32_t eax, ebx, ecx, edx;
};
static inline struct cpuid asm_cpuid(uint32_t leaf, uint32_t subleaf) {
struct cpuid out = {0};
asm volatile("cpuid" : "=a"(out.eax), "=b"(out.ebx), "=c"(out.ecx), "=d"(out.edx) : "0"(leaf), "2"(subleaf));
return out;
}
static inline uint64_t asm_rdmsr(uint32_t msr) {
uint32_t eax;
uint32_t edx;
asm volatile("rdmsr" : "=a"(eax), "=d"(edx) : "c"(msr) : "memory");
return ((uint64_t)edx << 32) | eax;
}
static inline uint64_t asm_rdtsc() {
uint32_t eax;
uint32_t edx;
asm volatile("rdtsc" : "=a"(eax), "=d"(edx) : : "memory");
return ((uint64_t)edx << 32) | eax;
}
// Writes a 64-bit value to a given MSR.
static inline void asm_wrmsr(uint32_t msr, uint64_t val) {
uint32_t eax = (uint32_t)val;
uint32_t edx = val >> 32;
asm volatile("wrmsr" : : "a"(eax), "d"(edx), "c"(msr) : "memory");
}
// Writes a 64-bit value to a control register using XSETBV.
static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
uint32_t eax = val;
uint32_t edx = val >> 32;
asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory");
}
// Saves the FPU state to a 512-byte region of memory using FXSAVE.
// Pointer must be 16-byte aligned.
static inline void asm_fxsave(void* mem) {
asm volatile("fxsave [%0]" ::"r"(mem) : "memory");
}
// Restores the FPU state from a 512-byte region of memory using FXRSTOR.
// Pointer must be 16-byte aligned.
static inline void asm_fxrstor(const void* mem) {
asm volatile("fxrstor [%0]" ::"r"(mem) : "memory");
}
// Saves the FPU state to a region of memory using XSAVE.
// Pointer must be 16-byte aligned.
static inline void asm_xsave(void* mem) {
asm volatile("xsave [%0]" ::"r"(mem) : "memory");
}
// Restores the FPU state from a region of memory using XRSTOR.
// Pointer must be 16-byte aligned.
static inline void asm_xrstor(const void* mem) {
asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
}
static inline uint8_t asm_inb(uint16_t port) {
uint8_t result;
asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
return result;
}
static inline uint16_t asm_inw(uint16_t port) {
uint16_t result;
asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
return result;
}
static inline uint32_t asm_inl(uint16_t port) {
uint32_t result;
asm volatile("in %0, %1" : "=a"(result) : "Nd"(port));
return result;
}
static inline void asm_outb(uint16_t port, uint8_t value) {
asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
}
static inline void asm_outw(uint16_t port, uint16_t value) {
asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
}
static inline void asm_outl(uint16_t port, uint32_t value) {
asm volatile("out %0, %1" : : "Nd"(port), "a"(value));
}
static inline uint16_t asm_read_ds() {
uint16_t value = 0;
asm volatile("mov %0, ds" : "=r"(value));
return value;
}
static inline uint16_t asm_read_es() {
uint16_t value = 0;
asm volatile("mov %0, es" : "=r"(value));
return value;
}
static inline uint16_t asm_read_fs() {
uint16_t value = 0;
asm volatile("mov %0, fs" : "=r"(value));
return value;
}
static inline uint16_t asm_read_gs() {
uint16_t value = 0;
asm volatile("mov %0, gs" : "=r"(value));
return value;
}
static inline void asm_write_ds(uint16_t value) {
asm volatile("mov ds, %0" ::"r"(value));
}
static inline void asm_write_es(uint16_t value) {
asm volatile("mov es, %0" ::"r"(value));
}
static inline void asm_write_fs(uint16_t value) {
asm volatile("mov fs, %0" ::"r"(value));
}
static inline void asm_write_gs(uint16_t value) {
asm volatile("mov gs, %0" ::"r"(value));
}
kernel/arch/x86_64/include/x86_64/defs.h
+201
View File
@@ -0,0 +1,201 @@
#pragma once
#define CPUID_1C_SSE3 (1 << 0)
#define CPUID_1C_PCLMUL (1 << 1)
#define CPUID_1C_DTES64 (1 << 2)
#define CPUID_1C_MONITOR (1 << 3)
#define CPUID_1C_DS_CPL (1 << 4)
#define CPUID_1C_VMX (1 << 5)
#define CPUID_1C_SMX (1 << 6)
#define CPUID_1C_EST (1 << 7)
#define CPUID_1C_TM2 (1 << 8)
#define CPUID_1C_SSSE3 (1 << 9)
#define CPUID_1C_CID (1 << 10)
#define CPUID_1C_SDBG (1 << 11)
#define CPUID_1C_FMA (1 << 12)
#define CPUID_1C_CX16 (1 << 13)
#define CPUID_1C_XTPR (1 << 14)
#define CPUID_1C_PDCM (1 << 15)
#define CPUID_1C_PCID (1 << 17)
#define CPUID_1C_DCA (1 << 18)
#define CPUID_1C_SSE4_1 (1 << 19)
#define CPUID_1C_SSE4_2 (1 << 20)
#define CPUID_1C_X2APIC (1 << 21)
#define CPUID_1C_MOVBE (1 << 22)
#define CPUID_1C_POPCNT (1 << 23)
#define CPUID_1C_TSC (1 << 24)
#define CPUID_1C_AES (1 << 25)
#define CPUID_1C_XSAVE (1 << 26)
#define CPUID_1C_OSXSAVE (1 << 27)
#define CPUID_1C_AVX (1 << 28)
#define CPUID_1C_F16C (1 << 29)
#define CPUID_1C_RDRAND (1 << 30)
#define CPUID_1C_HYPERVISOR (1 << 31)
#define CPUID_1D_FPU (1 << 0)
#define CPUID_1D_VME (1 << 1)
#define CPUID_1D_DE (1 << 2)
#define CPUID_1D_PSE (1 << 3)
#define CPUID_1D_TSC (1 << 4)
#define CPUID_1D_MSR (1 << 5)
#define CPUID_1D_PAE (1 << 6)
#define CPUID_1D_MCE (1 << 7)
#define CPUID_1D_CX8 (1 << 8)
#define CPUID_1D_APIC (1 << 9)
#define CPUID_1D_SEP (1 << 11)
#define CPUID_1D_MTRR (1 << 12)
#define CPUID_1D_PGE (1 << 13)
#define CPUID_1D_MCA (1 << 14)
#define CPUID_1D_CMOV (1 << 15)
#define CPUID_1D_PAT (1 << 16)
#define CPUID_1D_PSE36 (1 << 17)
#define CPUID_1D_PSN (1 << 18)
#define CPUID_1D_CLFLUSH (1 << 19)
#define CPUID_1D_DS (1 << 21)
#define CPUID_1D_ACPI (1 << 22)
#define CPUID_1D_MMX (1 << 23)
#define CPUID_1D_FXSR (1 << 24)
#define CPUID_1D_SSE (1 << 25)
#define CPUID_1D_SSE2 (1 << 26)
#define CPUID_1D_SS (1 << 27)
#define CPUID_1D_HTT (1 << 28)
#define CPUID_1D_TM (1 << 29)
#define CPUID_1D_IA64 (1 << 30)
#define CPUID_1D_PBE (1 << 31)
#define CPUID_7B_FSGSBASE (1 << 0)
#define CPUID_7B_SGX (1 << 2)
#define CPUID_7B_BMI (1 << 3)
#define CPUID_7B_HLE (1 << 4)
#define CPUID_7B_AVX2 (1 << 5)
#define CPUID_7B_SMEP (1 << 7)
#define CPUID_7B_BMI2 (1 << 8)
#define CPUID_7B_RTM (1 << 11)
#define CPUID_7B_AVX512F (1 << 16)
#define CPUID_7B_AVX512DQ (1 << 17)
#define CPUID_7B_RDSEED (1 << 18)
#define CPUID_7B_ADX (1 << 19)
#define CPUID_7B_SMAP (1 << 20)
#define CPUID_7B_AVX512IFMA (1 << 21)
#define CPUID_7B_CLFLUSHOPT (1 << 23)
#define CPUID_7B_CLWB (1 << 24)
#define CPUID_7B_AVX512CD (1 << 28)
#define CPUID_7B_SHA (1 << 29)
#define CPUID_7B_AVX512BW (1 << 30)
#define CPUID_7B_AVX512VL (1 << 31)
#define CPUID_7C_AVX512VBMI (1 << 1)
#define CPUID_7C_UMIP (1 << 2)
#define CPUID_7C_PKU (1 << 3)
#define CPUID_7C_OSPKE (1 << 4)
#define CPUID_7C_WAITPKG (1 << 5)
#define CPUID_7C_AVX512VBMI2 (1 << 6)
#define CPUID_7C_SHSTK (1 << 7)
#define CPUID_7C_GFNI (1 << 8)
#define CPUID_7C_VAES (1 << 9)
#define CPUID_7C_VPCLMULQDQ (1 << 10)
#define CPUID_7C_AVX512VNNI (1 << 11)
#define CPUID_7C_AVX512BITALG (1 << 12)
#define CPUID_7C_TME_EN (1 << 13)
#define CPUID_7C_AVX512VPOPCNTDQ (1 << 14)
#define CPUID_7C_RDPID (1 << 22)
#define CPUID_7C_KL (1 << 23)
#define CPUID_7C_CLDEMOTE (1 << 25)
#define CPUID_7C_MOVDIRI (1 << 27)
#define CPUID_7C_MOVDIR64B (1 << 28)
#define CPUID_7C_ENQCMD (1 << 29)
#define CPUID_7D_UINTR (1 << 5)
#define CPUID_7D_AVX512VP2INTERSECT (1 << 8)
#define CPUID_7D_SERIALIZE (1 << 14)
#define CPUID_7D_TSXLDTRK (1 << 16)
#define CPUID_7D_PCONFIG (1 << 18)
#define CPUID_7D_IBT (1 << 20)
#define CPUID_7D_AMX_BF16 (1 << 22)
#define CPUID_7D_AVX512FP16 (1 << 23)
#define CPUID_7D_AMX_TILE (1 << 24)
#define CPUID_7D_AMX_INT8 (1 << 25)
#define RFLAGS_CF (1u << 0)
#define RFLAGS_PF (1u << 2)
#define RFLAGS_AF (1u << 4)
#define RFLAGS_ZF (1u << 6)
#define RFLAGS_SF (1u << 7)
#define RFLAGS_TF (1u << 8)
#define RFLAGS_IF (1u << 9)
#define RFLAGS_DF (1u << 10)
#define RFLAGS_OF (1u << 11)
#define RFLAGS_IOPL (3u << 12)
#define RFLAGS_NT (1u << 14)
#define RFLAGS_RF (1u << 16)
#define RFLAGS_VM (1u << 17)
#define RFLAGS_AC (1u << 18)
#define RFLAGS_VIF (1u << 19)
#define RFLAGS_VIP (1u << 20)
#define RFLAGS_ID (1u << 21)
#define XCR0_X87 (1 << 0)
#define XCR0_SSE (1 << 1)
#define XCR0_AVX (1 << 2)
#define XCR0_AXV512_OPMASK (1 << 5)
#define XCR0_AVX512_ZMM_HI256 (1 << 6)
#define XCR0_AVX512_HI16_ZMM (1 << 7)
#define CR0_PE (1 << 0)
#define CR0_MP (1 << 1)
#define CR0_EM (1 << 2)
#define CR0_TS (1 << 3)
#define CR0_ET (1 << 4)
#define CR0_NE (1 << 5)
#define CR0_WP (1 << 16)
#define CR0_AM (1 << 18)
#define CR0_NW (1 << 29)
#define CR0_CD (1 << 30)
#define CR0_PG (1 << 31)
#define CR4_VME (1 << 0)
#define CR4_PVI (1 << 1)
#define CR4_TSD (1 << 2)
#define CR4_DE (1 << 3)
#define CR4_PSE (1 << 4)
#define CR4_PAE (1 << 5)
#define CR4_MCE (1 << 6)
#define CR4_PGE (1 << 7)
#define CR4_PCE (1 << 8)
#define CR4_OSFXSR (1 << 9)
#define CR4_OSXMMEXCPT (1 << 10)
#define CR4_UMIP (1 << 11)
#define CR4_VMXE (1 << 13)
#define CR4_SMXE (1 << 14)
#define CR4_FSGSBASE (1 << 16)
#define CR4_PCIDE (1 << 17)
#define CR4_OSXSAVE (1 << 18)
#define CR4_SMEP (1 << 20)
#define CR4_SMAP (1 << 21)
#define CR4_PKE (1 << 22)
#define CR4_CET (1 << 23)
#define CR4_PKS (1 << 24)
#define MSR_EFER_SCE (1 << 0)
#define MSR_EFER_LME (1 << 8)
#define MSR_EFER_LMA (1 << 10)
#define MSR_EFER_NXE (1 << 11)
#define MSR_EFER_SVME (1 << 12)
#define MSR_EFER_LMSLE (1 << 13)
#define MSR_EFER_FFXSR (1 << 14)
#define MSR_EFER_TCE (1 << 15)
#define MSR_EFER (0xC0000080)
#define MSR_STAR (0xC0000081)
#define MSR_LSTAR (0xC0000082)
#define MSR_CSTAR (0xC0000083)
#define MSR_SFMASK (0xC0000084)
#define MSR_FS_BASE (0xC0000100)
#define MSR_GS_BASE (0xC0000101)
#define MSR_KERNEL_GS_BASE (0xC0000102)
#define CPL_USER (0b11)
#define CPL_KERNEL (0b00)
#define IDT_PF (0xE)
#define IDT_IPI_RESCHED (0xFF)
kernel/arch/x86_64/include/x86_64/gdt.h
+56
View File
@@ -0,0 +1,56 @@
#pragma once
#include <common/compiler.h>
#include <stdint.h>
union gdt_desc {
uint64_t val;
struct {
uint16_t limit0;
uint16_t base0;
uint8_t base1;
uint8_t access;
uint8_t limit1_flags;
uint8_t base2;
};
};
static_assert(sizeof(union gdt_desc) == 0x8);
union gdt_long_desc {
uint64_t val[2];
struct {
uint16_t limit0;
uint16_t base0;
uint8_t base1;
uint8_t access;
uint8_t limit1_flags;
uint8_t base2;
uint32_t base3;
uint32_t reserved;
};
};
static_assert(sizeof(union gdt_long_desc) == 0x10);
struct [[__packed]] gdt {
union gdt_desc null;
union gdt_desc kernel_code32;
union gdt_desc kernel_data32;
union gdt_desc kernel_code64;
union gdt_desc kernel_data64;
union gdt_desc user_code32;
union gdt_desc user_data;
union gdt_desc user_code64;
union gdt_long_desc tss;
};
struct [[__packed]] gdtr {
uint16_t limit;
struct gdt* base;
};
struct percpu;
// Initializes a GDT on the local CPU.
void gdt_load();
kernel/arch/x86_64/include/x86_64/hpet.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
void hpet_init();
kernel/arch/x86_64/include/x86_64/idt.h
+11
View File
@@ -0,0 +1,11 @@
#pragma once
#include <kernel/init.h>
[[__init]]
void idt_init();
// Loads the IDT on this CPU.
void idt_load();
void interrupt_return();
kernel/arch/x86_64/include/x86_64/smp.h
+49
View File
@@ -0,0 +1,49 @@
#pragma once
#define SMP_GDTR_OFFSET 80
#define SMP_FARJMP_OFFSET 90
#define SMP_TEMP_STACK_OFFSET 98
#define SMP_TEMP_CR3_OFFSET 102
#define SMP_ENTRY_OFFSET 106
#define SMP_HHDM_OFFSET 114
#define SMP_KERNEL32_DS 16
#define SMP_KERNEL64_CS 24
#define SMP_KERNEL64_DS 32
#define SMP_INFO_SIZE 127
#ifndef __ASSEMBLER__
#include <common/compiler.h>
#include <common/utils.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/gdt.h>
struct [[__packed]] smp_info {
struct gdt gdt;
struct gdtr gdtr;
uint32_t farjmp_offset;
uint32_t farjmp_segment;
uint32_t temp_stack;
uint32_t temp_cr3;
uint64_t entry;
uint64_t hhdm_offset;
uint32_t lapic_id;
uint8_t booted;
};
static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr));
static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset));
static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack));
static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3));
static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry));
static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset));
static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32));
static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64));
static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64));
static_assert(SMP_INFO_SIZE == sizeof(struct smp_info));
// Initializes a LAPIC.
void x86_64_smp_init(uint32_t id);
#endif
kernel/arch/x86_64/include/x86_64/syscall.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
void arch_syscall_stub();
kernel/arch/x86_64/include/x86_64/tsc.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
void tsc_init();
kernel/arch/x86_64/include/x86_64/tss.h
+26
View File
@@ -0,0 +1,26 @@
#pragma once
#include <common/compiler.h>
#include <stdint.h>
struct [[__packed]] tss {
uint32_t reserved0;
uint64_t rsp0;
uint64_t rsp1;
uint64_t rsp2;
uint32_t reserved1;
uint32_t reserved2;
uint64_t ist1;
uint64_t ist2;
uint64_t ist3;
uint64_t ist4;
uint64_t ist5;
uint64_t ist6;
uint64_t ist7;
uint32_t reserved3;
uint32_t reserved4;
uint16_t reserved5;
uint16_t iopb;
};
static_assert(sizeof(struct tss) == 0x68);
kernel/arch/x86_64/meson.build
+22
View File
@@ -0,0 +1,22 @@
kernel_c_args += [
'-mgeneral-regs-only',
'-mno-red-zone',
'-mcmodel=kernel',
]
kernel_sources += files(
'apic.c',
'gdt.c',
'hpet.c',
'idt.c',
'percpu.c',
'pmap.c',
'sched.c',
'serial.c',
'smp.c',
'sys.c',
'syscall.c',
'trampoline.S',
'tsc.c',
'usercopy.c',
)
kernel/arch/x86_64/percpu.c
+116
View File
@@ -0,0 +1,116 @@
#include <kernel/percpu.h>
#include <kernel/print.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/hpet.h>
#include <x86_64/idt.h>
#include <x86_64/syscall.h>
#include <x86_64/tsc.h>
void arch_percpu_bsp_init() {
asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
pic_disable();
gdt_load();
idt_init();
idt_load();
}
union ia32_star {
uint64_t value;
struct {
uint32_t rsvd;
uint16_t kernel_cs_ss;
uint16_t user_cs_ss;
};
};
static void setup_cpu(struct percpu* cpu) {
asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
gdt_load();
idt_load();
// Syscall extension.
asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
union ia32_star star = {0};
star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
star.user_cs_ss = offsetof(struct gdt, user_code32);
asm_wrmsr(MSR_STAR, star.value);
asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
uint64_t cr0, cr4;
asm volatile("mov %0, cr0" : "=r"(cr0));
asm volatile("mov %0, cr4" : "=r"(cr4));
struct cpuid cpuid1 = asm_cpuid(1, 0);
struct cpuid cpuid7 = asm_cpuid(7, 0);
struct cpuid cpuid13 = asm_cpuid(13, 0);
// Enable SSE.
cr0 &= ~CR0_EM;
cr0 |= CR0_MP;
cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
if (cpuid1.ecx & CPUID_1C_XSAVE) {
cr4 |= CR4_OSXSAVE;
asm volatile("mov cr4, %0" ::"r"(cr4));
uint64_t xcr0 = XCR0_X87 | XCR0_SSE;
// AVX
if (cpuid1.ecx & CPUID_1C_AVX)
xcr0 |= XCR0_AVX;
// AVX-512
if (cpuid7.ebx & CPUID_7B_AVX512F)
xcr0 |= XCR0_AXV512_OPMASK | XCR0_AVX512_ZMM_HI256 | XCR0_AVX512_HI16_ZMM;
asm_wrxcr(0, xcr0);
cpu->arch.fpu_size = cpuid13.ecx;
cpu->arch.fpu_save = asm_xsave;
cpu->arch.fpu_restore = asm_xrstor;
} else {
cpu->arch.fpu_size = 512;
cpu->arch.fpu_save = asm_fxsave;
cpu->arch.fpu_restore = asm_fxrstor;
}
if (cpuid7.ecx & CPUID_7C_UMIP)
cr4 |= CR4_UMIP;
if (cpuid7.ebx & CPUID_7B_SMEP)
cr4 |= CR4_SMEP;
if (cpuid7.ebx & CPUID_7B_SMAP) {
cr4 |= CR4_SMAP;
cpu->arch.can_smap = true;
}
if (cpuid7.ebx & CPUID_7B_FSGSBASE)
cr4 |= CR4_FSGSBASE;
asm volatile("mov %0, cr0" : "=r"(cr0));
asm volatile("mov %0, cr4" : "=r"(cr4));
lapic_init(&cpu->arch.lapic);
cpu->online = true;
}
void arch_percpu_init() {
hpet_init();
tsc_init();
// Init the BSP.
setup_cpu(&percpu_bsp);
// Init any APs.
}
kernel/arch/x86_64/pmap.c
+56
View File
@@ -0,0 +1,56 @@
#include <kernel/pmap.h>
#include <kernel/types.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
void pte_clear(pte_t* pte) {
*pte = 0;
}
pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
if (flags & PTE_USER)
result |= ARCH_FLAG_USER_MODE;
if (flags & PTE_DIR)
result |= ARCH_FLAG_READ_WRITE;
else {
if (flags & PTE_WRITE)
result |= ARCH_FLAG_READ_WRITE;
if (!(flags & PTE_EXEC))
result |= ARCH_FLAG_EXECUTE_DISABLE;
}
return result;
}
bool pte_is_present(pte_t* pte) {
return *pte & ARCH_FLAG_PRESENT;
}
bool pte_is_dir(pte_t* pte) {
return *pte & ARCH_FLAG_SIZE;
}
phys_t pte_address(pte_t* pte) {
return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
}
void pmap_set(struct pmap* pt) {
asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
}
kernel/arch/x86_64/sched.c
+206
View File
@@ -0,0 +1,206 @@
#include <zinnia/status.h>
#include <common/compiler.h>
#include <kernel/alloc.h>
#include <kernel/percpu.h>
#include <kernel/sched.h>
#include <kernel/virt.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/idt.h>
// The task frame consists of registers that the C ABI marks as callee-saved.
// If we don't save them, these registers are lost during a context switch.
// The order of these fields is important.
struct task_frame {
uint64_t rbx, rbp, r12, r13, r14, r15, rip;
};
[[__naked]]
static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) {
asm volatile(
"sub rsp, 0x30\n" // Make room for all regs (except RIP).
"mov [rsp + %c0], rbx\n"
"mov [rsp + %c1], rbp\n"
"mov [rsp + %c2], r12\n"
"mov [rsp + %c3], r13\n"
"mov [rsp + %c4], r14\n"
"mov [rsp + %c5], r15\n"
"mov [rdi], rsp\n" // rdi = old_rsp
"mov rsp, rsi\n" // rsi = new_rsp
"mov rbx, [rsp + %c0]\n"
"mov rbp, [rsp + %c1]\n"
"mov r12, [rsp + %c2]\n"
"mov r13, [rsp + %c3]\n"
"mov r14, [rsp + %c4]\n"
"mov r15, [rsp + %c5]\n"
"add rsp, 0x30\n"
"call %c6\n"
"ret" // This will conveniently move us to the RIP we put at this stack entry.
:
: "i"(offsetof(struct task_frame, rbx)),
"i"(offsetof(struct task_frame, rbp)),
"i"(offsetof(struct task_frame, r12)),
"i"(offsetof(struct task_frame, r13)),
"i"(offsetof(struct task_frame, r14)),
"i"(offsetof(struct task_frame, r15)),
"i"(irq_unlock)
: "memory"
);
}
void arch_sched_switch(struct task* from, struct task* to) {
struct arch_percpu* cpu = &percpu_get()->arch;
struct arch_task_context* from_ctx = &from->context;
struct arch_task_context* to_ctx = &to->context;
cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
if (from->user_stack) {
cpu->fpu_save(from_ctx->fpu_region);
from_ctx->ds = asm_read_ds();
from_ctx->es = asm_read_es();
from_ctx->fs = asm_read_fs();
from_ctx->gs = asm_read_gs();
from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE);
from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
if (to->user_stack) {
cpu->fpu_restore(to_ctx->fpu_region);
asm_write_ds(to_ctx->ds);
asm_write_es(to_ctx->es);
asm_write_fs(to_ctx->fs);
// If we have to change the GS segment we need to reload the MSR, otherwise we lose its value.
if (to_ctx->gs != asm_read_gs()) {
uint64_t percpu = asm_rdmsr(MSR_GS_BASE);
asm_write_gs(to_ctx->gs);
asm_wrmsr(MSR_GS_BASE, percpu);
}
asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base);
asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base);
}
perform_switch(&from_ctx->rsp, to_ctx->rsp);
}
[[__naked]]
void task_entry_thunk() {
asm volatile(
"mov rdi, rbx\n"
"mov rsi, r12\n"
"jmp %c0"
:
: "i"(task_entry)
: "memory"
);
}
zn_status_t arch_task_init(
struct arch_task_context* context,
void* entry,
uintptr_t arg0,
uintptr_t stack_start,
bool is_user
) {
struct arch_percpu* cpu = &percpu_get()->arch;
// Prepare a dummy stack with an entry point function to return to.
struct task_frame* frame = (struct task_frame*)stack_start - 1;
frame->rbx = (uint64_t)entry;
frame->r12 = arg0;
frame->rip = (uint64_t)task_entry_thunk;
context->rsp = (uint64_t)frame;
if (is_user) {
// Allocate FPU space.
const size_t fpu_size = cpu->fpu_size;
void* fpu_base = vm_alloc(fpu_size);
if (!fpu_base)
return ZN_ERR_NO_MEMORY;
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
phys_t page;
zn_status_t s = mem_phys_alloc(1, 0, &page);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
pmap_map(&kernel_vas.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
}
context->fpu_region = fpu_base;
context->ds = asm_read_ds();
context->es = asm_read_es();
context->fs = asm_read_fs();
context->gs = asm_read_gs();
context->fs_base = asm_rdmsr(MSR_FS_BASE);
context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
return ZN_OK;
}
[[__naked]]
void arch_sched_preempt_disable() {
asm volatile(
"inc qword ptr gs:%c0\n"
"ret"
:
: "i"(offsetof(struct percpu, sched.preempt_level))
);
}
[[__naked]]
bool arch_sched_preempt_enable() {
asm volatile(
"dec qword ptr gs:%c0\n"
"setz al\n"
"movzx eax, al\n"
"ret\n"
:
: "i"(offsetof(struct percpu, sched.preempt_level))
: "cc"
);
}
[[__naked]]
void arch_sched_jump_to_context(struct arch_context* context) {
asm volatile(
"mov rsp, rdi\n"
"jmp %c0\n" ::"i"(interrupt_return)
);
}
void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
struct arch_context context = {
.rip = ip,
.rsp = sp,
.rdi = arg,
.rflags = 0x202,
.cs = offsetof(struct gdt, user_code64) | CPL_USER,
.ss = offsetof(struct gdt, user_data) | CPL_USER,
};
uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
const uint16_t zero = 0;
asm volatile(
"mov ds, %0\n"
"mov es, %0\n"
"mov fs, %0\n"
"mov gs, %0\n"
:
: "r"(zero)
);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_GS_BASE, percpu);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
arch_sched_jump_to_context(&context);
}
kernel/arch/x86_64/serial.c
+47
View File
@@ -0,0 +1,47 @@
#include <kernel/console.h>
#include <stdint.h>
#include <x86_64/asm.h>
constexpr uint16_t COM1_BASE = 0x3F8;
static bool is_ready() {
return asm_inb(COM1_BASE + 5) & 0x20;
}
static void serial_write(struct console* con, const char* buf, size_t count) {
for (size_t i = 0; i < count; i++) {
while (!is_ready())
asm volatile("pause");
asm_outb(COM1_BASE, buf[i]);
}
}
bool serial_setup(struct console* con) {
asm_outb(COM1_BASE + 1, 0x00); // Disable interrupts
asm_outb(COM1_BASE + 3, 0x80); // Enable DLAB (set baud rate divisor)
asm_outb(COM1_BASE, 0x01); // Set divisor low byte (115200 baud if 1)
asm_outb(COM1_BASE + 1, 0x00); // Set divisor high byte
asm_outb(COM1_BASE + 3, 0x03); // 8 bits, no parity, one stop bit (8n1)
asm_outb(COM1_BASE + 2, 0xC7); // Enable FIFO, clear them, with 14-byte threshold
asm_outb(COM1_BASE + 4, 0x0B); // IRQs enabled, RTS/DSR set
asm_outb(COM1_BASE + 4, 0x1E); // Set to loopback mode.
asm_outb(COM1_BASE, 0xAE); // Send a test byte.
// If we don't get the same value back, this serial port doesn't work.
if (asm_inb(COM1_BASE) != 0xAE) {
return false;
}
asm_outb(COM1_BASE + 4, 0x0F); // Disable loopback mode.
return true;
}
static struct console serial_con = {
.name = "com1",
.write = serial_write,
.init = serial_setup,
};
DEFINE_CONSOLE(&serial_con);
kernel/arch/x86_64/smp.c
+17
View File
@@ -0,0 +1,17 @@
#include <common/compiler.h>
#include <kernel/types.h>
#include <x86_64/smp.h>
extern uint8_t smp_trampoline_start[];
extern uint8_t smp_trampoline_end[];
[[noreturn]]
static void ap_entry(phys_t info) {
// TODO
while (1) {
asm volatile("hlt");
}
}
void x86_64_smp_init(uint32_t id) {}
kernel/arch/x86_64/sys.c
+31
View File
@@ -0,0 +1,31 @@
#include <zinnia/archctl.h>
#include <zinnia/status.h>
#include <common/compiler.h>
#include <kernel/init.h>
#include <kernel/percpu.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
[[__init, __naked, __used]]
void _start() {
asm volatile(
"lea rsp, [rip + %0]\n"
"jmp %1"
:
: "i"(__ld_stack_top), "r"(kernel_entry)
);
}
void arch_panic() {}
zn_status_t arch_archctl(zn_archctl_t op, uintptr_t arg) {
switch (op) {
case ZN_ARCHCTL_SET_FSBASE:
asm_wrmsr(MSR_FS_BASE, arg);
return 0;
default:
return ZN_ERR_INVALID;
}
}
kernel/arch/x86_64/syscall.c
+68
View File
@@ -0,0 +1,68 @@
#include <common/compiler.h>
#include <kernel/percpu.h>
#include <kernel/syscalls.h>
#include <bits/sched.h>
#include <stddef.h>
#include <x86_64/defs.h>
[[__naked]]
void arch_syscall_stub() {
asm volatile(
"swapgs\n"
"mov gs:%c0, rsp\n"
"mov rsp, gs:%c1\n"
"cld\n"
// We're pretending to be an interrupt, so fill the bottom fields of `Context`.
"push %c3\n" // SS and CS are not changed during SYSCALL. Use `gdt::user_data | CPL_USER`.
"push gs:%c0\n"
"push r11\n" // RFLAGS is moved into r11 by the CPU.
"push %c2\n" // Same as SS. Use `gdt::user_code64 | CPL_USER`
"push rcx\n" // RIP is moved into rcx by the CPU.
"push 0x00\n" // arch_context::error field
"push 0x00\n" // arch_context::isr field
"push rax\n"
"push rbx\n"
"push rcx\n"
"push rdx\n"
"push rbp\n"
"push rdi\n"
"push rsi\n"
"push r8\n"
"push r9\n"
"push r10\n"
"push r11\n"
"push r12\n"
"push r13\n"
"push r14\n"
"push r15\n"
"xor rbp, rbp\n"
"mov rdi, rsp\n" // Put the trap frame struct as first argument.
"call %c4\n" // Call syscall handler
"cli\n"
"pop r15\n"
"pop r14\n"
"pop r13\n"
"pop r12\n"
"pop r11\n"
"pop r10\n"
"pop r9\n"
"pop r8\n"
"pop rsi\n"
"pop rdi\n"
"pop rbp\n"
"pop rdx\n"
"pop rcx\n"
"pop rbx\n"
"pop rax\n"
"add rsp, 0x10\n" // Skip .error and .isr fields (2 * sizeof(uint64_t))
"mov rsp, gs:%c0\n" // Load user stack from `percpu::user_stack`.
"swapgs\n"
"sysretq\n" // Return to user mode.
:
: "i"(offsetof(struct percpu, user_stack)),
"i"(offsetof(struct percpu, kernel_stack)),
"i"(offsetof(struct gdt, user_code64) | CPL_USER),
"i"(offsetof(struct gdt, user_data) | CPL_USER),
"i"(syscall_dispatch)
);
}
kernel/arch/x86_64/trampoline.S
+107
View File
@@ -0,0 +1,107 @@
#include <x86_64/smp.h>
#include <x86_64/defs.h>
.intel_syntax noprefix
.global smp_trampoline_start
.global smp_trampoline_end
.section .rodata
.code16
smp_trampoline_start:
cli
cld
jmp 1f
smp_trampoline_data:
.skip SMP_INFO_SIZE
.set data_offset, (smp_trampoline_data - smp_trampoline_start)
.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET)
.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET)
.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET)
.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET)
.set entry_offset, (data_offset + SMP_ENTRY_OFFSET)
.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET)
1:
mov bx, cs
shl ebx, 4
.set idtr_offset, (invalid_idtr - smp_trampoline_start)
lidtd cs:idtr_offset
lgdtd cs:gdtr_offset
.set mode32_offset, (mode32 - smp_trampoline_start)
lea eax, [ebx + mode32_offset]
mov dword ptr cs:farjmp_offset, eax
mov eax, CR0_PE | CR0_ET
mov cr0, eax
jmp fword ptr cs:farjmp_offset
.code32
mode32:
mov ax, SMP_KERNEL32_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
xor eax, eax
lldt ax
mov esp, dword ptr [ebx + temp_stack_offset]
xor eax, eax
mov cr4, eax
or eax, CR4_PAE
mov cr4, eax
mov ecx, MSR_EFER
mov eax, MSR_EFER_LME
xor edx, edx
wrmsr
mov eax, dword ptr [ebx + temp_cr3_offset]
mov cr3, eax
mov eax, cr0
or eax, CR0_PG
mov cr0, eax
.set mode64_offset, (mode64 - smp_trampoline_start)
lea eax, [ebx + mode64_offset]
push SMP_KERNEL64_CS
push eax
retf
.code64
mode64:
mov ax, SMP_KERNEL64_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
mov ecx, MSR_EFER
rdmsr
or eax, MSR_EFER_NXE
wrmsr
xor ebp, ebp
lea rdi, [rbx + data_offset]
mov rax, [rbx + entry_offset]
add rsp, [rbx + hhdm_offset]
jmp rax
invalid_idtr:
.word 0
.quad 0
smp_trampoline_end:
kernel/arch/x86_64/tsc.c
+54
View File
@@ -0,0 +1,54 @@
#include <kernel/clock.h>
#include <kernel/cmdline.h>
#include <kernel/print.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
static bool use_tsc = true;
void tsc_init() {
if (!use_tsc)
return;
if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
kprintf("No invariant TSC detected!\n");
return;
}
// Check for the TSC info leaf.
bool has_tsc_leaf = false;
struct cpuid tsc_leaf = {0};
if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) {
tsc_leaf = asm_cpuid(0x15, 0);
}
uint64_t freq = 0;
if (clock_available()) {
// Calibrate using existing clock.
uint64_t t1 = asm_rdtsc();
clock_spin_ns(10'000'000);
uint64_t t2 = asm_rdtsc();
freq = (t2 - t1) * 100;
} else if (has_tsc_leaf) {
// If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency.
// On a normal system, this should usually never be called and is a last resort
// since at this point we have at least the HPET timer.
if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
kprintf("Calibrating TSC using CPUID 0x15");
freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
} else {
kprintf("Unable to calibrate TSC using CPUID frequency information");
return;
}
} else {
kprintf("Unable to calibrate the TSC without a clock or static frequency info!\n");
return;
}
kprintf("Timer frequency is %lu MHz\n", freq / 1'000'000);
}
static void tsc_option(bool opt) {
use_tsc = opt;
}
CMDLINE_OPTION("tsc", tsc_option);
kernel/arch/x86_64/usercopy.c
+137
View File
@@ -0,0 +1,137 @@
#include <common/compiler.h>
#include <kernel/usercopy.h>
extern void x86_64_read_start();
extern void x86_64_read_end();
extern void x86_64_read_fault();
static const struct usercopy_region read_region = {
.start_ip = x86_64_read_start,
.end_ip = x86_64_read_end,
.fault_ip = x86_64_read_fault,
};
bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) {
asm volatile(
"stac\n"
// Setup regs for `rep movsb`.
"xchg rcx, rdx\n"
"mov rax, [rip + %0]\n"
"mov [rdx], rax\n"
".global x86_64_read_start\n"
"x86_64_read_start:\n"
"rep movsb\n"
".global x86_64_read_end\n"
"x86_64_read_end:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"mov rax, 1\n"
"clac\n"
"ret\n"
".global x86_64_read_fault\n"
"x86_64_read_fault:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"clac\n"
"ret\n"
:
: "i"(&read_region)
: "memory"
);
return false;
}
extern void x86_64_write_start();
extern void x86_64_write_end();
extern void x86_64_write_fault();
static const struct usercopy_region write_region = {
.start_ip = x86_64_write_start,
.end_ip = x86_64_write_end,
.fault_ip = x86_64_write_fault,
};
bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) {
asm volatile(
"stac\n"
// Setup regs for `rep movsb`.
"xchg rcx, rdx\n"
"mov rax, [rip + %0]\n"
"mov [rdx], rax\n"
".global x86_64_write_start\n"
"x86_64_write_start:\n"
"rep movsb\n"
".global x86_64_write_end\n"
"x86_64_write_end:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"mov rax, 1\n"
"clac\n"
"ret\n"
".global x86_64_write_fault\n"
"x86_64_write_fault:\n"
"xor rax, rax\n"
"mov [rdx], rax\n"
"clac\n"
"ret\n"
:
: "i"(&read_region)
: "memory"
);
return false;
}
extern void x86_64_strlen_start();
extern void x86_64_strlen_end();
extern void x86_64_strlen_fault();
static const struct usercopy_region strlen_region = {
.start_ip = x86_64_strlen_start,
.end_ip = x86_64_strlen_end,
.fault_ip = x86_64_strlen_fault,
};
[[__naked]]
bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
asm volatile(
"stac\n"
"mov rax, [rip + %0]\n"
"mov [rcx], rax\n"
".global x86_64_strlen_start\n"
"x86_64_strlen_start:\n"
"xor r8, r8\n"
".Lloop:\n"
"cmp byte ptr [rdi + r8], 0\n"
"je .Lleave\n"
"inc r8\n"
"cmp rsi, r8\n"
"jne .Lloop\n"
".Lleave:\n"
"mov [rdx], r8\n"
".global x86_64_strlen_end\n"
"x86_64_strlen_end:\n"
"xor rax, rax\n"
"mov [rcx], rax\n"
"mov rax, 1\n"
"clac\n"
"ret\n"
".global x86_64_strlen_fault\n"
"x86_64_strlen_fault:\n"
"xor rax, rax\n"
"mov [rcx], rax\n"
"clac\n"
"ret\n"
:
: "i"(&strlen_region)
: "memory"
);
}
kernel/boot/cmdline.c
+102
View File
@@ -0,0 +1,102 @@
#include <kernel/cmdline.h>
#include <kernel/init.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
// Defined in linker script.
extern const struct cmdline_option* __ld_cmdline_start[];
extern const struct cmdline_option* __ld_cmdline_end[];
static char cmdline_buf[0x1000];
[[__init]]
void cmdline_parse(const char* cmdline) {
const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
size_t idx = 0;
memcpy(cmdline_buf, cmdline, len);
while (1) {
const struct cmdline_option** opt = __ld_cmdline_start;
char* name = nullptr;
char* value = nullptr;
// Skip all leading spaces.
while (idx < len && cmdline_buf[idx] == ' ')
idx++;
if (idx >= len)
break;
size_t name_idx = idx;
name = cmdline_buf + name_idx;
// Find the next equal sign or space.
while (idx < len && cmdline_buf[idx] != '=' && cmdline_buf[idx] != ' ')
idx++;
if (idx > len)
break;
// Check if the option has a value (=foo).
char seperator = cmdline_buf[idx];
cmdline_buf[idx++] = 0;
if (seperator == '=') {
// Check if we need to escape the value.
char check;
if (cmdline_buf[idx] == '"') {
check = '"';
cmdline_buf[idx++] = 0;
} else {
check = ' ';
}
value = cmdline_buf + idx;
// Skip the value.
while (idx < len && cmdline_buf[idx] != check)
idx++;
if (idx > len)
break;
cmdline_buf[idx++] = 0;
}
// Find the corresponding option.
while (opt < __ld_cmdline_end) {
const struct cmdline_option* current = *opt;
if (!strcmp(current->name, name)) {
switch (current->option_type) {
case CMDLINE_STRING:
current->func_str(value);
break;
case CMDLINE_BOOL:
if (!strcmp(value, "on") || !strcmp(value, "yes") || !strcmp(value, "true") || !strcmp(value, "1"))
current->func_bool(true);
else
current->func_bool(false);
break;
case CMDLINE_INTPTR: {
int base = 10;
if (!strncmp(value, "0x", 2)) {
base = 0x10;
value += 2;
}
current->func_intptr(atol(value, base));
break;
}
case CMDLINE_UINTPTR: {
int base = 10;
if (!strncmp(value, "0x", 2)) {
base = 0x10;
value += 2;
}
current->func_uintptr(atolu(value, base));
break;
}
}
break;
}
opt++;
}
if (idx >= len)
break;
}
}
kernel/boot/limine/entry.c
+94
View File
@@ -0,0 +1,94 @@
#include <common/utils.h>
#include <kernel/alloc.h>
#include <kernel/assert.h>
#include <kernel/cmdline.h>
#include <kernel/init.h>
#include <kernel/print.h>
#include <string.h>
#include "limine.h"
[[__initdata_sorted("limine.0")]]
static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
[[__initdata_sorted("limine.1")]]
static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(4);
[[__initdata_sorted("limine.2")]]
static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
#define LIMINE_REQUEST(request, tag, rev) \
[[__initdata_sorted("limine.1")]] \
static volatile struct limine_##request request = { \
.id = tag, \
.revision = rev, \
.response = nullptr, \
}
LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
[[__initdata]]
static struct phys_mem mem[128];
[[__initdata]]
static struct boot_file files[8];
[[__init]]
void kernel_entry() {
kernel_early_init();
struct boot_info info;
struct limine_memmap_response* mm_res = memmap_request.response;
struct limine_module_response* module_res = module_request.response;
struct limine_executable_address_response* exec_res = executable_address_request.response;
struct limine_executable_file_response* exec_file_res = executable_file_request.response;
info.cmdline = exec_file_res->executable_file->string;
info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
for (size_t i = 0; i < info.num_mem_maps; i++) {
mem[i].address = mm_res->entries[i]->base;
mem[i].length = mm_res->entries[i]->length;
switch (mm_res->entries[i]->type) {
case LIMINE_MEMMAP_USABLE:
mem[i].usage = PHYS_USABLE;
break;
case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
mem[i].usage = PHYS_RECLAIMABLE;
break;
case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
mem[i].usage = PHYS_STATIC;
break;
default:
mem[i].usage = PHYS_RESERVED;
break;
}
}
if (module_res) {
info.files = files;
info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
for (size_t i = 0; i < info.num_files; i++) {
files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset;
files[i].length = module_res->modules[i]->size;
strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path));
}
}
info.mem_map = mem;
info.phys_base = (phys_t)exec_res->physical_base;
info.virt_base = (uintptr_t)exec_res->virtual_base;
info.hhdm_base = (uintptr_t)hhdm_request.response->offset;
info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base;
kernel_main(&info);
}
phys_t rsdp_addr;
kernel/boot/limine/limine.h
+576
View File
@@ -0,0 +1,576 @@
/* BSD Zero Clause License */
/* Copyright (C) 2022-2025 Mintsuki and contributors.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef LIMINE_H
#define LIMINE_H 1
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
/* Misc */
#ifdef LIMINE_NO_POINTERS
#define LIMINE_PTR(TYPE) uint64_t
#else
#define LIMINE_PTR(TYPE) TYPE
#endif
#define LIMINE_REQUESTS_START_MARKER {0xf6b8f4b39de7d1ae, 0xfab91a6940fcb9cf, 0x785c6ed015d3e316, 0x181e920a7852b9d9}
#define LIMINE_REQUESTS_END_MARKER {0xadc0e0531bb10d03, 0x9572709f31764c62}
#define LIMINE_BASE_REVISION(N) {0xf9562b2d5c95a6c8, 0x6a7b384944536bdc, (N)}
#define LIMINE_BASE_REVISION_SUPPORTED(VAR) ((VAR)[2] == 0)
#define LIMINE_LOADED_BASE_REVISION_VALID(VAR) ((VAR)[1] != 0x6a7b384944536bdc)
#define LIMINE_LOADED_BASE_REVISION(VAR) ((VAR)[1])
#define LIMINE_COMMON_MAGIC 0xc7b1dd30df4c8b88, 0x0a82e883a194f07b
struct limine_uuid {
uint32_t a;
uint16_t b;
uint16_t c;
uint8_t d[8];
};
#define LIMINE_MEDIA_TYPE_GENERIC 0
#define LIMINE_MEDIA_TYPE_OPTICAL 1
#define LIMINE_MEDIA_TYPE_TFTP 2
struct limine_file {
uint64_t revision;
LIMINE_PTR(void*) address;
uint64_t size;
LIMINE_PTR(char*) path;
LIMINE_PTR(char*) string;
uint32_t media_type;
uint32_t unused;
uint32_t tftp_ip;
uint32_t tftp_port;
uint32_t partition_index;
uint32_t mbr_disk_id;
struct limine_uuid gpt_disk_uuid;
struct limine_uuid gpt_part_uuid;
struct limine_uuid part_uuid;
};
/* Boot info */
#define LIMINE_BOOTLOADER_INFO_REQUEST_ID {LIMINE_COMMON_MAGIC, 0xf55038d8e2a1202f, 0x279426fcf5f59740}
struct limine_bootloader_info_response {
uint64_t revision;
LIMINE_PTR(char*) name;
LIMINE_PTR(char*) version;
};
struct limine_bootloader_info_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_bootloader_info_response*) response;
};
/* Executable command line */
#define LIMINE_EXECUTABLE_CMDLINE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x4b161536e598651e, 0xb390ad4a2f1f303a}
struct limine_executable_cmdline_response {
uint64_t revision;
LIMINE_PTR(char*) cmdline;
};
struct limine_executable_cmdline_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_executable_cmdline_response*) response;
};
/* Firmware type */
#define LIMINE_FIRMWARE_TYPE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x8c2f75d90bef28a8, 0x7045a4688eac00c3}
#define LIMINE_FIRMWARE_TYPE_X86BIOS 0
#define LIMINE_FIRMWARE_TYPE_EFI32 1
#define LIMINE_FIRMWARE_TYPE_EFI64 2
#define LIMINE_FIRMWARE_TYPE_SBI 3
struct limine_firmware_type_response {
uint64_t revision;
uint64_t firmware_type;
};
struct limine_firmware_type_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_firmware_type_response*) response;
};
/* Stack size */
#define LIMINE_STACK_SIZE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x224ef0460a8e8926, 0xe1cb0fc25f46ea3d}
struct limine_stack_size_response {
uint64_t revision;
};
struct limine_stack_size_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_stack_size_response*) response;
uint64_t stack_size;
};
/* HHDM */
#define LIMINE_HHDM_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x48dcf1cb8ad2b852, 0x63984e959a98244b}
struct limine_hhdm_response {
uint64_t revision;
uint64_t offset;
};
struct limine_hhdm_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_hhdm_response*) response;
};
/* Framebuffer */
#define LIMINE_FRAMEBUFFER_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x9d5827dcd881dd75, 0xa3148604f6fab11b}
#define LIMINE_FRAMEBUFFER_RGB 1
struct limine_video_mode {
uint64_t pitch;
uint64_t width;
uint64_t height;
uint16_t bpp;
uint8_t memory_model;
uint8_t red_mask_size;
uint8_t red_mask_shift;
uint8_t green_mask_size;
uint8_t green_mask_shift;
uint8_t blue_mask_size;
uint8_t blue_mask_shift;
};
struct limine_framebuffer {
LIMINE_PTR(void*) address;
uint64_t width;
uint64_t height;
uint64_t pitch;
uint16_t bpp;
uint8_t memory_model;
uint8_t red_mask_size;
uint8_t red_mask_shift;
uint8_t green_mask_size;
uint8_t green_mask_shift;
uint8_t blue_mask_size;
uint8_t blue_mask_shift;
uint8_t unused[7];
uint64_t edid_size;
LIMINE_PTR(void*) edid;
/* Response revision 1 */
uint64_t mode_count;
LIMINE_PTR(struct limine_video_mode**) modes;
};
struct limine_framebuffer_response {
uint64_t revision;
uint64_t framebuffer_count;
LIMINE_PTR(struct limine_framebuffer**) framebuffers;
};
struct limine_framebuffer_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_framebuffer_response*) response;
};
/* Paging mode */
#define LIMINE_PAGING_MODE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x95c1a0edab0944cb, 0xa4e5cb3842f7488a}
#if defined(__x86_64__) || defined(__i386__)
#define LIMINE_PAGING_MODE_X86_64_4LVL 0
#define LIMINE_PAGING_MODE_X86_64_5LVL 1
#define LIMINE_PAGING_MODE_MIN LIMINE_PAGING_MODE_X86_64_4LVL
#define LIMINE_PAGING_MODE_DEFAULT LIMINE_PAGING_MODE_X86_64_4LVL
#elif defined(__aarch64__)
#define LIMINE_PAGING_MODE_AARCH64_4LVL 0
#define LIMINE_PAGING_MODE_AARCH64_5LVL 1
#define LIMINE_PAGING_MODE_MIN LIMINE_PAGING_MODE_AARCH64_4LVL
#define LIMINE_PAGING_MODE_DEFAULT LIMINE_PAGING_MODE_AARCH64_4LVL
#elif defined(__riscv) && (__riscv_xlen == 64)
#define LIMINE_PAGING_MODE_RISCV_SV39 0
#define LIMINE_PAGING_MODE_RISCV_SV48 1
#define LIMINE_PAGING_MODE_RISCV_SV57 2
#define LIMINE_PAGING_MODE_MIN LIMINE_PAGING_MODE_RISCV_SV39
#define LIMINE_PAGING_MODE_DEFAULT LIMINE_PAGING_MODE_RISCV_SV48
#elif defined(__loongarch__) && (__loongarch_grlen == 64)
#define LIMINE_PAGING_MODE_LOONGARCH64_4LVL 0
#define LIMINE_PAGING_MODE_MIN LIMINE_PAGING_MODE_LOONGARCH64_4LVL
#define LIMINE_PAGING_MODE_DEFAULT LIMINE_PAGING_MODE_LOONGARCH64_4LVL
#else
#error Unknown architecture
#endif
struct limine_paging_mode_response {
uint64_t revision;
uint64_t mode;
};
struct limine_paging_mode_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_paging_mode_response*) response;
uint64_t mode;
uint64_t max_mode;
uint64_t min_mode;
};
/* MP */
#define LIMINE_MP_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x95a67b819a1b857e, 0xa0b61b723b6a73e0}
struct limine_mp_info;
typedef void (*limine_goto_address)(struct limine_mp_info*);
#if defined(__x86_64__) || defined(__i386__)
#define LIMINE_MP_X2APIC (1 << 0)
struct limine_mp_info {
uint32_t processor_id;
uint32_t lapic_id;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint32_t flags;
uint32_t bsp_lapic_id;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info**) cpus;
};
#elif defined(__aarch64__)
struct limine_mp_info {
uint32_t processor_id;
uint32_t reserved1;
uint64_t mpidr;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint64_t flags;
uint64_t bsp_mpidr;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info**) cpus;
};
#elif defined(__riscv) && (__riscv_xlen == 64)
struct limine_mp_info {
uint64_t processor_id;
uint64_t hartid;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint64_t flags;
uint64_t bsp_hartid;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info**) cpus;
};
#elif defined(__loongarch__) && (__loongarch_grlen == 64)
struct limine_mp_info {
uint64_t reserved;
};
struct limine_mp_response {
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info**) cpus;
};
#else
#error Unknown architecture
#endif
struct limine_mp_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_mp_response*) response;
uint64_t flags;
};
/* Memory map */
#define LIMINE_MEMMAP_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x67cf3d9d378a806f, 0xe304acdfc50c3c62}
#define LIMINE_MEMMAP_USABLE 0
#define LIMINE_MEMMAP_RESERVED 1
#define LIMINE_MEMMAP_ACPI_RECLAIMABLE 2
#define LIMINE_MEMMAP_ACPI_NVS 3
#define LIMINE_MEMMAP_BAD_MEMORY 4
#define LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE 5
#define LIMINE_MEMMAP_EXECUTABLE_AND_MODULES 6
#define LIMINE_MEMMAP_FRAMEBUFFER 7
#define LIMINE_MEMMAP_ACPI_TABLES 8
struct limine_memmap_entry {
uint64_t base;
uint64_t length;
uint64_t type;
};
struct limine_memmap_response {
uint64_t revision;
uint64_t entry_count;
LIMINE_PTR(struct limine_memmap_entry**) entries;
};
struct limine_memmap_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_memmap_response*) response;
};
/* Entry point */
#define LIMINE_ENTRY_POINT_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x13d86c035a1cd3e1, 0x2b0caa89d8f3026a}
typedef void (*limine_entry_point)(void);
struct limine_entry_point_response {
uint64_t revision;
};
struct limine_entry_point_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_entry_point_response*) response;
LIMINE_PTR(limine_entry_point) entry;
};
/* Executable File */
#define LIMINE_EXECUTABLE_FILE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0xad97e90e83f1ed67, 0x31eb5d1c5ff23b69}
struct limine_executable_file_response {
uint64_t revision;
LIMINE_PTR(struct limine_file*) executable_file;
};
struct limine_executable_file_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_executable_file_response*) response;
};
/* Module */
#define LIMINE_MODULE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x3e7e279702be32af, 0xca1c4f3bd1280cee}
#define LIMINE_INTERNAL_MODULE_REQUIRED (1 << 0)
#define LIMINE_INTERNAL_MODULE_COMPRESSED (1 << 1)
struct limine_internal_module {
LIMINE_PTR(const char*) path;
LIMINE_PTR(const char*) string;
uint64_t flags;
};
struct limine_module_response {
uint64_t revision;
uint64_t module_count;
LIMINE_PTR(struct limine_file**) modules;
};
struct limine_module_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_module_response*) response;
/* Request revision 1 */
uint64_t internal_module_count;
LIMINE_PTR(struct limine_internal_module**) internal_modules;
};
/* RSDP */
#define LIMINE_RSDP_REQUEST_ID {LIMINE_COMMON_MAGIC, 0xc5e77b6b397e7b43, 0x27637845accdcf3c}
struct limine_rsdp_response {
uint64_t revision;
LIMINE_PTR(void*) address;
};
struct limine_rsdp_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_rsdp_response*) response;
};
/* SMBIOS */
#define LIMINE_SMBIOS_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x9e9046f11e095391, 0xaa4a520fefbde5ee}
struct limine_smbios_response {
uint64_t revision;
uint64_t entry_32;
uint64_t entry_64;
};
struct limine_smbios_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_smbios_response*) response;
};
/* EFI system table */
#define LIMINE_EFI_SYSTEM_TABLE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x5ceba5163eaaf6d6, 0x0a6981610cf65fcc}
struct limine_efi_system_table_response {
uint64_t revision;
uint64_t address;
};
struct limine_efi_system_table_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_efi_system_table_response*) response;
};
/* EFI memory map */
#define LIMINE_EFI_MEMMAP_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x7df62a431d6872d5, 0xa4fcdfb3e57306c8}
struct limine_efi_memmap_response {
uint64_t revision;
LIMINE_PTR(void*) memmap;
uint64_t memmap_size;
uint64_t desc_size;
uint64_t desc_version;
};
struct limine_efi_memmap_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_efi_memmap_response*) response;
};
/* Date at boot */
#define LIMINE_DATE_AT_BOOT_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x502746e184c088aa, 0xfbc5ec83e6327893}
struct limine_date_at_boot_response {
uint64_t revision;
int64_t timestamp;
};
struct limine_date_at_boot_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_date_at_boot_response*) response;
};
/* Executable address */
#define LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x71ba76863cc55f63, 0xb2644a48c516a487}
struct limine_executable_address_response {
uint64_t revision;
uint64_t physical_base;
uint64_t virtual_base;
};
struct limine_executable_address_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_executable_address_response*) response;
};
/* Device Tree Blob */
#define LIMINE_DTB_REQUEST_ID {LIMINE_COMMON_MAGIC, 0xb40ddb48fb54bac7, 0x545081493f81ffb7}
struct limine_dtb_response {
uint64_t revision;
LIMINE_PTR(void*) dtb_ptr;
};
struct limine_dtb_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_dtb_response*) response;
};
/* RISC-V Boot Hart ID */
#define LIMINE_RISCV_BSP_HARTID_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x1369359f025525f9, 0x2ff2a56178391bb6}
struct limine_riscv_bsp_hartid_response {
uint64_t revision;
uint64_t bsp_hartid;
};
struct limine_riscv_bsp_hartid_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_riscv_bsp_hartid_response*) response;
};
/* Bootloader Performance */
#define LIMINE_BOOTLOADER_PERFORMANCE_REQUEST_ID {LIMINE_COMMON_MAGIC, 0x6b50ad9bf36d13ad, 0xdc4c7e88fc759e17}
struct limine_bootloader_performance_response {
uint64_t revision;
uint64_t reset_usec;
uint64_t init_usec;
uint64_t exec_usec;
};
struct limine_bootloader_performance_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_bootloader_performance_response*) response;
};
#ifdef __cplusplus
}
#endif
#endif
kernel/boot/limine/meson.build
+1
View File
@@ -0,0 +1 @@
kernel_sources += files('entry.c')
kernel/boot/main.c
+151
View File
@@ -0,0 +1,151 @@
#include <zinnia/ipc/init.h>
#include <common/compiler.h>
#include <kernel/alloc.h>
#include <kernel/assert.h>
#include <kernel/channel.h>
#include <kernel/clock.h>
#include <kernel/cmdline.h>
#include <kernel/console.h>
#include <kernel/elf.h>
#include <kernel/futex.h>
#include <kernel/init.h>
#include <kernel/irq.h>
#include <kernel/namespace.h>
#include <kernel/percpu.h>
#include <kernel/print.h>
#include <kernel/sched.h>
#include <kernel/vas.h>
#include <kernel/vmo.h>
#include <config.h>
#include <stdint.h>
static zn_status_t load_server(
struct boot_file* file,
const char** argv,
const char** envp,
struct namespace_desc* init_handle,
struct task** out
) {
struct vas* space;
zn_status_t status = vma_new(&space);
if (status != ZN_OK)
return status;
struct paged_vmo* file_vmo;
status = vmo_new_phys(&file_vmo);
if (status != ZN_OK)
return status;
vmo_write(&file_vmo->object, 0, HHDM_PTR(file->data), file->length, nullptr);
struct namespace* ns;
status = namespace_new(&ns);
if (status != ZN_OK)
return status;
struct exec_info info = {
.file_obj = &file_vmo->object,
.ns = ns,
.space = space,
.argv = argv,
.envp = envp,
.init_handle = init_handle,
};
kprintf("Loading executable \"%s\"\n", file->path);
return elf_load(&info, out);
}
static const char zinnia_banner[] =
"Zinnia " ZINNIA_VERSION " (" ZINNIA_ARCH ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
static void kernel_main_task(uintptr_t arg0) {
struct boot_info* info = (struct boot_info*)arg0;
kprintf("%s\n", zinnia_banner); // Say hello!
ASSERT(info->num_files >= 1, "No init executable provided\n");
struct channel* init_ep;
struct channel* other_ep;
ASSERT(channel_create(&init_ep, &other_ep) == ZN_OK, "Failed to create init channel\n");
struct vas* init_space;
ASSERT(vma_new(&init_space) == ZN_OK, "");
struct boot_file* file_data = &info->files[0];
struct paged_vmo* init_file;
ASSERT(vmo_new_phys(&init_file) == ZN_OK, "");
vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
struct namespace* ns;
ASSERT(namespace_new(&ns) == ZN_OK, "");
struct namespace_desc init_ch_desc = {
.type = NAMESPACE_DESC_CHANNEL,
.rights = ZN_RIGHTS_ALL,
.value.channel = init_ep,
};
// Load init.
const char* init_argv[] = {"init", nullptr};
const char* init_envp[] = {nullptr};
struct task* init_task;
zn_status_t status = load_server(&info->files[0], init_argv, init_envp, &init_ch_desc, &init_task);
ASSERT(status == ZN_OK, "Failed to load init process (%i)\n", status);
sched_add_task(&percpu_get()->sched, init_task);
// Load other daemons.
for (size_t i = 1; i < info->num_files; i++) {
struct namespace_desc ch_desc = {
.type = NAMESPACE_DESC_CHANNEL,
.rights = ZN_RIGHTS_ALL,
.value.channel = other_ep,
};
const char* argv[] = {info->files[i].path, nullptr};
const char* envp[] = {nullptr};
struct task* task;
status = load_server(&info->files[i], argv, envp, &ch_desc, &task);
ASSERT(status == ZN_OK, "Failed to load server \"%s\": (%i)\n", info->files[i].path, status);
sched_add_task(&percpu_get()->sched, task);
}
struct ipc_kernel_msg* msg = mem_alloc(0x40, 0);
msg->id = IPC_MSG_BAR;
memcpy(msg->data, "This message is from the kernel!", 33);
channel_write(other_ep, msg, 0x40, nullptr, 0);
while (true) {
arch_irq_wait();
}
}
[[noreturn]]
void kernel_main(struct boot_info* info) {
cmdline_parse(info->cmdline);
console_init();
mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
rsdp_addr = info->rsdp; // TODO
percpu_init();
sched_init(&percpu_get()->sched);
futex_init();
struct task* main_task;
task_create("main", &kernel_vas, nullptr, kernel_main_task, (uintptr_t)info, &main_task);
sched_add_task(&percpu_get()->sched, main_task);
irq_unlock();
sched_yield(&percpu_get()->sched);
__unreachable();
}
[[__init]]
void kernel_early_init() {
percpu_bsp_init();
percpu_get()->online = true;
irq_lock();
}
kernel/boot/meson.build
+6
View File
@@ -0,0 +1,6 @@
kernel_sources += files(
'cmdline.c',
'main.c',
)
subdir('limine')
kernel/core/channel.c
+112
View File
@@ -0,0 +1,112 @@
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <kernel/channel.h>
#include <kernel/namespace.h>
#include <kernel/percpu.h>
#include <kernel/sched.h>
#include <kernel/tailq.h>
#include <kernel/wait.h>
#include <stdatomic.h>
zn_status_t channel_create(struct channel** ep0, struct channel** ep1) {
struct channel* a = mem_alloc(sizeof(struct channel), 0);
if (!a)
return ZN_ERR_NO_MEMORY;
struct channel* b = mem_alloc(sizeof(struct channel), 0);
if (!b) {
mem_free(a);
return ZN_ERR_NO_MEMORY;
}
a->peer = b;
a->lock = (struct spinlock){0};
TAILQ_INIT(&a->messages);
wait_queue_init(&a->waiters);
b->peer = a;
b->lock = (struct spinlock){0};
TAILQ_INIT(&b->messages);
wait_queue_init(&b->waiters);
*ep0 = a;
*ep1 = b;
return ZN_OK;
}
void channel_msg_free(struct channel_msg* msg) {
mem_free(msg->data);
mem_free(msg->descs);
mem_free(msg);
}
zn_status_t channel_write(
struct channel* ch,
void* data,
size_t num_bytes,
struct namespace_desc* descs,
size_t num_handles
) {
// Check that the peer is still alive.
struct channel* peer = atomic_load_explicit(&ch->peer, memory_order_acquire);
if (!peer)
return ZN_ERR_BAD_HANDLE;
// Allocate the message.
struct channel_msg* msg = mem_alloc(sizeof(struct channel_msg), 0);
if (!msg)
return ZN_ERR_NO_MEMORY;
msg->data = data;
msg->num_bytes = num_bytes;
msg->descs = descs;
msg->num_handles = num_handles;
// Enqueue on the peer's message queue and wake a waiter.
spin_lock(&peer->lock);
TAILQ_INSERT_TAIL(&peer->messages, msg, next);
spin_unlock(&peer->lock);
struct sched_percpu* sched = &percpu_get()->sched;
sched_wake(sched, &peer->waiters);
return ZN_OK;
}
zn_status_t channel_read(struct channel* ch, struct channel_msg** msg_out) {
spin_lock(&ch->lock);
struct channel_msg* msg = TAILQ_FIRST(&ch->messages);
if (msg != nullptr) {
TAILQ_REMOVE(&ch->messages, msg, next);
spin_unlock(&ch->lock);
*msg_out = msg;
return ZN_OK;
}
// No message and peer is gone — nothing will ever arrive.
if (!ch->peer) {
spin_unlock(&ch->lock);
return ZN_ERR_BAD_HANDLE;
}
spin_unlock(&ch->lock);
return ZN_ERR_INVALID;
}
zn_status_t channel_poll(struct channel* ch) {
spin_lock(&ch->lock);
if (!TAILQ_EMPTY(&ch->messages)) {
spin_unlock(&ch->lock);
return ZN_OK;
}
if (!ch->peer) {
spin_unlock(&ch->lock);
return ZN_ERR_BAD_HANDLE;
}
spin_unlock(&ch->lock);
return ZN_ERR_INVALID;
}
kernel/core/clock.c
+54
View File
@@ -0,0 +1,54 @@
#include <common/compiler.h>
#include <kernel/clock.h>
#include <kernel/print.h>
#include <kernel/spin.h>
#include <stdatomic.h>
#include <stdint.h>
static struct clock* active_clock = nullptr;
static uint64_t counter_base = 0;
static struct spinlock lock = {};
bool clock_available() {
return active_clock != nullptr;
}
uint64_t clock_get_elapsed_ns() {
struct clock* active = atomic_load(&active_clock);
if (__unlikely(!active))
return 0;
return atomic_load(&counter_base) + active->get_elapsed_ns(active);
}
bool clock_switch(struct clock* clock) {
spin_lock(&lock);
if (active_clock) {
if (clock->priority <= active_clock->priority) {
spin_unlock(&lock);
return false;
}
}
kprintf("Switching to clock source \"%s\"\n", clock->name);
uint64_t elapsed = clock_get_elapsed_ns();
counter_base += elapsed;
active_clock = clock;
spin_unlock(&lock);
return true;
}
void clock_spin_ns(uint64_t ns) {
if (active_clock == nullptr) {
kprintf("Unable to sleep for %lu ns, no clock available!\n", ns);
return;
}
uint64_t target = clock_get_elapsed_ns() + ns;
while (clock_get_elapsed_ns() < target) {
// TODO: Maybe pause?
}
}
kernel/core/console.c
+44
View File
@@ -0,0 +1,44 @@
#include <kernel/cmdline.h>
#include <kernel/console.h>
#include <kernel/print.h>
#include <kernel/syscalls.h>
#include <string.h>
extern struct console* const __ld_console_start[];
extern struct console* const __ld_console_end[];
struct console* active = nullptr;
void console_write(const char* buf, size_t len) {
if (__likely(active))
active->write(active, buf, len);
}
static void console_setup(const char* value) {
if (!value)
return;
// Find a suitable boot console.
struct console* const* cur = __ld_console_start;
while (cur < __ld_console_end) {
if (!strcmp(value, (*cur)->name)) {
active = *cur;
}
cur++;
}
}
CMDLINE_OPTION("console", console_setup);
void console_init() {
// If we have no console set and at least one set, use that as a default.
if (!active && &__ld_console_start[0] != &__ld_console_end[0])
active = __ld_console_start[0];
if (active) {
if (active->init(active) == false)
active = nullptr;
else
kprintf("Logging on console \"%s\"\n", active->name);
}
}
kernel/core/elf.c
+201
View File
@@ -0,0 +1,201 @@
#include <zinnia/mem.h>
#include <zinnia/status.h>
#include <common/elf.h>
#include <common/utils.h>
#include <kernel/alloc.h>
#include <kernel/assert.h>
#include <kernel/elf.h>
#include <kernel/percpu.h>
#include <kernel/pmap.h>
#include <kernel/print.h>
#include <kernel/sched.h>
#include <kernel/vas.h>
#include <kernel/vdso.h>
#include <kernel/vmo.h>
#include <string.h>
zn_status_t elf_load(struct exec_info* info, struct task** out) {
zn_status_t status;
size_t read;
struct elf_ehdr ehdr = {};
vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
// Check if the file is an ELF.
if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
return ZN_ERR_INVALID;
if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
return ZN_ERR_INVALID;
if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS)
return ZN_ERR_INVALID;
if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
return ZN_ERR_INVALID;
if (ehdr.e_machine != ELF_ARCH_MACHINE)
return ZN_ERR_INVALID;
if (ehdr.e_type != ET_EXEC)
return ZN_ERR_INVALID;
const size_t page_size = arch_mem_page_size();
for (size_t i = 0; i < ehdr.e_phnum; i++) {
struct elf_phdr phdr = {};
vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
if (phdr.p_type == PT_LOAD) {
enum zn_vm_flags prot = 0;
if (phdr.p_flags & PF_R)
prot |= ZN_VM_MAP_READ;
if (phdr.p_flags & PF_W)
prot |= ZN_VM_MAP_WRITE;
if (phdr.p_flags & PF_X)
prot |= ZN_VM_MAP_EXEC;
ASSERT(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
const uintptr_t map_address = phdr.p_vaddr - misalign;
const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
// Copy the file data into its own mapping.
struct paged_vmo* phdr_obj;
status = vmo_new_phys(&phdr_obj);
if (status)
return status;
status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
if (status)
return status;
// We map more than we copied so the rest is filled with zeroed pages.
status = vma_map_vmo(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
if (status)
return status;
}
}
const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
const size_t stack_size = 2 * 1024 * 1024; // 2MiB
const uintptr_t stack_start = highest - stack_size;
// Allocate stack.
struct paged_vmo* stack;
status = vmo_new_phys(&stack);
if (status)
return status;
// Fill in stack info.
uintptr_t stack_off = stack_size;
// TODO: Instead of allocating, calculate it on the fly.
size_t num_envp = 0;
for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
num_envp++;
}
size_t num_argv = 0;
for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
num_argv++;
}
uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
for (size_t env = 0; env < num_envp; env++) {
const char nul = 0;
stack_off -= 1;
vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
const size_t len = strlen(info->envp[env]);
stack_off -= len;
vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr);
envp_offsets[env] = stack_start + stack_off;
}
for (size_t arg = 0; arg < num_argv; arg++) {
const char nul = 0;
stack_off -= 1;
vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
const size_t len = strlen(info->argv[arg]);
stack_off -= len;
vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr);
argv_offsets[arg] = stack_start + stack_off;
}
stack_off = ALIGN_DOWN(stack_off, 16);
// Align the stack if argc + argv + envp does not add up to 16 byte alignment.
if ((1 + num_argv + num_envp) & 1) {
stack_off -= sizeof(uintptr_t);
uintptr_t zero = 0;
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
}
// Add the init handle to the process namespace.
zn_handle_t init_handle = ZN_HANDLE_INVALID;
if (info->init_handle) {
status = namespace_add_desc(info->ns, *info->init_handle, &init_handle);
if (status)
return status;
}
#define WRITE_AUXV(auxv, value) \
do { \
stack_off -= sizeof(uintptr_t); \
uintptr_t auxv_val = value; \
vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
stack_off -= sizeof(uintptr_t); \
auxv_val = auxv; \
vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
} while (0)
// Write auxiliary values.
WRITE_AUXV(AT_NULL, 0);
WRITE_AUXV(AT_SECURE, 0);
WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
WRITE_AUXV(AT_INIT_HANDLE, init_handle);
// envp pointers.
stack_off -= sizeof(uintptr_t);
const uintptr_t zero = 0;
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
for (size_t env = 0; env < num_envp; env++) {
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr);
}
// argv pointers.
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
for (size_t arg = 0; arg < num_argv; arg++) {
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr);
}
mem_free(envp_offsets);
mem_free(argv_offsets);
// argc
stack_off -= sizeof(uintptr_t);
uintptr_t argc = num_argv;
vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
status = vma_map_vmo(info->space, &stack->object, stack_start, stack_size, ZN_VM_MAP_READ | ZN_VM_MAP_WRITE, 0);
if (status)
return status;
struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
start_info->ip = ehdr.e_entry;
start_info->sp = highest - stack_size + stack_off;
start_info->arg = 0;
struct task* result;
status = task_create(info->argv[0], info->space, info->ns, sched_to_user, (uintptr_t)start_info, &result);
if (status)
return status;
*out = result;
return ZN_OK;
}
kernel/core/futex.c
+120
View File
@@ -0,0 +1,120 @@
#include <zinnia/status.h>
#include <kernel/futex.h>
#include <kernel/percpu.h>
#include <kernel/print.h>
#include <kernel/sched.h>
#include <kernel/spin.h>
#include <kernel/usercopy.h>
struct futex_bucket {
struct wait_queue wq;
uintptr_t key; // Virtual address used as the futex key.
bool in_use;
};
#define FUTEX_BUCKET_COUNT 64
static struct futex_bucket futex_table[FUTEX_BUCKET_COUNT];
static struct spinlock futex_table_lock;
void futex_init(void) {
futex_table_lock = (struct spinlock){0};
for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) {
wait_queue_init(&futex_table[i].wq);
futex_table[i].key = 0;
futex_table[i].in_use = false;
}
}
static size_t futex_hash(uintptr_t addr) {
// Simple hash: mix the address bits and mod by bucket count.
addr = (addr >> 2) ^ (addr >> 12);
return addr % FUTEX_BUCKET_COUNT;
}
static struct futex_bucket* futex_bucket_get(uintptr_t addr) {
size_t idx = futex_hash(addr);
// Linear probing to find a matching or free bucket.
for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) {
size_t slot = (idx + i) % FUTEX_BUCKET_COUNT;
struct futex_bucket* b = &futex_table[slot];
if (b->in_use && b->key == addr)
return b;
if (!b->in_use) {
b->key = addr;
b->in_use = true;
return b;
}
}
return nullptr;
}
zn_status_t futex_wait(__user int* addr, int expected) {
uintptr_t key = (uintptr_t)addr;
spin_lock(&futex_table_lock);
// Read the current value at the userspace address.
int current;
if (!usercopy_read(&current, addr, sizeof(*addr))) {
spin_unlock(&futex_table_lock);
return ZN_ERR_BAD_BUFFER;
}
// If the value has already changed, don't block.
if (current != expected) {
spin_unlock(&futex_table_lock);
return ZN_OK;
}
struct futex_bucket* bucket = futex_bucket_get(key);
if (!bucket) {
spin_unlock(&futex_table_lock);
return ZN_ERR_NO_MEMORY;
}
spin_unlock(&futex_table_lock);
// Block on the bucket's wait queue.
struct sched_percpu* sched = &percpu_get()->sched;
sched_block(sched, &bucket->wq);
return ZN_OK;
}
zn_status_t futex_wake(__user int* addr, int count) {
uintptr_t key = (uintptr_t)addr;
spin_lock(&futex_table_lock);
size_t idx = futex_hash(key);
struct futex_bucket* bucket = nullptr;
for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) {
size_t slot = (idx + i) % FUTEX_BUCKET_COUNT;
struct futex_bucket* b = &futex_table[slot];
if (b->in_use && b->key == key) {
bucket = b;
break;
}
if (!b->in_use)
break;
}
if (!bucket) {
spin_unlock(&futex_table_lock);
return ZN_OK; // No waiters, nothing to do.
}
spin_unlock(&futex_table_lock);
struct sched_percpu* sched = &percpu_get()->sched;
if (count == INT32_MAX) {
sched_wake_all(sched, &bucket->wq);
} else {
for (int i = 0; i < count; i++) {
sched_wake(sched, &bucket->wq);
}
}
return ZN_OK;
}
kernel/core/irq.c
+28
View File
@@ -0,0 +1,28 @@
#include <kernel/irq.h>
#include <kernel/percpu.h>
#include <stdatomic.h>
void irq_lock() {
struct irq_percpu* cpu = &percpu_get()->irq;
if (atomic_load_explicit(&cpu->in_interrupt, memory_order_acquire))
return;
arch_irq_set_state(false);
atomic_fetch_add_explicit(&percpu_get()->irq.level, 1, memory_order_acq_rel);
}
void irq_unlock() {
struct irq_percpu* cpu = &percpu_get()->irq;
if (atomic_load_explicit(&cpu->in_interrupt, memory_order_acquire))
return;
uint32_t old_level = atomic_fetch_sub_explicit(&cpu->level, 1, memory_order_acq_rel);
// If it was 1, the new IRQ level is now 0.
if (old_level == 1) {
arch_irq_set_state(true);
}
}
bool irq_set_interrupted(bool is_interrupted) {
return atomic_exchange_explicit(&percpu_get()->irq.in_interrupt, is_interrupted, memory_order_release);
}
kernel/core/meson.build
+15
View File
@@ -0,0 +1,15 @@
kernel_sources += files(
'channel.c',
'clock.c',
'futex.c',
'console.c',
'elf.c',
'irq.c',
'namespace.c',
'panic.c',
'percpu.c',
'print.c',
'sched.c',
'spin.c',
'uacpi_glue.c',
)
kernel/core/namespace.c
+115
View File
@@ -0,0 +1,115 @@
#include <zinnia/handle.h>
#include <zinnia/rights.h>
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <kernel/namespace.h>
#include <stdatomic.h>
zn_status_t namespace_new(struct namespace** out) {
if (!out)
return ZN_ERR_INTERNAL;
struct namespace* namespace = mem_alloc(sizeof(struct namespace), 0);
if (!namespace)
return ZN_ERR_NO_MEMORY;
namespace->mutex = (struct mutex){0};
namespace->next_handle = 1;
HASHMAP_INIT(&namespace->descriptors);
*out = namespace;
return ZN_OK;
}
zn_status_t namespace_add_desc(struct namespace* namespace, struct namespace_desc desc, zn_handle_t* out) {
if (!namespace)
return ZN_ERR_INTERNAL;
// Skip over handles that are already in use (e.g. hard-coded handles).
zn_handle_t handle;
do {
handle = atomic_fetch_add_explicit(&namespace->next_handle, 1, memory_order_acq_rel);
if (handle <= ZN_HANDLE_INVALID)
continue;
} while (HASHMAP_GET(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default) != nullptr);
zn_status_t status =
HASHMAP_INSERT(&namespace->descriptors, handle, desc, hashmap_hash_default, hashmap_eq_default);
if (status != ZN_OK)
return status;
*out = handle;
return ZN_OK;
}
zn_status_t namespace_add_desc_at(struct namespace* namespace, struct namespace_desc desc, zn_handle_t handle) {
if (!namespace)
return ZN_ERR_INTERNAL;
if (handle <= ZN_HANDLE_INVALID)
return ZN_ERR_BAD_HANDLE;
// Fail if a descriptor already exists at this handle.
if (HASHMAP_GET(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default) != nullptr)
return ZN_ERR_ALREADY_EXISTS;
return HASHMAP_INSERT(&namespace->descriptors, handle, desc, hashmap_hash_default, hashmap_eq_default);
}
zn_status_t namespace_get(struct namespace* namespace, zn_handle_t handle, struct namespace_desc* out) {
if (!namespace)
return ZN_ERR_INTERNAL;
if (!out)
return ZN_ERR_INTERNAL;
if (handle <= ZN_HANDLE_INVALID)
return ZN_ERR_BAD_HANDLE;
struct namespace_desc* desc =
HASHMAP_GET(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default);
if (!desc)
return ZN_ERR_BAD_HANDLE;
*out = *desc;
return ZN_OK;
}
zn_status_t namespace_del_desc(struct namespace* namespace, zn_handle_t handle) {
if (!namespace)
return ZN_ERR_INTERNAL;
if (handle <= ZN_HANDLE_INVALID)
return ZN_ERR_BAD_HANDLE;
if (!HASHMAP_REMOVE(&namespace->descriptors, handle, hashmap_hash_default, hashmap_eq_default))
return ZN_ERR_BAD_HANDLE;
return ZN_OK;
}
zn_status_t namespace_clone_desc(
struct namespace* ns,
struct namespace_desc* desc,
zn_rights_t cloned_rights,
zn_handle_t* out
) {
// The source handle must have the clone right.
if (!(desc->rights & ZN_RIGHT_CLONE))
return ZN_ERR_BAD_PERMS;
// Determine the rights for the cloned handle.
zn_rights_t new_rights;
if (cloned_rights & ZN_RIGHTS_IDENTICAL) {
new_rights = desc->rights;
} else {
// The requested rights must be a subset of the original rights.
if (cloned_rights & ~desc->rights)
return ZN_ERR_BAD_PERMS;
new_rights = cloned_rights;
}
// Create the cloned descriptor.
struct namespace_desc new_desc = *desc;
new_desc.rights = new_rights;
return namespace_add_desc(ns, new_desc, out);
}
kernel/core/panic.c
+32
View File
@@ -0,0 +1,32 @@
#include <kernel/irq.h>
#include <kernel/panic.h>
#include <kernel/percpu.h>
#include <kernel/print.h>
#include <stdarg.h>
[[noreturn]]
void panic(const char* msg, ...) {
// TODO: Stop all other CPUs with IPI.
arch_irq_set_state(false);
struct percpu* cpu = percpu_get();
struct task* current = cpu->sched.current;
kprintf(
"----[ Kernel panic ]----\n"
"In task \"%s\" (TID %zu) on CPU %zu!\n",
current ? current->name : nullptr,
current ? current->id : 0,
cpu->id
);
va_list args;
va_start(args, msg);
kvprintf(msg, args);
va_end(args);
kprintf("----[ End of panic ]----\n");
while (1) {
arch_irq_wait();
}
}
kernel/core/percpu.c
+14
View File
@@ -0,0 +1,14 @@
#include <common/compiler.h>
#include <kernel/percpu.h>
[[__used, __section(".percpu")]]
struct percpu percpu_bsp = {
.self = &percpu_bsp,
.id = 0,
.online = false,
};
struct percpu* percpu_new() {
// TODO
return nullptr;
}
kernel/core/print.c
+399
View File
@@ -0,0 +1,399 @@
#include <kernel/clock.h>
#include <kernel/console.h>
#include <kernel/panic.h>
#include <kernel/print.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
static void reverse(char* s) {
int i, j;
char c;
for (i = 0, j = strlen(s) - 1; i < j; i++, j--) {
c = s[i];
s[i] = s[j];
s[j] = c;
}
}
static int char_to_digit(char c) {
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'a' && c <= 'z')
return c - 'a' + 10;
if (c >= 'A' && c <= 'Z')
return c - 'A' + 10;
return -1;
}
unsigned long atolu(const char* str, int base) {
unsigned long result = 0;
if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
str += 2;
}
while (*str) {
int digit = char_to_digit(*str++);
if (digit < 0 || digit >= base)
break;
result = result * base + digit;
}
return result;
}
long atol(const char* str, int base) {
int neg = 0;
if (*str == '-') {
neg = 1;
str++;
}
unsigned long val = atolu(str, base);
return neg ? -(long)val : (long)val;
}
char* lutoa(uint64_t value, char* str, int base) {
if (base < 2 || base > 36)
return nullptr;
size_t i = 0;
do {
uint64_t digit = value % base;
str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
value /= base;
} while (value);
str[i] = '\0';
reverse(str);
return str;
}
char* ltoa(int64_t value, char* str, int base) {
if (base < 2 || base > 36)
return nullptr;
uint64_t magnitude;
int negative = 0;
if (value < 0) {
negative = 1;
magnitude = (uint64_t)(-(value + 1)) + 1;
// avoids INT64_MIN overflow
} else {
magnitude = (uint64_t)value;
}
size_t i = 0;
do {
uint64_t digit = magnitude % base;
str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
magnitude /= base;
} while (magnitude);
if (negative)
str[i++] = '-';
str[i] = '\0';
reverse(str);
return str;
}
typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
// Amount of bytes written.
int32_t written = 0;
while (*fmt != '\0') {
size_t maxrem = INT32_MAX - written;
if (fmt[0] != '%' || fmt[1] == '%') {
if (fmt[0] == '%')
fmt++;
size_t amount = 1;
while (fmt[amount] && fmt[amount] != '%')
amount++;
if (maxrem < amount) {
return;
}
callback(ctx, fmt, amount);
fmt += amount;
written += amount;
continue;
}
const char* format_begun_at = fmt++;
bool force_sign = false;
bool blank_sign = false;
bool write_prefix = false;
bool pad_zero = false;
bool has_width = false;
size_t width = 0;
bool has_precision = false;
size_t precision = 0;
size_t size = sizeof(uint32_t);
char number[32];
check_fmt:
memset(number, 0, sizeof(number));
switch (*fmt) {
// Flags
case '+': {
force_sign = true;
fmt++;
goto check_fmt;
}
case ' ': {
blank_sign = true;
fmt++;
goto check_fmt;
}
case '#': {
write_prefix = true;
fmt++;
goto check_fmt;
}
case '0': {
pad_zero = true;
fmt++;
goto check_fmt;
}
// Width
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
has_width = true;
char number[10 + 1] = {0};
size_t idx = 0;
while (*fmt >= '0' && *fmt <= '9') {
number[idx++] = *fmt;
fmt++;
}
width = atolu(number, 10);
goto check_fmt;
}
// Precision
case '.': {
has_precision = true;
fmt++;
if (*fmt == '*') {
precision = (char)va_arg(args, int32_t);
} else {
char number[10 + 1] = {0};
size_t idx = 0;
while (*fmt >= '0' && *fmt <= '9') {
number[idx++] = *fmt;
fmt++;
}
precision = atolu(number, 10);
}
goto check_fmt;
}
// Length
case 'h': {
if (fmt[1] == 'h') {
size = sizeof(uint8_t);
fmt++;
} else
size = sizeof(uint16_t);
fmt++;
goto check_fmt;
}
case 'l': {
// There is no difference between `long int` and `long long int`.
if (fmt[1] == 'l')
fmt++;
fmt++;
size = sizeof(uint64_t);
goto check_fmt;
}
case 'z': {
size = sizeof(size_t);
fmt++;
goto check_fmt;
}
// Character
case 'c': {
char c = (char)va_arg(args, int32_t);
callback(ctx, &c, sizeof(c));
written++;
fmt++;
break;
}
// String of characters
case 's': {
const char* str = va_arg(args, const char*);
if (!str)
str = "(null)";
size_t len = 0;
if (has_precision)
len = strnlen(str, precision);
else
len = strlen(str);
callback(ctx, str, len);
written += len;
fmt++;
break;
}
// Signed decimal integer
case 'i':
case 'd': {
switch (size) {
case sizeof(int8_t):
ltoa((int8_t)va_arg(args, int32_t), number, 10);
break;
case sizeof(int16_t):
ltoa((int16_t)va_arg(args, int32_t), number, 10);
break;
case sizeof(int32_t):
ltoa((int32_t)va_arg(args, int32_t), number, 10);
break;
default:
ltoa(va_arg(args, int64_t), number, 10);
break;
}
goto print_num;
}
// Unsigned decimal integer
case 'u': {
switch (size) {
case sizeof(uint8_t):
lutoa((uint8_t)va_arg(args, uint32_t), number, 10);
break;
case sizeof(uint16_t):
lutoa((uint16_t)va_arg(args, uint32_t), number, 10);
break;
case sizeof(uint32_t):
lutoa(va_arg(args, uint32_t), number, 10);
break;
case sizeof(uint64_t):
lutoa(va_arg(args, uint64_t), number, 10);
break;
}
goto print_num;
}
// Unsigned hexadecimal integer
case 'X':
case 'x': {
switch (size) {
case sizeof(uint8_t):
lutoa((uint8_t)va_arg(args, uint32_t), number, 16);
break;
case sizeof(uint16_t):
lutoa((uint16_t)va_arg(args, uint32_t), number, 16);
break;
case sizeof(uint32_t):
lutoa(va_arg(args, uint32_t), number, 16);
break;
case sizeof(uint64_t):
lutoa(va_arg(args, uint64_t), number, 16);
break;
}
print_num:
size_t len = strlen(number);
if (has_precision) {
for (size_t i = 0; i < precision - len; i++)
callback(ctx, " ", 1);
}
if (has_width) {
char c = pad_zero ? '0' : ' ';
// We don't have to pad anything.
if (len < width) {
for (size_t i = 0; i < width - len; i++)
callback(ctx, &c, 1);
}
}
if (write_prefix) {
callback(ctx, "0x", 2);
}
if (force_sign) {
char c = '+';
if (number[0] != '-')
callback(ctx, &c, 1);
}
if (blank_sign) {
char c = ' ';
if (number[0] != '-')
callback(ctx, &c, 1);
}
callback(ctx, number, len);
written += len;
fmt++;
break;
}
// Pointer address
case 'p': {
const size_t num = va_arg(args, size_t);
const size_t buf_size = sizeof(size_t) * 2 + 1;
char str[buf_size];
lutoa(num, number, 0x10);
const size_t len = strlen(number); // Get the length of the final number.
for (size_t i = 0; i < buf_size; i++)
str[i] = '0';
size_t offset = sizeof(str) - len - 1;
callback(ctx, str, offset);
callback(ctx, number, len);
fmt++;
written += len;
break;
}
// The number of characters written so far is stored in the pointed location.
case 'n': {
int32_t* const ptr = va_arg(args, int32_t*);
*ptr = written;
fmt++;
break;
}
// No format, just normal text.
default: {
fmt = format_begun_at;
size_t len = strlen(fmt);
callback(ctx, fmt, len);
written += len;
fmt += len;
break;
}
}
}
}
static void printf_console(void*, const char* msg, size_t len) {
console_write(msg, len);
}
void kprintf(const char* message, ...) {
va_list args;
va_start(args, message);
do_print(printf_console, nullptr, message, args);
va_end(args);
}
void kvprintf(const char* message, va_list args) {
do_print(printf_console, nullptr, message, args);
}
kernel/core/sched.c
+185
View File
@@ -0,0 +1,185 @@
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <kernel/assert.h>
#include <kernel/irq.h>
#include <kernel/list.h>
#include <kernel/percpu.h>
#include <kernel/print.h>
#include <kernel/sched.h>
#include <kernel/spin.h>
#include <kernel/tailq.h>
#include <bits/irq.h>
#include <stdatomic.h>
#include <stdint.h>
#include <string.h>
static void idle_func(uintptr_t) {
while (1) {
arch_irq_set_state(true);
arch_irq_wait();
}
}
void sched_to_user(uintptr_t info_ptr) {
struct task_start_info* info = (struct task_start_info*)info_ptr;
uintptr_t ip = info->ip;
uintptr_t sp = info->sp;
uintptr_t arg = info->arg;
mem_free(info);
arch_sched_jump_to_user(ip, sp, arg);
}
void sched_to_user_context(uintptr_t ctx) {
arch_sched_jump_to_context((struct arch_context*)ctx);
}
void sched_init(struct sched_percpu* sched) {
struct task* idle_task;
ASSERT(
task_create("idle", &kernel_vas, nullptr, idle_func, 0, &idle_task) == ZN_OK,
"Unable to create idle task!\n"
);
sched->idle_task = idle_task;
sched->current = idle_task;
TAILQ_INIT(&sched->run_queue);
kprintf("Scheduler initialized\n");
}
void sched_add_task(struct sched_percpu* sched, struct task* task) {
ASSERT(task != nullptr, "Tried to add NULL to the run queue\n");
TAILQ_INSERT_TAIL(&sched->run_queue, task, next);
}
static struct task* next_task(struct sched_percpu* sched) {
struct task* t = TAILQ_FIRST(&sched->run_queue);
if (t == nullptr)
return sched->idle_task;
TAILQ_REMOVE(&sched->run_queue, t, next);
return t;
}
static void do_reschedule(struct sched_percpu* sched) {
struct task* from = atomic_load_explicit(&sched->current, memory_order_acquire);
struct task* to = next_task(sched);
if (from == to) {
irq_unlock();
return;
}
atomic_store_explicit(&sched->current, to, memory_order_relaxed);
pmap_set(&to->space->pmap);
struct percpu* cpu = percpu_get();
from->kernel_stack = cpu->kernel_stack;
from->user_stack = cpu->user_stack;
cpu->kernel_stack = to->kernel_stack;
cpu->user_stack = to->user_stack;
arch_sched_switch(from, to);
}
void sched_reschedule(struct sched_percpu* sched) {
irq_lock();
if (sched->current->state != TASK_STATE_DEAD)
sched_add_task(sched, sched->current);
do_reschedule(sched);
}
void sched_yield(struct sched_percpu* sched) {
irq_lock();
do_reschedule(sched);
}
zn_status_t task_create(
const char* name,
struct vas* space,
struct namespace* ns,
task_fn_t entry,
uintptr_t arg0,
struct task** out
) {
if (!out)
return ZN_ERR_INVALID;
struct task* new_task = mem_alloc(sizeof(struct task), 0);
if (new_task == nullptr)
return ZN_ERR_NO_MEMORY;
static size_t last_tid = 0;
new_task->id = atomic_fetch_add(&last_tid, 1);
new_task->space = space;
new_task->namespace = ns;
strncpy(new_task->name, name, sizeof(new_task->name));
// Allocate a kernel stack.
new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0);
if (!new_task->kernel_stack)
return ZN_ERR_NO_MEMORY;
new_task->user_stack = 0;
// Initialize the arch-dependent parts.
zn_status_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
if (s != ZN_OK)
return s;
*out = new_task;
return ZN_OK;
}
void task_entry(task_fn_t entry, uintptr_t arg0) {
entry(arg0);
struct sched_percpu* sched = &percpu_get()->sched;
sched->current->state = TASK_STATE_DEAD;
sched_yield(sched);
panic("unreachable");
}
void wait_queue_init(struct wait_queue* wq) {
TAILQ_INIT(&wq->waiters);
wq->lock = (struct spinlock){0};
}
void sched_block(struct sched_percpu* sched, struct wait_queue* wq) {
irq_lock();
spin_lock(&wq->lock);
sched->current->state = TASK_STATE_BLOCKED;
TAILQ_INSERT_TAIL(&wq->waiters, sched->current, next);
spin_unlock(&wq->lock);
do_reschedule(sched);
}
void sched_wake(struct sched_percpu* sched, struct wait_queue* wq) {
spin_lock(&wq->lock);
struct task* t = TAILQ_FIRST(&wq->waiters);
if (t != nullptr) {
TAILQ_REMOVE(&wq->waiters, t, next);
t->state = TASK_STATE_READY;
sched_add_task(sched, t);
}
spin_unlock(&wq->lock);
}
void sched_wake_all(struct sched_percpu* sched, struct wait_queue* wq) {
spin_lock(&wq->lock);
struct task* t;
while ((t = TAILQ_FIRST(&wq->waiters)) != nullptr) {
TAILQ_REMOVE(&wq->waiters, t, next);
t->state = TASK_STATE_READY;
sched_add_task(sched, t);
}
spin_unlock(&wq->lock);
}
kernel/core/spin.c
+16
View File
@@ -0,0 +1,16 @@
#include <kernel/spin.h>
#include <stdatomic.h>
// TODO: Ticketing system
void spin_lock(struct spinlock* spin) {
while (atomic_exchange_explicit(&spin->locked, true, memory_order_acquire)) {
asm volatile("pause");
}
atomic_store_explicit(&spin->locked, true, memory_order_release);
}
void spin_unlock(struct spinlock* spin) {
atomic_store_explicit(&spin->locked, false, memory_order_release);
}
kernel/core/uacpi_glue.c
+34
View File
@@ -0,0 +1,34 @@
#include <common/utils.h>
#include <kernel/init.h>
#include <kernel/mmio.h>
#include <kernel/print.h>
#include <stdarg.h>
#include <uacpi/kernel_api.h>
#include <uacpi/status.h>
#include <uacpi/types.h>
uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) {
*out_rsdp_address = (uacpi_phys_addr)rsdp_addr;
return UACPI_STATUS_OK;
}
void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
return mmio_new(addr, len);
}
void uacpi_kernel_unmap(void* addr, uacpi_size len) {
mmio_free(addr, len);
}
void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
va_list args;
va_start(args, msg);
kvprintf(msg, args);
va_end(args);
}
void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
kvprintf(msg, args);
}
kernel/include/kernel/alloc.h
+57
View File
@@ -0,0 +1,57 @@
#pragma once
#include <zinnia/status.h>
#include <bits/mem.h>
#include <stddef.h>
enum alloc_flags {
ALLOC_NOZERO = 1 << 0, // Don't zero out the allocated memory.
ALLOC_MEM32 = 1 << 1, // Allocated memory needs to fit inside 32 bits.
ALLOC_MEM20 = 1 << 2, // Allocated memory needs to fit inside 20 bits.
};
enum phys_mem_usage {
PHYS_RESERVED,
PHYS_USABLE,
PHYS_STATIC,
PHYS_RECLAIMABLE,
};
struct phys_mem {
phys_t address;
size_t length;
enum phys_mem_usage usage;
};
// Base address of the HHDM.
extern uintptr_t mem_hhdm_base;
#define HHDM_PTR(paddr) (void*)(mem_hhdm_base + (paddr))
// Allocates a region of memory which can be smaller than the page size.
// Returns `nullptr` if the allocator cannot provide an allocation for the given `length` + `flags`.
// Always returns `nullptr` if `length` is 0.
void* mem_alloc(size_t length, enum alloc_flags flags);
// Frees an allocation created by `mem_alloc`.
// Passing `nullptr` is a no-op.
void mem_free(void* mem);
zn_status_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out);
zn_status_t mem_phys_free(phys_t start, size_t num_pages);
void mem_init(struct phys_mem* map, size_t length, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm);
void slab_init();
void mem_phys_bootstrap(struct phys_mem* mem);
void mem_phys_init(struct phys_mem* map, size_t length);
uintptr_t arch_mem_hhdm_addr();
uintptr_t arch_vm_pfndb_addr();
uintptr_t arch_mem_mapping_addr();
size_t arch_mem_page_bits();
size_t arch_mem_page_size();
size_t arch_mem_level_bits();
size_t arch_mem_num_levels();
static inline size_t mem_high_shift() {
return arch_mem_level_bits() * arch_mem_num_levels() + arch_mem_page_bits();
}
kernel/include/kernel/assert.h
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <common/compiler.h>
#include <kernel/panic.h>
#define ASSERT(expr, msg, ...) \
({ \
if (__unlikely(!(expr))) { \
panic( \
"In function \"%s\" (%s:%u):\n" \
"Assertion \"%s\" failed!\n" msg, \
__FUNCTION__, \
__FILE__, \
__LINE__, \
#expr, \
##__VA_ARGS__ \
); \
} \
})
#ifndef NDEBUG
#define DEBUG_ASSERT(...) ASSERT(__VA_ARGS__)
#else
#define DEBUG_ASSERT(...) ((void)0)
#endif
kernel/include/kernel/channel.h
+47
View File
@@ -0,0 +1,47 @@
#pragma once
#include <zinnia/status.h>
#include <kernel/spin.h>
#include <kernel/tailq.h>
#include <kernel/wait.h>
#include <stddef.h>
struct namespace_desc;
struct channel_msg {
void* data;
size_t num_bytes;
struct namespace_desc* descs;
size_t num_handles;
TAILQ_LINK(struct channel_msg) next;
};
struct channel {
struct channel* peer;
struct spinlock lock;
TAILQ_HEAD(struct channel_msg) messages;
struct wait_queue waiters;
};
// Creates a linked pair of channel endpoints.
zn_status_t channel_create(struct channel** ep0, struct channel** ep1);
// Writes a message to the peer channel.
// On success, ownership of `data` and `descs` transfers to the message.
// On failure, the caller retains ownership and must free them.
zn_status_t channel_write(
struct channel* ch,
void* data,
size_t num_bytes,
struct namespace_desc* descs,
size_t num_handles
);
// Non-blocking dequeue of the next message.
zn_status_t channel_read(struct channel* ch, struct channel_msg** msg);
// Non-blocking check for message availability.
zn_status_t channel_poll(struct channel* ch);
// Frees a channel message and its associated data and descriptors.
void channel_msg_free(struct channel_msg* msg);
kernel/include/kernel/clock.h
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include <stdint.h>
struct clock {
const char* name;
uint8_t priority;
void (*reset)(struct clock* c);
uint64_t (*get_elapsed_ns)(struct clock* c);
};
// Attempts to set a new clock. Returns true if the clock was changed.
bool clock_switch(struct clock* clock);
// Returns true, if a clock is available.
bool clock_available();
// Gets the current timestamp in nanoseconds since the last reset.
uint64_t clock_get_elapsed_ns();
// Spin for `ns` nanoseconds.
void clock_spin_ns(uint64_t ns);
kernel/include/kernel/cmdline.h
+46
View File
@@ -0,0 +1,46 @@
#pragma once
#include <common/compiler.h>
#include <common/utils.h>
#include <stdint.h>
#define _CMDLINE_OPTION(x, opt_name, opt_func) \
static const struct cmdline_option x = { \
.name = opt_name, \
.option_type = _Generic( \
(opt_func), \
void (*)(const char*): CMDLINE_STRING, \
void (*)(bool): CMDLINE_BOOL, \
void (*)(intptr_t): CMDLINE_INTPTR, \
void (*)(uintptr_t): CMDLINE_UINTPTR \
), \
.func = (void*)(opt_func), \
}; \
[[__used, __section(".cmdline")]] \
static const struct cmdline_option* CONCAT(x, _ptr) = &x;
#define CMDLINE_OPTION(opt_name, opt_func) _CMDLINE_OPTION(UNIQUE_IDENT(cmdline_option), opt_name, opt_func)
enum cmdline_type {
CMDLINE_STRING,
CMDLINE_BOOL,
CMDLINE_INTPTR,
CMDLINE_UINTPTR,
};
struct cmdline_option {
// The name of this option.
const char* name;
// Gets called if this option is present on the command line.
union {
void (*func_str)(const char* value);
void (*func_bool)(bool value);
void (*func_intptr)(intptr_t value);
void (*func_uintptr)(uintptr_t value);
void* func;
};
enum cmdline_type option_type;
};
// Parses the command line and invokes all options.
void cmdline_parse(const char* cmdline);
kernel/include/kernel/console.h
+27
View File
@@ -0,0 +1,27 @@
#pragma once
#include <common/compiler.h>
#include <common/utils.h>
#include <stddef.h>
struct console {
const char* name;
void* private;
// Initializes the console.
bool (*init)(struct console* con);
// Writes to the console.
void (*write)(struct console* con, const char* buf, size_t count);
};
// Initializes a console for logging.
void console_init();
// Writes a message to a console.
void console_write(const char* buf, size_t len);
// Defines a new early console which can be used to log early boot messages.
// An early console must not make any heap allocations.
#define DEFINE_CONSOLE(con) \
[[__used, __section(".console")]] \
static struct console* UNIQUE_IDENT(console) = (con)
kernel/include/kernel/elf.h
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include <zinnia/status.h>
#include <kernel/namespace.h>
#include <kernel/sched.h>
#include <kernel/vas.h>
#include <kernel/vmo.h>
struct exec_info {
struct vas* space;
struct vmo* file_obj;
struct namespace* ns;
const char** argv;
const char** envp;
struct namespace_desc* init_handle;
};
zn_status_t elf_load(struct exec_info* info, struct task** out);
kernel/include/kernel/futex.h
+15
View File
@@ -0,0 +1,15 @@
#pragma once
#include <zinnia/status.h>
#include <common/compiler.h>
#include <kernel/wait.h>
// Initializes the global futex table.
void futex_init(void);
// Blocks the calling task if *addr == expected.
zn_status_t futex_wait(__user int* addr, int expected);
// Wakes up to `count` tasks waiting on the given address.
// Returns the number of tasks woken.
zn_status_t futex_wake(__user int* addr, int count);
kernel/include/kernel/handle.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
#include <zinnia/handle.h>
kernel/include/kernel/hashmap.h
+223
View File
@@ -0,0 +1,223 @@
#pragma once
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
// FNV-1a hash over raw bytes.
static inline uint64_t hashmap_hash_default(const void* key, size_t len) {
const uint8_t* data = (const uint8_t*)key;
uint64_t hash = 0xcbf29ce484222325;
for (size_t i = 0; i < len; i++) {
hash ^= data[i];
hash *= 0x100000001b3;
}
return hash;
}
// Byte-wise equality.
static inline bool hashmap_eq_default(const void* a, const void* b, size_t len) {
return memcmp(a, b, len) == 0;
}
// Ensure a hash is never zero (zero is the empty sentinel).
#define _HASHMAP_FIXHASH(h) ((h) == 0 ? (uint64_t)1 : (h))
#define HASHMAP(K, V) \
struct { \
K* keys; \
V* values; \
uint64_t* hashes; \
size_t capacity; \
size_t count; \
}
#define HASHMAP_INIT(map) \
do { \
(map)->keys = nullptr; \
(map)->values = nullptr; \
(map)->hashes = nullptr; \
(map)->capacity = 0; \
(map)->count = 0; \
} while (0)
#define HASHMAP_DESTROY(map) \
do { \
mem_free((map)->keys); \
mem_free((map)->values); \
mem_free((map)->hashes); \
(map)->keys = nullptr; \
(map)->values = nullptr; \
(map)->hashes = nullptr; \
(map)->capacity = 0; \
(map)->count = 0; \
} while (0)
#define HASHMAP_COUNT(map) ((map)->count)
// Internal: probe distance from ideal position.
#define _HASHMAP_PROBE_DIST(hash, pos, cap) (((pos) - ((hash) & ((cap) - 1))) & ((cap) - 1))
// Internal: grow and rehash.
#define _HASHMAP_GROW(map, hash_fn, eq_fn) \
do { \
size_t _old_cap = (map)->capacity; \
size_t _new_cap = _old_cap == 0 ? 16 : _old_cap * 2; \
typeof((map)->keys) _old_keys = (map)->keys; \
typeof((map)->values) _old_values = (map)->values; \
uint64_t* _old_hashes = (map)->hashes; \
(map)->keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \
(map)->values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \
(map)->hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \
(map)->capacity = _new_cap; \
(map)->count = 0; \
for (size_t _i = 0; _i < _old_cap; _i++) { \
if (_old_hashes[_i] != 0) { \
_HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \
} \
} \
mem_free(_old_keys); \
mem_free(_old_values); \
mem_free(_old_hashes); \
} while (0)
// Internal: insert with a precomputed hash. Used by grow and the public INSERT macro.
#define _HASHMAP_INSERT_INNER(map, k, v, h) \
do { \
size_t _cap = (map)->capacity; \
size_t _pos = (h) & (_cap - 1); \
typeof(*(map)->keys) _ik = (k); \
typeof(*(map)->values) _iv = (v); \
uint64_t _ih = (h); \
for (;;) { \
if ((map)->hashes[_pos] == 0) { \
(map)->keys[_pos] = _ik; \
(map)->values[_pos] = _iv; \
(map)->hashes[_pos] = _ih; \
(map)->count++; \
break; \
} \
size_t _existing_dist = _HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap); \
size_t _insert_dist = _HASHMAP_PROBE_DIST(_ih, _pos, _cap); \
if (_insert_dist > _existing_dist) { \
typeof(*(map)->keys) _tk = (map)->keys[_pos]; \
typeof(*(map)->values) _tv = (map)->values[_pos]; \
uint64_t _th = (map)->hashes[_pos]; \
(map)->keys[_pos] = _ik; \
(map)->values[_pos] = _iv; \
(map)->hashes[_pos] = _ih; \
_ik = _tk; \
_iv = _tv; \
_ih = _th; \
} \
_pos = (_pos + 1) & (_cap - 1); \
} \
} while (0)
// Insert or update a key-value pair. Returns ZN_OK or ZN_ERR_NO_MEMORY.
#define HASHMAP_INSERT(map, key, val, hash_fn, eq_fn) \
({ \
zn_status_t _status = ZN_OK; \
typeof(*(map)->keys) _k = (key); \
uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \
/* Check for existing key first. */ \
bool _found = false; \
if ((map)->capacity > 0) { \
size_t _cap = (map)->capacity; \
size_t _pos = _h & (_cap - 1); \
for (size_t _d = 0; _d < _cap; _d++) { \
if ((map)->hashes[_pos] == 0) \
break; \
if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \
break; \
if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \
(map)->values[_pos] = (val); \
_found = true; \
break; \
} \
_pos = (_pos + 1) & (_cap - 1); \
} \
} \
if (!_found) { \
if ((map)->count * 4 >= (map)->capacity * 3 || (map)->capacity == 0) { \
_HASHMAP_GROW(map, hash_fn, eq_fn); \
if ((map)->keys == nullptr || (map)->values == nullptr || (map)->hashes == nullptr) { \
_status = ZN_ERR_NO_MEMORY; \
} \
} \
if (_status == ZN_OK) { \
_HASHMAP_INSERT_INNER(map, _k, (val), _h); \
} \
} \
_status; \
})
// Look up a key. Returns a pointer to the value, or nullptr if not found.
#define HASHMAP_GET(map, key, hash_fn, eq_fn) \
({ \
typeof(*(map)->values)* _result = nullptr; \
if ((map)->capacity > 0) { \
typeof(*(map)->keys) _k = (key); \
uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \
size_t _cap = (map)->capacity; \
size_t _pos = _h & (_cap - 1); \
for (size_t _d = 0; _d < _cap; _d++) { \
if ((map)->hashes[_pos] == 0) \
break; \
if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \
break; \
if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \
_result = &(map)->values[_pos]; \
break; \
} \
_pos = (_pos + 1) & (_cap - 1); \
} \
} \
_result; \
})
// Remove a key. Returns true if the key was found and removed.
// Uses backward-shift deletion to avoid tombstones.
#define HASHMAP_REMOVE(map, key, hash_fn, eq_fn) \
({ \
bool _removed = false; \
if ((map)->capacity > 0) { \
typeof(*(map)->keys) _k = (key); \
uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \
size_t _cap = (map)->capacity; \
size_t _pos = _h & (_cap - 1); \
for (size_t _d = 0; _d < _cap; _d++) { \
if ((map)->hashes[_pos] == 0) \
break; \
if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \
break; \
if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \
(map)->hashes[_pos] = 0; \
(map)->count--; \
/* Backward-shift: move subsequent displaced entries back. */ \
size_t _empty = _pos; \
size_t _next = (_pos + 1) & (_cap - 1); \
while ((map)->hashes[_next] != 0 && _HASHMAP_PROBE_DIST((map)->hashes[_next], _next, _cap) > 0) { \
(map)->keys[_empty] = (map)->keys[_next]; \
(map)->values[_empty] = (map)->values[_next]; \
(map)->hashes[_empty] = (map)->hashes[_next]; \
(map)->hashes[_next] = 0; \
_empty = _next; \
_next = (_next + 1) & (_cap - 1); \
} \
_removed = true; \
break; \
} \
_pos = (_pos + 1) & (_cap - 1); \
} \
} \
_removed; \
})
// Iterate over all occupied entries. `idx` is a size_t loop variable.
// Access keys and values via (map)->keys[idx] and (map)->values[idx].
#define HASHMAP_FOREACH(map, idx) \
for (size_t idx = 0; idx < (map)->capacity; idx++) \
if ((map)->hashes[idx] != 0)
kernel/include/kernel/init.h
+43
View File
@@ -0,0 +1,43 @@
#pragma once
#include <common/compiler.h>
#include <common/utils.h>
#include <kernel/types.h>
#include <stddef.h>
#include <stdint.h>
#define __init __used, __section(".init.text"), __cold
#define __initdata __used, __section(".init.data")
#define __initdata_sorted(name) __used, __section(".init.data." name)
struct boot_file {
phys_t data;
size_t length;
char path[128];
};
struct boot_info {
char* cmdline;
struct phys_mem* mem_map;
size_t num_mem_maps;
phys_t phys_base;
uintptr_t virt_base;
uintptr_t hhdm_base;
struct boot_file* files;
size_t num_files;
phys_t rsdp;
};
extern uint8_t __ld_stack_top[];
extern uint8_t __ld_stack_bottom[];
extern phys_t rsdp_addr;
// Entry point for the kernel after arch-specific setup has finished.
void kernel_entry();
// Initializes the early parts of the kernel.
void kernel_early_init();
[[noreturn]]
void kernel_main(struct boot_info* info);
kernel/include/kernel/irq.h
+51
View File
@@ -0,0 +1,51 @@
#pragma once
#include <common/compiler.h>
#include <kernel/list.h>
#include <bits/irq.h>
#include <stdint.h>
enum irq_status : uint8_t {
IRQ_STATUS_HANDLED,
IRQ_STATUS_IGNORED,
};
enum irq_polarity : uint8_t {
IRQ_POLARITY_LOW,
IRQ_POLARITY_HIGH,
};
enum irq_trigger_mode : uint8_t {
IRQ_TRIGGER_EDGE,
IRQ_TRIGGER_LEVEL,
};
struct irq_handler {
enum irq_status (*handler)(void* self);
SLIST_LINK(struct irq_handler*) next;
};
struct irq_line {
int (*set_config)(enum irq_trigger_mode mode, enum irq_polarity polarity);
void (*mask)(struct irq_line* self);
void (*unmask)(struct irq_line* self);
void (*eoi)(struct irq_line* self);
SLIST_HEAD(struct irq_handler*) handlers;
bool is_busy;
};
void irq_line_attach(struct irq_line* self, struct irq_handler* handler);
struct irq_percpu {
uint32_t level;
bool in_interrupt;
};
void irq_lock();
void irq_unlock();
bool irq_set_interrupted(bool is_interrupted);
void arch_irq_set_state(bool state);
bool arch_irq_get_state();
void arch_irq_wait();
kernel/include/kernel/list.h
+29
View File
@@ -0,0 +1,29 @@
#pragma once
#define SLIST_HEAD(type) \
struct { \
type sl_head; \
}
#define SLIST_LINK(type) \
struct { \
type sl_link; \
}
#define SLIST_EMPTY(head) ((head)->sl_head == NULL)
#define SLIST_FIRST(head) ((head)->sl_head)
#define SLIST_NEXT(elm, field) ((elm)->field.sl_link)
#define SLIST_INSERT_END(slistelm, elm, field) \
do { \
SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \
SLIST_NEXT((slistelm), field) = (elm); \
} while (0)
#define SLIST_INSERT_HEAD(head, elm, field) \
do { \
SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \
SLIST_FIRST((head)) = (elm); \
} while (0)
#define SLIST_FOREACH(var, head, field) for ((var) = SLIST_FIRST((head)); (var); (var) = SLIST_NEXT((var), field))
kernel/include/kernel/mmio.h
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#include <kernel/types.h>
#include <stddef.h>
void* mmio_new(phys_t addr, size_t length);
void mmio_free(void* ptr, size_t length);
kernel/include/kernel/mutex.h
+11
View File
@@ -0,0 +1,11 @@
#pragma once
struct task;
struct mutex {
struct task* owner;
bool flag;
};
void mutex_lock(struct mutex* mutex);
void mutex_unlock(struct mutex* mutex);
kernel/include/kernel/namespace.h
+61
View File
@@ -0,0 +1,61 @@
#pragma once
#include <zinnia/handle.h>
#include <kernel/channel.h>
#include <kernel/hashmap.h>
#include <kernel/mutex.h>
#include <kernel/spin.h>
#include <kernel/vas.h>
#include <kernel/vmo.h>
enum namespace_desc_type {
NAMESPACE_DESC_NS,
NAMESPACE_DESC_TASK,
NAMESPACE_DESC_VAS,
NAMESPACE_DESC_VMO,
NAMESPACE_DESC_CHANNEL,
};
struct namespace_desc {
enum namespace_desc_type type;
zn_rights_t rights;
union {
struct namespace* namespace;
struct task* task;
struct channel* channel;
struct vas* vas;
struct vmo* vmo;
} value;
};
// Translates between handles and objects.
struct namespace {
zn_handle_t next_handle;
struct mutex mutex;
HASHMAP(zn_handle_t, struct namespace_desc) descriptors;
};
// Creates a new namespace with no contents.
zn_status_t namespace_new(struct namespace** out);
// Adds a descriptor to this namespace with an auto-allocated handle.
zn_status_t namespace_add_desc(struct namespace* namespace, struct namespace_desc desc, zn_handle_t* out);
// Adds a descriptor to this namespace at a specific handle value.
zn_status_t namespace_add_desc_at(struct namespace* namespace, struct namespace_desc desc, zn_handle_t handle);
// Gets the descriptor referenced by the given handle.
zn_status_t namespace_get(struct namespace* namespace, zn_handle_t handle, struct namespace_desc* out);
// Deletes a descriptor from this namespace.
zn_status_t namespace_del_desc(struct namespace* namespace, zn_handle_t handle);
// Clones a descriptor with optionally restricted rights.
// If cloned_rights has ZN_RIGHTS_IDENTICAL set, the clone keeps the original rights.
// Otherwise, cloned_rights must be a subset of the original descriptor's rights.
zn_status_t namespace_clone_desc(
struct namespace* ns,
struct namespace_desc* desc,
zn_rights_t cloned_rights,
zn_handle_t* out
);
kernel/include/kernel/pager.h
+8
View File
@@ -0,0 +1,8 @@
#pragma once
#include <zinnia/status.h>
#include <stdint.h>
struct pager {
zn_status_t (*try_get_page)(struct pager* pager, uintptr_t offset);
};
kernel/include/kernel/panic.h
+6
View File
@@ -0,0 +1,6 @@
#pragma once
#include <common/compiler.h>
// Stop all execution upon panic.
[[noreturn, __format(printf, 1, 2)]]
void panic(const char* msg, ...);
kernel/include/kernel/percpu.h
+43
View File
@@ -0,0 +1,43 @@
#pragma once
#include <common/compiler.h>
#include <kernel/irq.h>
#include <kernel/sched.h>
#include <kernel/usercopy.h>
#include <bits/percpu.h>
#include <stddef.h>
ASSERT_TYPE(struct arch_percpu);
// CPU-relative information.
struct percpu {
struct percpu* self; // A pointer to this structure.
size_t id; // The virtual ID of this CPU.
bool online; // Whether this CPU is initialized and active.
uintptr_t kernel_stack; // The kernel mode stack.
uintptr_t user_stack; // The user mode stack.
struct arch_percpu arch; // Architecture-specific fields.
struct irq_percpu irq; // IRQ information.
struct sched_percpu sched; // Scheduler information.
};
// Per-CPU data for the bootstrap processor.
extern struct percpu percpu_bsp;
// Allocates a block of memory for a new CPU.
struct percpu* percpu_new();
// Gets the per-CPU data on the current CPU.
static inline struct percpu* percpu_get() {
return arch_percpu_get();
}
// Initializes the bootstrap processor.
static inline void percpu_bsp_init() {
arch_percpu_bsp_init();
}
// Initializes all processors.
static inline void percpu_init() {
arch_percpu_init();
}
kernel/include/kernel/pmap.h
+44
View File
@@ -0,0 +1,44 @@
#pragma once
#include <zinnia/status.h>
#include <common/utils.h>
#include <kernel/alloc.h>
#include <kernel/spin.h>
#include <bits/mmu.h>
ASSERT_TYPE(pte_t);
enum pte_flags {
PTE_READ = 1 << 0, // Can read from this page.
PTE_WRITE = 1 << 1, // Can write to this page.
PTE_EXEC = 1 << 2, // Can execute code on this page.
PTE_USER = 1 << 3, // Can be accessed by the user.
PTE_DIR = 1 << 4, // Is a non-leaf page.
};
enum cache_mode {
CACHE_NONE,
CACHE_WRITE_COMBINE,
CACHE_WRITE_THROUGH,
CACHE_WRITE_BACK,
CACHE_MMIO,
};
struct pmap {
phys_t root;
struct spinlock lock;
bool is_user;
};
zn_status_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags);
zn_status_t pmap_new_user(struct pmap* pt, enum alloc_flags flags);
zn_status_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache);
zn_status_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags);
zn_status_t pmap_unmap(struct pmap* pt, uintptr_t vaddr);
void pmap_set(struct pmap* pt);
void pte_clear(pte_t* pte);
pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache);
bool pte_is_present(pte_t* pte);
bool pte_is_dir(pte_t* pte);
phys_t pte_address(pte_t* pte);
kernel/include/kernel/print.h
+9
View File
@@ -0,0 +1,9 @@
#pragma once
#include <common/compiler.h>
#include <stdarg.h>
[[__format(printf, 1, 2)]]
void kprintf(const char* message, ...);
void kvprintf(const char* message, va_list args);
kernel/include/kernel/sched.h
+104
View File
@@ -0,0 +1,104 @@
#pragma once
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <kernel/init.h>
#include <kernel/namespace.h>
#include <kernel/tailq.h>
#include <kernel/types.h>
#include <kernel/wait.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
ASSERT_TYPE(struct arch_context);
ASSERT_TYPE(struct arch_task_context);
enum task_state {
TASK_STATE_RUNNING, // Task is active and running.
TASK_STATE_READY, // Task is active, but not currently scheduled.
TASK_STATE_BLOCKED, // Task is waiting on another object.
TASK_STATE_DEAD, // Task is killed.
};
#define TASK_NAME_MAX 128
#define KERNEL_STACK_SIZE 0x4000
// A task is the smallest unit of the scheduler.
struct task {
size_t id;
char name[TASK_NAME_MAX];
struct vas* space;
struct namespace* namespace;
enum task_state state;
struct arch_task_context context;
struct usercopy_region* usercopy_region;
uintptr_t kernel_stack;
uintptr_t user_stack;
size_t time_slice;
int8_t priority;
TAILQ_LINK(struct task) next;
};
// Per-CPU data for scheduling.
struct sched_percpu {
struct task* current;
struct task* idle_task;
size_t preempt_level;
TAILQ_HEAD(struct task) run_queue;
};
// Blocks the current task on the given wait queue.
// The caller must NOT hold any spinlocks. Returns after the task is woken.
void sched_block(struct sched_percpu* sched, struct wait_queue* wq);
// Wakes the first task blocked on the given wait queue, if any.
void sched_wake(struct sched_percpu* sched, struct wait_queue* wq);
// Wakes all tasks blocked on the given wait queue.
void sched_wake_all(struct sched_percpu* sched, struct wait_queue* wq);
typedef void (*task_fn_t)(uintptr_t arg0);
zn_status_t task_create(
const char* name,
struct vas* space,
struct namespace* ns,
task_fn_t entry,
uintptr_t arg0,
struct task** out
);
[[noreturn]]
void task_entry(task_fn_t entry, uintptr_t arg0);
void sched_init(struct sched_percpu* sched);
void sched_reschedule(struct sched_percpu* sched);
void sched_yield(struct sched_percpu* sched);
void sched_add_task(struct sched_percpu* sched, struct task* task);
[[noreturn]]
void sched_to_user(uintptr_t ctx);
[[noreturn]]
void sched_to_user_context(uintptr_t ctx);
// Info block for sched_to_user. Heap-allocated by the caller, freed by sched_to_user_arg.
struct task_start_info {
uintptr_t ip;
uintptr_t sp;
uintptr_t arg;
};
void arch_sched_preempt_disable();
bool arch_sched_preempt_enable();
void arch_sched_switch(struct task* from, struct task* to);
zn_status_t arch_task_init(
struct arch_task_context* context,
void* entry,
uintptr_t arg0,
uintptr_t stack_start,
bool is_user
);
[[noreturn]]
void arch_sched_jump_to_context(struct arch_context* context);
[[noreturn]]
void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg);
kernel/include/kernel/spin.h
+13
View File
@@ -0,0 +1,13 @@
#pragma once
// Busy-waits in a loop until the lock is freed.
// Does not put the CPU to sleep.
struct spinlock {
bool locked;
};
// Attempts to lock a spinlock. If it's already locked, waits until it's freed.
void spin_lock(struct spinlock* spin);
// Unlocks a spinlock.
void spin_unlock(struct spinlock* spin);
kernel/include/kernel/syscalls.h
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include <kernel/sched.h>
typedef zn_status_t (*syscall_fn_t)(struct arch_context* ctx);
void syscall_dispatch(struct arch_context* ctx);
zn_status_t syscall_log(struct arch_context* ctx);
zn_status_t syscall_archctl(struct arch_context* ctx);
zn_status_t syscall_page_size(struct arch_context* ctx);
zn_status_t syscall_time(struct arch_context* ctx);
zn_status_t syscall_handle_validate(struct arch_context* ctx);
zn_status_t syscall_handle_drop(struct arch_context* ctx);
zn_status_t syscall_handle_clone(struct arch_context* ctx);
zn_status_t syscall_vas_create(struct arch_context* ctx);
zn_status_t syscall_vmo_create(struct arch_context* ctx);
zn_status_t syscall_vmo_create_phys(struct arch_context* ctx);
zn_status_t syscall_vmo_map(struct arch_context* ctx);
zn_status_t syscall_channel_create(struct arch_context* ctx);
zn_status_t syscall_channel_wait(struct arch_context* ctx);
zn_status_t syscall_channel_read(struct arch_context* ctx);
zn_status_t syscall_channel_write(struct arch_context* ctx);
zn_status_t syscall_thread_create(struct arch_context* ctx);
zn_status_t syscall_thread_start(struct arch_context* ctx);
zn_status_t syscall_thread_exit(struct arch_context* ctx);
zn_status_t syscall_random_get(struct arch_context* ctx);
zn_status_t syscall_random_entropy(struct arch_context* ctx);
zn_status_t syscall_futex_wait(struct arch_context* ctx);
zn_status_t syscall_futex_wake(struct arch_context* ctx);
kernel/include/kernel/tailq.h
+52
View File
@@ -0,0 +1,52 @@
#pragma once
#define TAILQ_HEAD(type) \
struct { \
type* tqh_first; \
type** tqh_last; \
}
#define TAILQ_LINK(type) \
struct { \
type* tqe_next; \
type** tqe_prev; \
}
#define TAILQ_INIT(head) \
do { \
(head)->tqh_first = nullptr; \
(head)->tqh_last = &(head)->tqh_first; \
} while (0)
#define TAILQ_FIRST(head) ((head)->tqh_first)
#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next)
#define TAILQ_EMPTY(head) (TAILQ_FIRST(head) == nullptr)
#define TAILQ_INSERT_TAIL(head, elm, field) \
do { \
(elm)->field.tqe_next = nullptr; \
(elm)->field.tqe_prev = (head)->tqh_last; \
*(head)->tqh_last = (elm); \
(head)->tqh_last = &(elm)->field.tqe_next; \
} while (0)
#define TAILQ_INSERT_HEAD(head, elm, field) \
do { \
if (((elm)->field.tqe_next = (head)->tqh_first) != nullptr) \
(head)->tqh_first->field.tqe_prev = &(elm)->field.tqe_next; \
else \
(head)->tqh_last = &(elm)->field.tqe_next; \
(head)->tqh_first = (elm); \
(elm)->field.tqe_prev = &(head)->tqh_first; \
} while (0)
#define TAILQ_REMOVE(head, elm, field) \
do { \
if (((elm)->field.tqe_next) != nullptr) \
(elm)->field.tqe_next->field.tqe_prev = (elm)->field.tqe_prev; \
else \
(head)->tqh_last = (elm)->field.tqe_prev; \
*(elm)->field.tqe_prev = (elm)->field.tqe_next; \
} while (0)
kernel/include/kernel/types.h
+8
View File
@@ -0,0 +1,8 @@
#pragma once
#include <common/utils.h>
#include <bits/types.h>
ASSERT_TYPE(phys_t);
ASSERT_TYPE(uintptr_t);
ASSERT_TYPE(reg_t);
kernel/include/kernel/usercopy.h
+24
View File
@@ -0,0 +1,24 @@
#pragma once
#include <common/compiler.h>
#include <kernel/percpu.h>
#include <stddef.h>
struct usercopy_region {
void (*start_ip)();
void (*end_ip)();
void (*fault_ip)();
};
// Copies a block of data from user to kernel memory.
bool usercopy_read(void* dst, const __user void* src, size_t len);
// Copies a block of data from kernel to user memory.
bool usercopy_write(__user void* dst, const void* src, size_t len);
// Performs a strlen() on a user string.
bool usercopy_strlen(const __user char* str, size_t max, size_t* len);
bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region);
bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region);
bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region);
kernel/include/kernel/vas.h
+39
View File
@@ -0,0 +1,39 @@
#pragma once
#include <zinnia/mem.h>
#include <zinnia/status.h>
#include <kernel/mutex.h>
#include <kernel/pmap.h>
#include <kernel/vmo.h>
// Virtual address space.
struct vas {
// Architecture-specific virtual memory map implementation.
struct pmap pmap;
};
// Try to create a new, empty virtual address space.
zn_status_t vma_new(struct vas** out);
// Delete a virtual address space.
void vma_delete(struct vas* vm);
zn_status_t vma_map_vmo(
struct vas* vm,
struct vmo* vmo,
uintptr_t addr,
size_t len,
enum zn_vm_flags flags,
uintptr_t vmo_offset
);
// Remove any mappings in the given address range.
zn_status_t vma_unmap(struct vas* vm, uintptr_t vaddr, size_t size);
// Change the protection of an existing mapping.
zn_status_t vma_protect(struct vas* vm, uintptr_t vaddr, size_t size, enum zn_vm_flags flags);
// Try to handle a page fault.
bool vma_page_fault(struct vas* vm, uintptr_t vaddr, enum zn_vm_flags access_type);
extern struct vas kernel_vas;
kernel/include/kernel/vdso.h
+6
View File
@@ -0,0 +1,6 @@
#pragma once
#include <stdint.h>
extern uint8_t __ld_vdso_start[];
extern uint8_t __ld_vdso_end[];
kernel/include/kernel/vec.h
+105
View File
@@ -0,0 +1,105 @@
#pragma once
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <stddef.h>
#include <string.h>
#define VEC(T) \
struct { \
T* data; \
size_t length; \
size_t capacity; \
}
#define VEC_INIT(vec) \
do { \
(vec)->data = nullptr; \
(vec)->length = 0; \
(vec)->capacity = 0; \
} while (0)
#define VEC_DESTROY(vec) \
do { \
mem_free((vec)->data); \
(vec)->data = nullptr; \
(vec)->length = 0; \
(vec)->capacity = 0; \
} while (0)
#define VEC_LENGTH(vec) ((vec)->length)
#define VEC_CAPACITY(vec) ((vec)->capacity)
#define VEC_EMPTY(vec) ((vec)->length == 0)
#define VEC_AT(vec, i) ((vec)->data[i])
// Internal: grow to at least `needed` capacity.
#define _VEC_GROW(vec, needed) \
do { \
size_t _new_cap = (vec)->capacity == 0 ? 8 : (vec)->capacity; \
while (_new_cap < (needed)) \
_new_cap *= 2; \
typeof((vec)->data) _new = mem_alloc(_new_cap * sizeof(*(vec)->data), ALLOC_NOZERO); \
if (_new != nullptr && (vec)->length > 0) \
memcpy(_new, (vec)->data, (vec)->length * sizeof(*(vec)->data)); \
mem_free((vec)->data); \
(vec)->data = _new; \
(vec)->capacity = _new_cap; \
} while (0)
// Append an element. Returns ZN_OK or ZN_ERR_NO_MEMORY.
#define VEC_PUSH(vec, val) \
({ \
zn_status_t _status = ZN_OK; \
if ((vec)->length >= (vec)->capacity) { \
_VEC_GROW(vec, (vec)->length + 1); \
if ((vec)->data == nullptr) \
_status = ZN_ERR_NO_MEMORY; \
} \
if (_status == ZN_OK) \
(vec)->data[(vec)->length++] = (val); \
_status; \
})
// Remove and return the last element. Undefined if empty.
#define VEC_POP(vec) ((vec)->data[--(vec)->length])
// Insert at index, shifting subsequent elements right. Returns ZN_OK or ZN_ERR_NO_MEMORY.
#define VEC_INSERT(vec, i, val) \
({ \
zn_status_t _status = ZN_OK; \
size_t _idx = (i); \
if ((vec)->length >= (vec)->capacity) { \
_VEC_GROW(vec, (vec)->length + 1); \
if ((vec)->data == nullptr) \
_status = ZN_ERR_NO_MEMORY; \
} \
if (_status == ZN_OK) { \
memmove(&(vec)->data[_idx + 1], &(vec)->data[_idx], ((vec)->length - _idx) * sizeof(*(vec)->data)); \
(vec)->data[_idx] = (val); \
(vec)->length++; \
} \
_status; \
})
// Remove element at index, shifting subsequent elements left.
#define VEC_REMOVE(vec, i) \
do { \
size_t _idx = (i); \
(vec)->length--; \
if (_idx < (vec)->length) \
memmove(&(vec)->data[_idx], &(vec)->data[_idx + 1], ((vec)->length - _idx) * sizeof(*(vec)->data)); \
} while (0)
// Remove element at index by swapping with the last element (O(1), unordered).
#define VEC_SWAP_REMOVE(vec, i) \
do { \
size_t _idx = (i); \
(vec)->length--; \
if (_idx < (vec)->length) \
(vec)->data[_idx] = (vec)->data[(vec)->length]; \
} while (0)
// Clear all elements without freeing memory.
#define VEC_CLEAR(vec) ((vec)->length = 0)
#define VEC_FOREACH(vec, idx) for (size_t idx = 0; idx < (vec)->length; idx++)
kernel/include/kernel/virt.h
+30
View File
@@ -0,0 +1,30 @@
#pragma once
#include <stddef.h>
#include <stdint.h>
enum page_type : uint32_t {
PAGE_PHYS = 0, // Regular physical memory.
};
struct page {
enum page_type type;
uint32_t flags;
size_t refcount;
union {
struct {
struct page* next; // Pointer to the next chunk.
size_t count; // Amount of free pages.
} freelist;
};
};
static_assert(0x1000 % sizeof(struct page) == 0, "struct must fit cleanly inside a page!");
static_assert(sizeof(struct page) <= 64, "struct must be smaller than 64 bytes!");
void* vm_alloc(size_t length);
void vm_free(void* addr, size_t length);
// Base address of the `struct page` array.
extern struct page* vm_pfndb;
uintptr_t vm_page_addr(struct page* page);
kernel/include/kernel/vmo.h
+47
View File
@@ -0,0 +1,47 @@
#pragma once
#include <zinnia/status.h>
#include <kernel/list.h>
#include <kernel/spin.h>
#include <kernel/virt.h>
#include <stddef.h>
#include <stdint.h>
struct vmo {
zn_status_t (*get_page)(struct vmo* vmo, uintptr_t offset_idx, struct page** out);
};
zn_status_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read);
zn_status_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written);
zn_status_t vmo_copy(
struct vmo* target,
uintptr_t target_offset,
struct vmo* src,
uintptr_t src_offset,
size_t len,
size_t* actual_copied
);
struct pager_ops {
zn_status_t (*get_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page** out);
zn_status_t (*put_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page* page);
};
struct page_list {
uintptr_t offset;
struct page* value;
SLIST_LINK(struct page_list*) next;
};
struct paged_vmo {
struct vmo object;
struct pager_ops source;
SLIST_HEAD(struct page_list*) cache;
struct spinlock lock;
};
zn_status_t vmo_new_paged(struct pager_ops ops, struct paged_vmo** out);
zn_status_t vmo_new_phys(struct paged_vmo** out);
// Creates a VMO backed by a specific physical address range (e.g. for MMIO).
zn_status_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out);
kernel/include/kernel/wait.h
+14
View File
@@ -0,0 +1,14 @@
#pragma once
#include <kernel/spin.h>
#include <kernel/tailq.h>
struct task;
// A queue of tasks waiting for an event.
struct wait_queue {
TAILQ_HEAD(struct task) waiters;
struct spinlock lock;
};
void wait_queue_init(struct wait_queue* wq);
kernel/include/stdarg.h
+10
View File
@@ -0,0 +1,10 @@
#ifndef ZINNIA_STDARG_H
#define ZINNIA_STDARG_H
typedef __builtin_va_list va_list;
#define va_start(v, l) __builtin_va_start(v, l)
#define va_end(v) __builtin_va_end(v)
#define va_arg(v, T) __builtin_va_arg(v, T)
#define va_copy(d, s) __builtin_va_copy(d, s)
#endif
kernel/include/stdatomic.h
+39
View File
@@ -0,0 +1,39 @@
#ifndef ZINNIA_STDATOMIC_H
#define ZINNIA_STDATOMIC_H
typedef enum memory_order {
memory_order_relaxed = __ATOMIC_RELAXED,
memory_order_consume = __ATOMIC_CONSUME,
memory_order_acquire = __ATOMIC_ACQUIRE,
memory_order_release = __ATOMIC_RELEASE,
memory_order_acq_rel = __ATOMIC_ACQ_REL,
memory_order_seq_cst = __ATOMIC_SEQ_CST
} memory_order;
#define atomic_store_explicit(ptr, des, ord) __atomic_store_n(ptr, des, ord)
#define atomic_store(ptr, des) atomic_store_explicit(ptr, des, __ATOMIC_SEQ_CST)
#define atomic_load_explicit(ptr, ord) __atomic_load_n(ptr, ord)
#define atomic_load(ptr) atomic_load_explicit(ptr, __ATOMIC_SEQ_CST)
#define atomic_exchange_explicit(ptr, des, ord) __atomic_exchange_n(ptr, des, ord)
#define atomic_exchange(ptr, des) atomic_exchange_explicit(ptr, des, __ATOMIC_SEQ_CST)
#define atomic_fetch_add_explicit(ptr, op, ord) __atomic_fetch_add(ptr, op, ord)
#define atomic_fetch_add(ptr, op) atomic_fetch_add_explicit(ptr, op, __ATOMIC_SEQ_CST)
#define atomic_fetch_sub_explicit(ptr, op, ord) __atomic_fetch_sub(ptr, op, ord)
#define atomic_fetch_sub(ptr, op) atomic_fetch_sub_explicit(ptr, op, __ATOMIC_SEQ_CST)
#define atomic_fetch_or_explicit(ptr, op, ord) __atomic_fetch_or(ptr, op, ord)
#define atomic_fetch_or(ptr, op) atomic_fetch_or_explicit(ptr, op, __ATOMIC_SEQ_CST)
#define atomic_fetch_xor_explicit(ptr, op, ord) __atomic_fetch_xor(ptr, op, ord)
#define atomic_fetch_xor(ptr, op) atomic_fetch_xor_explicit(ptr, op, __ATOMIC_SEQ_CST)
#define atomic_fetch_and_explicit(ptr, op, ord) __atomic_fetch_and(ptr, op, ord)
#define atomic_fetch_and(ptr, op) atomic_fetch_and_explicit(ptr, op, __ATOMIC_SEQ_CST)
#define atomic_thread_fence(ord) __atomic_thread_fence(ord)
#endif
kernel/include/stddef.h
+9
View File
@@ -0,0 +1,9 @@
#ifndef ZINNIA_STDDEF_H
#define ZINNIA_STDDEF_H
typedef unsigned long size_t;
typedef signed long ssize_t;
#define offsetof(type, member) __builtin_offsetof(type, member)
#endif
kernel/include/stdint.h
+29
View File
@@ -0,0 +1,29 @@
#ifndef ZINNIA_STDINT_H
#define ZINNIA_STDINT_H
typedef __INT8_TYPE__ int8_t;
typedef __UINT8_TYPE__ uint8_t;
#define INT8_MAX __INT8_MAX__
#define UINT8_MAX __UINT8_MAX__
typedef __INT16_TYPE__ int16_t;
typedef __UINT16_TYPE__ uint16_t;
#define INT16_MAX __INT16_MAX__
#define UINT16_MAX __UINT16_MAX__
typedef __INT32_TYPE__ int32_t;
typedef __UINT32_TYPE__ uint32_t;
#define INT32_MAX __INT32_MAX__
#define UINT32_MAX __UINT32_MAX__
typedef __INT64_TYPE__ int64_t;
typedef __UINT64_TYPE__ uint64_t;
#define INT64_MAX __INT64_MAX__
#define UINT64_MAX __UINT64_MAX__
typedef __INTPTR_TYPE__ intptr_t;
typedef __UINTPTR_TYPE__ uintptr_t;
#define INTPTR_MAX __INTPTR_MAX__
#define UINTPTR_MAX __UINTPTR_MAX__
#endif
kernel/include/stdlib.h
+7
View File
@@ -0,0 +1,7 @@
#ifndef ZINNIA_STDLIB_H
#define ZINNIA_STDLIB_H
long atol(const char* str, int base);
unsigned long atolu(const char* str, int base);
#endif
kernel/include/string.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef ZINNIA_STRING_H
#define ZINNIA_STRING_H
#include <stddef.h>
size_t strlen(const char* str);
size_t strnlen(const char* str, size_t len);
int memcmp(const void* s1, const void* s2, size_t size);
void* memcpy(void* restrict dest, const void* restrict src, size_t n);
void* memmove(void* dstptr, const void* srcptr, size_t size);
void* memset(void* dest, int value, size_t n);
char* strncpy(char* restrict dst, const char* restrict src, size_t len);
int strcmp(const char* str1, const char* str2);
int strncmp(const char* str1, const char* str2, size_t len);
char* strchr(const char* c, int s);
#endif
kernel/include/uacpi_types.h
+39
View File
@@ -0,0 +1,39 @@
#pragma once
#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
typedef uint8_t uacpi_u8;
typedef uint16_t uacpi_u16;
typedef uint32_t uacpi_u32;
typedef uint64_t uacpi_u64;
typedef int8_t uacpi_i8;
typedef int16_t uacpi_i16;
typedef int32_t uacpi_i32;
typedef int64_t uacpi_i64;
#define UACPI_TRUE true
#define UACPI_FALSE false
typedef bool uacpi_bool;
#define UACPI_NULL nullptr
typedef uintptr_t uacpi_uintptr;
typedef uacpi_uintptr uacpi_virt_addr;
typedef size_t uacpi_size;
typedef va_list uacpi_va_list;
#define uacpi_va_start va_start
#define uacpi_va_end va_end
#define uacpi_va_arg va_arg
typedef char uacpi_char;
#define uacpi_offsetof offsetof
#define UACPI_PRIu64 "llu"
#define UACPI_PRIx64 "llx"
#define UACPI_PRIX64 "llX"
#define UACPI_FMT64(val) ((unsigned long long)(val))
kernel/kernel.ld
+97
View File
@@ -0,0 +1,97 @@
ENTRY(_start)
PHDRS {
text PT_LOAD FLAGS(5);
rodata PT_LOAD FLAGS(4);
data PT_LOAD FLAGS(6);
dynamic PT_DYNAMIC FLAGS(6);
percpu PT_LOAD FLAGS(6);
}
SECTIONS
{
. = 0xffffffff80000000;
__ld_kernel_start = .;
.text ALIGN(CONSTANT(MAXPAGESIZE)) : {
__ld_text_start = .;
KEEP(*(.init.text))
*(.text .text.*)
*(.plt .plt.*)
__ld_text_end = .;
} :text
.rodata ALIGN(CONSTANT(MAXPAGESIZE)) : {
__ld_rodata_start = .;
__ld_cmdline_start = .;
KEEP(*(.cmdline))
__ld_cmdline_end = .;
*(.rodata .rodata.*)
*(.got .got.*)
__ld_rodata_end = .;
} :rodata
.data ALIGN(CONSTANT(MAXPAGESIZE)) : {
__ld_data_start = .;
__ld_console_start = .;
KEEP(*(.console))
__ld_console_end = .;
KEEP(*(SORT_BY_NAME(.init.data.*)))
KEEP(*(.init.data))
*(.data .data.*)
} :data
.dynsym : {
__ld_dynsym_start = .;
*(.dynsym)
__ld_dynsym_end = .;
} :data
.dynstr : {
__ld_dynstr_start = .;
*(.dynstr)
__ld_dynstr_end = .;
} :data
.dynamic : {
*(.dynamic)
} :data :dynamic
.bss : {
/* Small stack to bootstrap the allocator */
__ld_stack_bottom = .;
. += 0x2000;
__ld_stack_top = .;
*(.*bss .*bss.*)
*(COMMON)
} :data
/* Per-CPU data block. MUST always come last. */
.percpu ALIGN(CONSTANT(MAXPAGESIZE)) : {
__ld_percpu_start = .;
KEEP(*(.percpu))
. = ALIGN(CONSTANT(MAXPAGESIZE));
__ld_percpu_end = .;
__ld_data_end = .;
} :percpu
__ld_kernel_end = .;
/DISCARD/ :
{
*(.comment*)
*(.eh_frame*)
*(.rel.eh_frame*)
*(.gcc_except_table*)
*(.note.gnu.property)
*(.interp)
}
}
kernel/klibc/meson.build
+5
View File
@@ -0,0 +1,5 @@
# Freestanding libc functions that are needed in the kernel
kernel_sources += files(
'string.c',
)
kernel/klibc/string.c
+164
View File
@@ -0,0 +1,164 @@
#include <common/compiler.h>
#include <common/utils.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
[[__hot]]
size_t strlen(const char* str) {
size_t result = 0;
while (*str++) {
result++;
}
return result;
}
[[__hot]]
size_t strnlen(const char* str, size_t len) {
size_t result = 0;
while (result < len && *str++) {
result++;
}
return result;
}
[[__hot]]
int memcmp(const void* s1, const void* s2, size_t size) {
int diff = 0;
char* s1ptr = (char*)s1;
char* s2ptr = (char*)s2;
for (size_t i = 0; i < size; i++) {
if (s1ptr[i] != s2ptr[i])
diff++;
}
return diff;
}
[[__hot]]
void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
if (__unlikely(n == 0))
return dest;
size_t d = (size_t)dest;
size_t s = (size_t)src;
if (d % sizeof(size_t) != 0 || s % sizeof(size_t) != 0) {
while (n && (d % sizeof(size_t) != 0) && (s % sizeof(size_t) != 0)) {
*((uint8_t*)d) = *((uint8_t*)s);
d++;
s++;
n--;
}
}
size_t* qword_dest = (size_t*)d;
const size_t* qword_src = (const size_t*)s;
const size_t word_count = n / sizeof(size_t);
for (size_t i = 0; i < word_count; i++) {
qword_dest[i] = qword_src[i];
}
size_t remaining_bytes = n % sizeof(size_t);
d = (size_t)(qword_dest + word_count);
s = (size_t)(qword_src + word_count);
while (remaining_bytes--) {
*((uint8_t*)d) = *((uint8_t*)s);
d++;
s++;
}
return dest;
}
[[__hot]]
void* memmove(void* dstptr, const void* srcptr, size_t size) {
uint8_t* dst = (uint8_t*)dstptr;
const uint8_t* src = (const uint8_t*)srcptr;
if (dst < src) {
for (size_t i = 0; i < size; i++)
dst[i] = src[i];
} else {
for (size_t i = size; i != 0; i--)
dst[i - 1] = src[i - 1];
}
return dstptr;
}
[[__hot]]
void* memset(void* dest, int value, size_t n) {
if (__unlikely(n == 0))
return dest;
size_t d = (size_t)dest;
if (d % sizeof(size_t) != 0) {
while (n && (d % sizeof(size_t) != 0)) {
*((uint8_t*)d) = value;
d++;
n--;
}
}
size_t* qword_dest = (size_t*)d;
size_t qword_value = value;
qword_value |= (qword_value << 8);
qword_value |= (qword_value << 16);
qword_value |= (qword_value << 32);
const size_t word_count = n / sizeof(size_t);
for (size_t i = 0; i < word_count; i++) {
qword_dest[i] = qword_value;
}
size_t remaining_bytes = n % sizeof(size_t);
d = (size_t)(qword_dest + word_count);
while (remaining_bytes--) {
*((uint8_t*)d) = value;
d++;
}
return dest;
}
char* strncpy(char* restrict dst, const char* restrict src, size_t len) {
size_t src_len = strnlen(src, len) + 1;
return memcpy(dst, src, MIN(len, src_len));
}
int strncmp(const char* str1, const char* str2, size_t len) {
while (len && *str1 && (*str1 == *str2)) {
++str1;
++str2;
--len;
}
if (len == 0) {
return 0;
} else {
return (*(unsigned char*)str1 - *(unsigned char*)str2);
}
}
int strcmp(const char* str1, const char* str2) {
while (*str1 && (*str1 == *str2)) {
++str1;
++str2;
}
return (*(unsigned char*)str1 - *(unsigned char*)str2);
}
char* strchr(const char* s, int c) {
char ch = (char)c;
while (true) {
char cur = *s;
if (cur == ch)
return (char*)s;
if (cur == 0)
return nullptr;
s++;
}
}
kernel/mem/init.c
+166
View File
@@ -0,0 +1,166 @@
#include <common/utils.h>
#include <kernel/alloc.h>
#include <kernel/assert.h>
#include <kernel/init.h>
#include <kernel/panic.h>
#include <kernel/print.h>
#include <kernel/vas.h>
#include <kernel/virt.h>
#include <string.h>
extern uint8_t __ld_text_start[];
extern uint8_t __ld_text_end[];
extern uint8_t __ld_rodata_start[];
extern uint8_t __ld_rodata_end[];
extern uint8_t __ld_data_start[];
extern uint8_t __ld_data_end[];
extern uint8_t __ld_kernel_start[];
struct vas kernel_vas = {0};
[[__init]]
void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) {
// This function creates a kernel page table and initializes all memory managers.
const size_t pgsz = arch_mem_page_size();
kprintf("Memory map:\n");
for (size_t i = 0; i < map_len; i++) {
if (map[i].length == 0)
continue;
const char* label = nullptr;
switch (map[i].usage) {
case PHYS_RESERVED:
label = "Reserved";
break;
case PHYS_USABLE:
label = "Usable";
break;
case PHYS_STATIC:
label = "Static";
break;
case PHYS_RECLAIMABLE:
label = "Reclaimable";
break;
}
kprintf("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
}
// Set up the bootstrap allocator.
struct phys_mem* largest = map;
for (size_t i = 0; i < map_len; i++) {
if (map[i].length > largest->length && map[i].usage == PHYS_USABLE)
largest = &map[i];
}
mem_phys_bootstrap(largest);
kprintf(
"Using region [%p - %p] for bootstrap allocator\n",
(void*)largest->address,
(void*)(largest->address + largest->length - 1)
);
// Set the HHDM base address to the address given by the loader.
// We must not keep any virtual addresses to this region,
// since we're likely going to map it at a different base address.
mem_hhdm_base = tmp_hhdm;
ASSERT(pmap_new_kernel(&kernel_vas.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
// text
kprintf("Mapping text segment at %p\n", __ld_text_start);
for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
zn_status_t status = pmap_map(
&kernel_vas.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ | PTE_EXEC,
CACHE_NONE
);
ASSERT(!status, "Failed to map %p with error %i\n", p, status);
}
// rodata
kprintf("Mapping rodata segment at %p\n", __ld_rodata_start);
for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
zn_status_t status = pmap_map(
&kernel_vas.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ,
CACHE_NONE
);
ASSERT(!status, "Failed to map %p with error %i\n", p, status);
}
// data
kprintf("Mapping data segment at %p\n", __ld_data_start);
for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
zn_status_t status = pmap_map(
&kernel_vas.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ | PTE_WRITE,
CACHE_NONE
);
ASSERT(!status, "Failed to map %p with error %i\n", p, status);
}
// The kernel address space is divided into 3 segments (plus kernel).
// On x86_64 for example, they live at:
// HHDM FFFF'8000'0000'0000
// PFNDB FFFF'A000'0000'0000
// Mappings FFFF'C000'0000'0000
// Map all physical memory to the HHDM address.
tmp_hhdm = arch_mem_hhdm_addr();
for (size_t i = 0; i < map_len; i++) {
if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
continue;
for (size_t p = 0; p <= map[i].length; p += pgsz) {
uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
phys_t paddr = (phys_t)(map[i].address + p);
zn_status_t status = pmap_map(&kernel_vas.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_NONE);
ASSERT(!status, "Failed to map HHDM page %p to %p with error %i\n", (void*)vaddr, (void*)paddr, status);
}
}
mem_hhdm_base = tmp_hhdm;
// Switch to our own page table.
pmap_set(&kernel_vas.pmap);
// We record metadata for every single page of available memory in a large array.
// This array is contiguous in virtual memory, but is sparsely populated.
// Only those array entries which represent usable memory are mapped.
for (size_t i = 0; i < map_len; i++) {
if (map[i].length == 0 || map[i].usage != PHYS_USABLE)
continue;
const size_t num_pages = (map[i].length + pgsz - 1) / pgsz;
const size_t metadata_size = num_pages * sizeof(struct page);
const uintptr_t metadata_start_vaddr = arch_vm_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size;
// Find the page-aligned boundaries that contain this metadata.
const uintptr_t aligned_start = ALIGN_DOWN(metadata_start_vaddr, pgsz);
const uintptr_t aligned_end = ALIGN_UP(metadata_end_vaddr, pgsz);
const size_t length = aligned_end - aligned_start;
const uintptr_t vaddr = aligned_start;
for (size_t page = 0; page < length; page += pgsz) {
phys_t paddr;
ASSERT(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!\n");
pmap_map(&kernel_vas.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_NONE);
}
}
vm_pfndb = (struct page*)arch_vm_pfndb_addr();
// We don't need the bootstrap allocator from this point on.
// Initialize the real page allocator.
mem_phys_init(map, map_len);
slab_init();
kprintf("Memory initialization complete\n");
}
kernel/mem/meson.build
+11
View File
@@ -0,0 +1,11 @@
kernel_sources += files(
'init.c',
'mmio.c',
'pmap.c',
'pmm.c',
'slab.c',
'usercopy.c',
'virt.c',
'vas.c',
'vmo.c',
)
kernel/mem/mmio.c
+22
View File
@@ -0,0 +1,22 @@
#include <kernel/mmio.h>
#include <kernel/pmap.h>
#include <kernel/vas.h>
#include <kernel/virt.h>
#include <bits/mem.h>
void* mmio_new(phys_t addr, size_t length) {
void* ptr = vm_alloc(length);
for (size_t i = 0; i < length; i += arch_mem_page_size()) {
if (pmap_map(&kernel_vas.pmap, (uintptr_t)ptr + i, addr + i, PTE_READ | PTE_WRITE, CACHE_MMIO)) {
vm_free(ptr, length);
return nullptr;
}
}
return ptr;
}
void mmio_free(void* ptr, size_t length) {
// TODO
}
kernel/mem/pmap.c
+104
View File
@@ -0,0 +1,104 @@
#include <zinnia/status.h>
#include <kernel/alloc.h>
#include <kernel/pmap.h>
#include <kernel/spin.h>
#include <kernel/vas.h>
#include <string.h>
zn_status_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
flags &= ~ALLOC_NOZERO;
phys_t addr;
zn_status_t status = mem_phys_alloc(1, flags, &addr);
if (status)
return status;
struct pmap result = {
.root = addr,
.lock = (struct spinlock){0},
.is_user = false,
};
*pt = result;
return ZN_OK;
}
zn_status_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
phys_t user_l1;
zn_status_t status = mem_phys_alloc(1, 0, &user_l1);
if (status)
return status;
void* user_l1_ptr = HHDM_PTR(user_l1);
void* kernel_l1_ptr = HHDM_PTR(kernel_vas.pmap.root);
memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
struct pmap result = {
.root = user_l1,
.lock = (struct spinlock){0},
.is_user = true,
};
*pt = result;
return ZN_OK;
}
// Gets a reference to the PTE at the given virtual address.
// If `check_only` is set, only checks if the PTE exists,
// and doesn't allocate new levels if they don't already exist.
// If it can't allocate a page if it has to, returns `nullptr`.
static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) {
pte_t* current_head = HHDM_PTR(pt->root);
size_t index = 0;
for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) {
const size_t addr_mask = (1 << arch_mem_level_bits()) - 1;
const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level);
const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0);
index = (vaddr >> addr_shift) & addr_mask;
// The last level is used to access the actual PTE, so break the loop then.
// We still need to update the index beforehand, that's why we can't just end early.
if (level == 0)
break;
pte_t* pte = &current_head[index];
if (pte_is_present(pte)) {
// Get the next level.
*pte = pte_build(pte_address(pte), level_flags, CACHE_NONE);
current_head = HHDM_PTR(pte_address(pte));
} else {
// If the current level isn't present, we can skip the rest.
if (check_only)
return nullptr;
phys_t addr;
if (mem_phys_alloc(1, 0, &addr) != ZN_OK)
return nullptr;
*pte = pte_build(addr, level_flags, CACHE_NONE);
current_head = HHDM_PTR(addr);
}
}
return &current_head[index];
}
zn_status_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
spin_lock(&pt->lock);
zn_status_t status = ZN_OK;
pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
if (pte == nullptr) {
status = ZN_ERR_NO_MEMORY;
goto fail;
}
*pte = pte_build(paddr, flags, cache);
fail:
spin_unlock(&pt->lock);
return status;
}
kernel/mem/pmm.c
+154
View File
@@ -0,0 +1,154 @@
#include <common/compiler.h>
#include <kernel/alloc.h>
#include <kernel/panic.h>
#include <kernel/print.h>
#include <kernel/spin.h>
#include <kernel/virt.h>
#include <stdint.h>
#include <string.h>
uintptr_t mem_hhdm_base = 0;
static zn_status_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
static zn_status_t (*free_fn)(phys_t start, size_t num_pages);
zn_status_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
return alloc_fn(num_pages, flags, out);
}
zn_status_t mem_phys_free(phys_t start, size_t num_pages) {
return free_fn(start, num_pages);
}
static struct phys_mem bump_mem = {0};
static struct phys_mem* bump_region = nullptr;
static zn_status_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
const size_t bytes = num_pages * arch_mem_page_size();
if (((intptr_t)bump_mem.length - bytes) <= 0)
return ZN_ERR_NO_MEMORY;
*out = bump_mem.address;
bump_mem.length -= bytes;
bump_mem.address += bytes;
if (!(flags & ALLOC_NOZERO)) {
memset(HHDM_PTR(*out), 0, bytes);
}
return 0;
}
static zn_status_t bump_free(phys_t addr, size_t num_pages) {
// We don't free bootstrap memory.
panic("Attempted to free bootstrap memory!");
}
void mem_phys_bootstrap(struct phys_mem* mem) {
alloc_fn = bump_alloc;
free_fn = bump_free;
// Save the region by reference and value.
// We can't directly modify the region by reference, because that
// interferes with e.g. HHDM mapping logic.
bump_mem = *mem;
bump_region = mem;
}
static struct page* pmm_head = nullptr;
static struct spinlock pmm_lock = {0};
static size_t pmm_total_free = 0;
static zn_status_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
spin_lock(&pmm_lock);
const size_t bytes = num_pages * arch_mem_page_size();
phys_t limit = UINTPTR_MAX;
if (flags & ALLOC_MEM20) {
limit = (phys_t)1 << 20;
} else if (flags & ALLOC_MEM32) {
limit = (phys_t)1 << 32;
}
struct page* it = pmm_head;
struct page* prev_it = nullptr;
while (it) {
const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
// If the region is too high up or doesn't have enough pages left, continue searching.
if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
prev_it = it;
it = it->freelist.next;
continue;
}
// If the current block has exactly enough pages left, consume it entirely.
if (__unlikely(it->freelist.count == num_pages)) {
if (prev_it) {
prev_it->freelist.next = it->freelist.next;
} else {
pmm_head = it->freelist.next;
}
it->freelist.next = nullptr;
it->freelist.count = 0;
if (__likely(out))
*out = page_addr;
}
// If not, we can just shrink the block.
else {
it->freelist.count -= num_pages;
if (__likely(out))
*out = page_addr + (it->freelist.count * arch_mem_page_size());
}
goto success;
}
spin_unlock(&pmm_lock);
return ZN_ERR_NO_MEMORY;
success:
// If the NOZERO flag is *not* specified, zero memory.
if (__likely(out) && __likely(!(flags & ALLOC_NOZERO))) {
memset(HHDM_PTR(*out), 0, bytes);
}
pmm_total_free -= num_pages;
spin_unlock(&pmm_lock);
return 0;
}
static zn_status_t freelist_free(phys_t addr, size_t num_pages) {
spin_lock(&pmm_lock);
struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
page->freelist.count = num_pages;
page->freelist.next = pmm_head;
pmm_head = page;
spin_unlock(&pmm_lock);
return 0;
}
void mem_phys_init(struct phys_mem* map, size_t length) {
alloc_fn = freelist_alloc;
free_fn = freelist_free;
bump_region->address = bump_mem.address;
bump_region->length = bump_mem.length;
for (size_t i = 0; i < length; i++) {
// Regions smaller than a page are useless.
if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
continue;
struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
page->freelist.count = map[i].length / arch_mem_page_size();
page->freelist.next = pmm_head;
pmm_head = page;
pmm_total_free += page->freelist.count;
}
const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
kprintf("Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
}
kernel/mem/slab.c
+143
View File
@@ -0,0 +1,143 @@
#include <zinnia/status.h>
#include <common/compiler.h>
#include <kernel/alloc.h>
#include <kernel/spin.h>
#include <string.h>
typedef struct {
size_t num_pages; // Amount of pages connected to this slab.
size_t size; // Size of this slab.
} slab_info;
struct slab {
struct spinlock lock; // Access lock.
size_t ent_size; // Size of one entry.
void** head;
};
typedef struct {
struct slab* slab;
} slab_header;
// Initializes the SLAB structures.
static struct slab slabs[8] = {0};
static zn_status_t slab_new(struct slab* slab, size_t size) {
slab->lock = (struct spinlock){0};
// Allocate a new page for the head.
phys_t addr;
zn_status_t status = mem_phys_alloc(1, 0, &addr);
if (status)
return status;
slab->head = (void**)(addr + arch_mem_hhdm_addr());
slab->ent_size = size;
const size_t offset = ALIGN_UP(sizeof(slab_header), size);
const size_t available_size = arch_mem_page_size() - offset;
slab_header* ptr = (slab_header*)slab->head;
ptr->slab = slab;
slab->head = (void**)((void*)slab->head + offset);
void** arr = slab->head;
const size_t max = available_size / size - 1;
const size_t fact = size / sizeof(void*);
for (size_t i = 0; i < max; i++) {
arr[i * fact] = &arr[(i + 1) * fact];
}
arr[max * fact] = nullptr;
return ZN_OK;
}
void slab_init(void) {
slab_new(&slabs[0], 16);
slab_new(&slabs[1], 32);
slab_new(&slabs[2], 64);
slab_new(&slabs[3], 128);
slab_new(&slabs[4], 256);
slab_new(&slabs[5], 512);
slab_new(&slabs[6], 1024);
slab_new(&slabs[7], 2048);
}
static void* slab_do_alloc(struct slab* slab) {
spin_lock(&slab->lock);
if (__unlikely(slab->head == nullptr))
slab_new(slab, slab->ent_size);
void** old_free = slab->head;
slab->head = *old_free;
memset(old_free, 0, slab->ent_size);
spin_unlock(&slab->lock);
return old_free;
}
static void slab_do_free(struct slab* slab, void* addr) {
spin_lock(&slab->lock);
if (__unlikely(addr == nullptr))
goto cleanup;
void** new_head = addr;
*new_head = slab->head;
slab->head = new_head;
cleanup:
spin_unlock(&slab->lock);
}
static inline struct slab* slab_find_size(size_t size) {
for (size_t i = 0; i < ARRAY_SIZE(slabs); i++) {
if (slabs[i].ent_size >= size)
return &slabs[i];
}
return nullptr;
}
void* mem_alloc(size_t size, enum alloc_flags flags) {
if (__unlikely(size == 0))
return nullptr;
struct slab* slab = slab_find_size(size);
if (slab != nullptr) {
return slab_do_alloc(slab);
}
size_t num_pages = ROUND_UP(size, arch_mem_page_size());
// Allocate the pages plus an additional page for metadata.
phys_t ret;
zn_status_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
if (__unlikely(status != ZN_OK))
return nullptr;
ret = ret + (phys_t)arch_mem_hhdm_addr();
// Write metadata into the first page.
slab_info* info = (slab_info*)ret;
info->num_pages = num_pages;
info->size = size;
// Skip the first page and return the next one.
return (void*)(ret + arch_mem_page_size());
}
void mem_free(void* addr) {
if (__unlikely(addr == nullptr))
return;
// If the address is page aligned.
if ((size_t)addr == ALIGN_DOWN((size_t)addr, arch_mem_page_size())) {
slab_info* info = (slab_info*)(addr - arch_mem_page_size());
mem_phys_free(((phys_t)info - (phys_t)arch_mem_hhdm_addr()), info->num_pages + 1);
return;
}
slab_header* header = (slab_header*)(ALIGN_DOWN((size_t)addr, arch_mem_page_size()));
slab_do_free(header->slab, addr);
}

Some files were not shown because too many files have changed in this diff Show More