commit e71245c08261200f87733fe5a0c374c6d779e832 Author: Marvin Friedrich Date: Mon Jul 20 01:53:04 2026 +0200 test diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..37c7f3c --- /dev/null +++ b/.clang-format @@ -0,0 +1,66 @@ +--- +AlignAfterOpenBracket: BlockIndent +BinPackArguments: false +BinPackParameters: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowAllArgumentsOnNextLine: false +BreakAfterReturnType: Automatic +PenaltyReturnTypeOnItsOwnLine: 1000 +AlignArrayOfStructures: None +IndentWidth: 4 +InsertNewlineAtEOF: true +IntegerLiteralSeparator: + HexMinDigits: 8 +IndentCaseBlocks: false +IndentCaseLabels: false +IndentPPDirectives: None +IndentGotoLabels: false +TabWidth: 4 +Standard: Latest +Language: Cpp +LineEnding: LF +AlignOperands: DontAlign +BitFieldColonSpacing: None +ColumnLimit: 120 +MaxEmptyLinesToKeep: 1 +PointerAlignment: Left +QualifierAlignment: Leave +SpaceAfterLogicalNot: false +SpaceAfterCStyleCast: false +AllowShortEnumsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortBlocksOnASingleLine: Empty +AlwaysBreakBeforeMultilineStrings: true +BreakAfterAttributes: Always +BreakConstructorInitializers: BeforeComma +BreakBeforeBraces: Attach +AllowShortLoopsOnASingleLine: false +SpacesInSquareBrackets: false +AlignConsecutiveMacros: + Enabled: true + AcrossEmptyLines: false + AcrossComments: true + AlignCompound: true + AlignFunctionPointers: false +AlignEscapedNewlines: DontAlign +BreakBeforeInlineASMColon: OnlyMultiline +AccessModifierOffset: -4 +SpacesInParens: Never +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpacesBeforeTrailingComments: 1 +IncludeBlocks: Merge +IncludeCategories: +- Regex: "^" + Priority: 1 +- Regex: "^" + Priority: 2 +- Regex: "^" + Priority: 3 +- Regex: "^" + Priority: 4 +- Regex: '^<.*\.h>' + Priority: 5 +- Regex: '^".*\.h"' + Priority: 6 +SortIncludes: CaseInsensitive diff --git a/.clangd b/.clangd new file mode 100644 index 0000000..67ae5cb --- /dev/null +++ b/.clangd @@ -0,0 +1,2 @@ +Style: + AngledHeaders: [ "zinnia/.*", "uapi/.*" ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1301351 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +build/ +.cache/ +.wraplock +*.o +*.a +*.so diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..9057c85 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "subprojects/uacpi"] + path = subprojects/uacpi + url = https://github.com/uACPI/uacpi diff --git a/README.md b/README.md new file mode 100644 index 0000000..36ec603 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ + + +# Zinnia + +![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=green) +![GitHub Repo stars](https://img.shields.io/github/stars/zinnia-os/zinnia?style=flat) +![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/zinnia-os/zinnia/ci.yml) +![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/zinnia-os/zinnia?style=flat) +![Discord](https://img.shields.io/discord/1305893510704267327?logo=discord&logoColor=white&label=discord) + +Zinnia is a lightweight BSD-like monolithic kernel targeting 64-bit devices. + +## Building Zinnia + +To build the kernel you need `meson` and a GNU-like C compiler with support for C23. + +To configure, run: +```sh +meson setup $build_dir +``` + +And to build: +```sh +meson compile -C $build_dir +``` diff --git a/arch/x86_64/apic.c b/arch/x86_64/apic.c new file mode 100644 index 0000000..450e5b8 --- /dev/null +++ b/arch/x86_64/apic.c @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +static uint64_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) { + if (lapic->xapic_regs.base) { + if (reg.offset == APIC_REG_ICR.offset) { + uint64_t lo, hi; + lo = mmio_load(&lapic->xapic_regs, APIC_REG_ICR); + hi = mmio_load(&lapic->xapic_regs, APIC_REG_ICR_HI); + return (hi << 32) | lo; + } else { + return mmio_load(&lapic->xapic_regs, reg); + } + } else { + return asm_rdmsr(0x800 + (reg.offset >> 4)); + } +} + +static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint64_t value) { + if (lapic->xapic_regs.base) { + if (reg.offset == APIC_REG_ICR.offset) { + mmio_store(&lapic->xapic_regs, APIC_REG_ICR_HI, value >> 32); + mmio_store(&lapic->xapic_regs, APIC_REG_ICR, (uint32_t)value); + } else { + mmio_store(&lapic->xapic_regs, reg, (uint32_t)value); + } + } else { + asm_wrmsr(0x800 + (reg.offset >> 4), value); + } +} + +void lapic_init(struct local_apic* lapic) { + uint64_t apic_msr = asm_rdmsr(0x1B); + apic_msr |= 1 << 11; // Enable APIC. + + struct cpuid cpuid = asm_cpuid(1, 0); + if (cpuid.ecx & CPUID_1C_X2APIC) + apic_msr |= 1 << 10; + else { + lapic->xapic_regs = mmio_new(apic_msr & 0xFFFF'F000, 0x1000); + assert(lapic->xapic_regs.base, "Failed to allocate virtual memory!\n"); + } + + asm_wrmsr(0x1B, apic_msr); + + // Reset the TPR. + lapic_write_reg(lapic, APIC_REG_TPR, 0); + // Enable APIC bit in the SIVR. + lapic_write_reg(lapic, APIC_REG_SIVR, lapic_read_reg(lapic, APIC_REG_SIVR) | 0x100); + + if (lapic->xapic_regs.base) { + lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000); + // Logical destination = LAPIC ID. + lapic_write_reg(lapic, APIC_REG_LDR, lapic_read_reg(lapic, APIC_REG_ID)); + } + + // TODO: Parse MADT and setup NMI sources. + + // Tell the APIC timer to divide by 16. + lapic_write_reg(lapic, APIC_REG_DCR, 3); + // Set the timer to the highest value possible. + lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0xFFFF'FFFF); + + // Sleep for 10 milliseconds. + clock_spin_ns(10'000'000); + + lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR); + lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000); + lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms); +} + +void lapic_eoi(struct local_apic* lapic) { + lapic_write_reg(lapic, APIC_REG_EOI, 0); +} + +#define PIC1_COMMAND_PORT 0x20 +#define PIC1_DATA_PORT 0x21 +#define PIC2_COMMAND_PORT 0xA0 +#define PIC2_DATA_PORT 0xA1 + +void pic_disable() { + // We initialize the PIC properly, but completely disable it and use the APIC in favor of it. + // Remap IRQs so they start at 0x20 since interrupts 0x00..0x1F are used by CPU exceptions. + asm_outb(PIC1_COMMAND_PORT, 0x11); + asm_outb(PIC1_DATA_PORT, 0x20); + asm_outb(PIC1_DATA_PORT, 0x04); + asm_outb(PIC1_DATA_PORT, 0x01); + asm_outb(PIC1_DATA_PORT, 0xFF); + + // Same for the slave PIC. + asm_outb(PIC2_COMMAND_PORT, 0x11); + asm_outb(PIC2_DATA_PORT, 0x28); + asm_outb(PIC2_DATA_PORT, 0x02); + asm_outb(PIC2_DATA_PORT, 0x01); + asm_outb(PIC2_DATA_PORT, 0xFF); +} diff --git a/arch/x86_64/archctl.c b/arch/x86_64/archctl.c new file mode 100644 index 0000000..1f9e8a5 --- /dev/null +++ b/arch/x86_64/archctl.c @@ -0,0 +1,18 @@ +#include +#include +#include +#include + +errno_t arch_archctl(uintptr_t op, uintptr_t arg) { + switch (op) { + case __ARCHCTL_SET_FSBASE: + asm_wrmsr(MSR_FS_BASE, arg); + return 0; + case __ARCHCTL_SET_GSBASE: + // Update the inactive GSBASE MSR, not ours. + asm_wrmsr(MSR_KERNEL_GS_BASE, arg); + return 0; + default: + return EINVAL; + } +} diff --git a/arch/x86_64/gdt.c b/arch/x86_64/gdt.c new file mode 100644 index 0000000..78e4371 --- /dev/null +++ b/arch/x86_64/gdt.c @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include +#include +#include + +static const struct gdt main_gdt = { + .null = {.val = 0}, + .kernel_code32 = {.val = 0x00cf9b000000ffff}, + .kernel_data32 = {.val = 0x00cf93000000ffff}, + .kernel_code64 = {.val = 0x00a09b0000000000}, + .kernel_data64 = {.val = 0x0000930000000000}, + .user_code32 = {.val = 0x00cffb000000ffff}, + .user_data = {.val = 0x0000f30000000000}, + .user_code64 = {.val = 0x00a0fb0000000000}, + .tss = { + .limit0 = sizeof(struct tss) - 1, + .access = 0x89, // Present | DPL=0 | System | 64-bit TSS + .limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F, + .reserved = 0, + }, +}; + +void gdt_load() { + struct gdt* gdt = &percpu_get()->arch.gdt; + memcpy(gdt, &main_gdt, sizeof(main_gdt)); + + struct tss* tss = &percpu_get()->arch.tss; + tss->iopb = sizeof(struct tss); + + uintptr_t tss_addr = (uintptr_t)tss; + gdt->tss.base0 = tss_addr; + gdt->tss.base1 = tss_addr >> 16; + gdt->tss.base2 = tss_addr >> 24; + gdt->tss.base3 = tss_addr >> 32; + + struct gdtr gdtr = { + .limit = sizeof(struct gdt) - 1, + .base = gdt, + }; + + asm volatile("lgdt %0" ::"m"(gdtr)); + + // Save the contents of MSR_GS_BASE, as they get cleared by a write to `gs`. + uintptr_t gs = asm_rdmsr(MSR_GS_BASE); + + // Flush and reload the segment registers. + asm volatile( + "push %0\n" + "lea rax, [rip + 1f]\n" + "push rax\n" + "retfq\n" + "1:\n" + "mov ax, %1\n" + "mov ds, ax\n" + "mov es, ax\n" + "mov fs, ax\n" + "mov gs, ax\n" + "mov ss, ax\n" + : + : "i"(offsetof(struct gdt, kernel_code64)), "i"(offsetof(struct gdt, kernel_data64)) + : "rax", "memory" + ); + + asm_wrmsr(MSR_GS_BASE, gs); + + asm volatile( + "mov ax, %c0\n" + "ltr ax" + : + : "i"(offsetof(struct gdt, tss)) + : "rax", "memory" + ); +} diff --git a/arch/x86_64/hpet.c b/arch/x86_64/hpet.c new file mode 100644 index 0000000..915b5af --- /dev/null +++ b/arch/x86_64/hpet.c @@ -0,0 +1,69 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool use_hpet = true; +static void hpet_option(bool use) { + use_hpet = use; +} +CMDLINE_OPTION("hpet", hpet_option); + +struct hpet { + struct clock clock; + struct mmio_view regs; + uint32_t period; +}; + +constexpr mmio_reg32 HPET_CAP = {.offset = 0}; +constexpr mmio_reg32 HPET_CFG = {.offset = 0x10}; +constexpr mmio_reg32 HPET_COUNTER = {.offset = 0xF0}; + +static void hpet_reset(struct clock* c) { + struct hpet* hpet = containerof(c, struct hpet, clock); + mmio_store(&hpet->regs, HPET_COUNTER, 0u); +} + +static uint64_t hpet_get_elapsed_ns(struct clock* c) { + struct hpet* hpet = containerof(c, struct hpet, clock); + uint64_t counter = mmio_load(&hpet->regs, HPET_COUNTER); + return (counter * (uint64_t)hpet->period / 1'000'000); +} + +static struct hpet hpet_clock = { + .clock = { + .name = "hpet", + .priority = 128, + .reset = hpet_reset, + .get_elapsed_ns = hpet_get_elapsed_ns, + }, +}; + +static char table_buf[0x1000]; + +void hpet_init() { + if (!use_hpet) + return; + + uacpi_setup_early_table_access(table_buf, sizeof(table_buf)); + uacpi_table table = {0}; + assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n"); + + struct acpi_hpet* hpet = table.ptr; + hpet_clock.regs = mmio_new(hpet->address.address, 0x1000); + uacpi_table_unref(&table); + printf("HPET regs at %p\n", hpet_clock.regs.base); + + mmio_store(&hpet_clock.regs, HPET_CFG, mmio_load(&hpet_clock.regs, HPET_CFG) | 1); + + uint64_t caps = mmio_load(&hpet_clock.regs, HPET_CAP); + hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF; + + assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n"); +} diff --git a/arch/x86_64/idt.c b/arch/x86_64/idt.c new file mode 100644 index 0000000..a72b81c --- /dev/null +++ b/arch/x86_64/idt.c @@ -0,0 +1,209 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct [[__packed]] idt_entry { + uint16_t base0; + uint16_t selector; + uint8_t ist; + uint8_t attrib; + uint16_t base1; + uint32_t base2; + uint32_t reserved; +}; + +static_assert(sizeof(struct idt_entry) == 16); + +struct [[__packed]] idtr { + uint16_t limit; + struct idt_entry* base; +}; + +static_assert(sizeof(struct idtr) == 10); + +void interrupt_handler(struct arch_context* ctx) { + struct percpu* cpu = percpu_get(); + bool old = irq_set_interrupted(true); + + uint64_t isr = ctx->isr; + switch (isr) { + case IDT_PF: + uint64_t cr2; + asm volatile("mov %0, cr2" : "=r"(cr2)); + panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2); + case IDT_IPI_RESCHED: + arch_sched_preempt_disable(); + if (arch_sched_preempt_enable()) { + lapic_eoi(&cpu->arch.lapic); + sched_reschedule(&cpu->sched); + } + break; + case 0x20 ...(0x20 + ARRAY_SIZE(cpu->arch.irq_lines)): + struct irq_line* line = cpu->arch.irq_lines[isr - 0x20]; + assert(line, "Unhandled interrupt on ISR %lu\n", isr); + // TODO: irq_raise(line); + break; + default: + panic("Unhandled exception on ISR %lu\n", isr); + } + + irq_set_interrupted(old); +} + +#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs)) + +[[__naked]] +void interrupt_return() { + asm volatile( + "pop r15\n" + "pop r14\n" + "pop r13\n" + "pop r12\n" + "pop r11\n" + "pop r10\n" + "pop r9\n" + "pop r8\n" + "pop rsi\n" + "pop rdi\n" + "pop rbp\n" + "pop rdx\n" + "pop rcx\n" + "pop rbx\n" + "pop rax\n" + // Change GS back if we came from user mode. + "cmp word ptr [rsp + %c0], %c1\n" + "je 2f\n" + "swapgs\n" + "2:\n" + // Skip .error and .isr fields. + "add rsp, 0x10\n" + "iretq\n" + : + : "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64)) + ); +} + +[[__naked]] +void interrupt_stub_internal() { + asm volatile( + // Load the kernel GS base if we're coming from user space. + "cmp word ptr [rsp + %c0], %c1\n" + "je 2f\n" + "swapgs\n" + "2:\n" + "push rax\n" + "push rbx\n" + "push rcx\n" + "push rdx\n" + "push rbp\n" + "push rdi\n" + "push rsi\n" + "push r8\n" + "push r9\n" + "push r10\n" + "push r11\n" + "push r12\n" + "push r13\n" + "push r14\n" + "push r15\n" + "cld\n" + // Zero out the base pointer since we can't trust it. + "xor rbp, rbp\n" + // Load the frame as first argument. + "mov rdi, rsp\n" + "call %c2\n" + "jmp %c3\n" + : + : "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64)), "i"(interrupt_handler), "i"(interrupt_return) + ); +} + +// clang-format off +#define REPEAT_256(M) \ + M(0) M(1) M(2) M(3) M(4) M(5) M(6) M(7) \ + M(8) M(9) M(10) M(11) M(12) M(13) M(14) M(15) \ + M(16) M(17) M(18) M(19) M(20) M(21) M(22) M(23) \ + M(24) M(25) M(26) M(27) M(28) M(29) M(30) M(31) \ + M(32) M(33) M(34) M(35) M(36) M(37) M(38) M(39) \ + M(40) M(41) M(42) M(43) M(44) M(45) M(46) M(47) \ + M(48) M(49) M(50) M(51) M(52) M(53) M(54) M(55) \ + M(56) M(57) M(58) M(59) M(60) M(61) M(62) M(63) \ + M(64) M(65) M(66) M(67) M(68) M(69) M(70) M(71) \ + M(72) M(73) M(74) M(75) M(76) M(77) M(78) M(79) \ + M(80) M(81) M(82) M(83) M(84) M(85) M(86) M(87) \ + M(88) M(89) M(90) M(91) M(92) M(93) M(94) M(95) \ + M(96) M(97) M(98) M(99) M(100) M(101) M(102) M(103) \ + M(104) M(105) M(106) M(107) M(108) M(109) M(110) M(111) \ + M(112) M(113) M(114) M(115) M(116) M(117) M(118) M(119) \ + M(120) M(121) M(122) M(123) M(124) M(125) M(126) M(127) \ + M(128) M(129) M(130) M(131) M(132) M(133) M(134) M(135) \ + M(136) M(137) M(138) M(139) M(140) M(141) M(142) M(143) \ + M(144) M(145) M(146) M(147) M(148) M(149) M(150) M(151) \ + M(152) M(153) M(154) M(155) M(156) M(157) M(158) M(159) \ + M(160) M(161) M(162) M(163) M(164) M(165) M(166) M(167) \ + M(168) M(169) M(170) M(171) M(172) M(173) M(174) M(175) \ + M(176) M(177) M(178) M(179) M(180) M(181) M(182) M(183) \ + M(184) M(185) M(186) M(187) M(188) M(189) M(190) M(191) \ + M(192) M(193) M(194) M(195) M(196) M(197) M(198) M(199) \ + M(200) M(201) M(202) M(203) M(204) M(205) M(206) M(207) \ + M(208) M(209) M(210) M(211) M(212) M(213) M(214) M(215) \ + M(216) M(217) M(218) M(219) M(220) M(221) M(222) M(223) \ + M(224) M(225) M(226) M(227) M(228) M(229) M(230) M(231) \ + M(232) M(233) M(234) M(235) M(236) M(237) M(238) M(239) \ + M(240) M(241) M(242) M(243) M(244) M(245) M(246) M(247) \ + M(248) M(249) M(250) M(251) M(252) M(253) M(254) M(255) +// clang-format on + +#define INTERRUPT_STUB(n) \ + [[__naked]] \ + static void interrupt_stub_##n() { \ + asm volatile( \ + ".if (%c0 == 8 || (%c0 >= 10 && %c0 <= 14) || %c0 == 17 || %c0 == 21 || %c0 == 29 || %c0 == 30)\n" \ + ".else\n" \ + "push 0\n" \ + ".endif\n" \ + "push %c0\n" \ + "jmp %c1\n" \ + : \ + : "i"(n), "i"(interrupt_stub_internal) \ + ); \ + } + +#define INTERRUPT_SETUP(n) \ + do { \ + uint64_t addr = (uint64_t)interrupt_stub_##n; \ + idt[n].base0 = addr & 0xFFFF; \ + idt[n].base1 = (addr >> 16) & 0xFFFF; \ + idt[n].base2 = (addr >> 32) & 0xFFFF'FFFF; \ + idt[n].ist = 0; \ + idt[n].attrib = (1 << 7) | 0xE; \ + idt[n].selector = offsetof(struct gdt, kernel_code64); \ + idt[n].reserved = 0; \ + } while (0); + +REPEAT_256(INTERRUPT_STUB) + +static struct idt_entry idt[256] = {}; + +void idt_init() { + REPEAT_256(INTERRUPT_SETUP) +} + +void idt_load() { + struct idtr idtr = { + .limit = sizeof(idt) - 1, + .base = idt, + }; + + asm volatile("lidt %0" ::"m"(idtr)); +} diff --git a/arch/x86_64/include/bits/elf.h b/arch/x86_64/include/bits/elf.h new file mode 100644 index 0000000..cdd37cc --- /dev/null +++ b/arch/x86_64/include/bits/elf.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#define ELF_ARCH_CLASS ELFCLASS64 +#define ELF_ARCH_DATA ELFDATA2LSB +#define ELF_ARCH_MACHINE EM_X86_64 + +#define ELF_R_SYM ELF64_R_SYM +#define ELF_R_TYPE ELF64_R_TYPE +#define ELF_ST_BIND ELF64_ST_BIND +#define ELF_ST_TYPE ELF64_ST_TYPE + +#define elf_ehdr elf64_ehdr +#define elf_phdr elf64_phdr +#define elf_dyn elf64_dyn +#define elf_addr elf64_addr +#define elf_off elf64_off +#define elf_nhdr elf64_nhdr +#define elf_auxv elf64_auxv +#define elf_sym elf64_sym +#define elf_rela elf64_rela +#define elf_rel elf64_rel + +enum { + R_NONE = 0, + R_COPY = 5, + R_GLOB_DAT = 6, + R_JUMP_SLOT = 7, + R_RELATIVE = 8, + R_IRELATIVE = 37, +}; diff --git a/arch/x86_64/include/bits/irq.h b/arch/x86_64/include/bits/irq.h new file mode 100644 index 0000000..b80da72 --- /dev/null +++ b/arch/x86_64/include/bits/irq.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +static inline bool arch_irq_get_state() { + uint64_t flags; + asm volatile("pushf; pop %0" : "=r"(flags)::"memory"); + return flags & RFLAGS_IF; +} + +static inline void arch_irq_set_state(bool state) { + if (state) + asm volatile("sti"); + else + asm volatile("cli"); +} + +static inline void arch_irq_wait() { + asm volatile("hlt"); +} diff --git a/arch/x86_64/include/bits/mem.h b/arch/x86_64/include/bits/mem.h new file mode 100644 index 0000000..689c0f7 --- /dev/null +++ b/arch/x86_64/include/bits/mem.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +static inline size_t arch_mem_page_size() { + return 1 << 12; +} + +static inline size_t arch_mem_page_bits() { + return 12; +} + +static inline size_t arch_mem_level_bits() { + return 9; +} + +static inline size_t arch_mem_num_levels() { + return 4; +} + +static inline uintptr_t arch_mem_hhdm_addr() { + return 0xFFFF'8000'0000'0000; +} + +static inline uintptr_t arch_vm_pfndb_addr() { + return 0xFFFF'A000'0000'0000; +} + +static inline uintptr_t arch_mem_mapping_addr() { + return 0xFFFF'C000'0000'0000; +} diff --git a/arch/x86_64/include/bits/mmu.h b/arch/x86_64/include/bits/mmu.h new file mode 100644 index 0000000..4f44129 --- /dev/null +++ b/arch/x86_64/include/bits/mmu.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +typedef uint64_t pte_t; diff --git a/arch/x86_64/include/bits/percpu.h b/arch/x86_64/include/bits/percpu.h new file mode 100644 index 0000000..4f097a6 --- /dev/null +++ b/arch/x86_64/include/bits/percpu.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include + +static inline struct percpu* arch_percpu_get() { + struct percpu* result; + asm volatile("mov %0, gs:0" : "=r"(result)::"memory"); + return result; +} + +struct arch_percpu { + struct gdt gdt; + struct tss tss; + struct local_apic lapic; + struct irq_line* irq_lines[128]; + size_t fpu_size; + void (*fpu_save)(void*); + void (*fpu_restore)(const void*); + bool can_smap; +}; + +void arch_percpu_bsp_init(); +void arch_percpu_init(); diff --git a/arch/x86_64/include/bits/sched.h b/arch/x86_64/include/bits/sched.h new file mode 100644 index 0000000..2772d26 --- /dev/null +++ b/arch/x86_64/include/bits/sched.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +struct arch_task_context { + uint64_t rsp; + uint8_t* fpu_region; + uint16_t ds, es, fs, gs; + uint64_t fs_base, gs_base; +}; + +struct arch_context { + uint64_t r15; + uint64_t r14; + uint64_t r13; + uint64_t r12; + uint64_t r11; + uint64_t r10; + uint64_t r9; + uint64_t r8; + uint64_t rsi; + uint64_t rdi; + uint64_t rbp; + uint64_t rdx; + uint64_t rcx; + uint64_t rbx; + uint64_t rax; + uint64_t isr; // Pushed onto the stack by the interrupt handler stubs. + uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code. + uint64_t rip; // The rest is pushed onto the stack by the CPU during an interrupt. + uint64_t cs; + uint64_t rflags; + uint64_t rsp; + uint64_t ss; +}; + +#define ARCH_CTX_NUM rax +#define ARCH_CTX_A0 rdi +#define ARCH_CTX_A1 rsi +#define ARCH_CTX_A2 rdx +#define ARCH_CTX_A3 r9 +#define ARCH_CTX_A4 r8 +#define ARCH_CTX_A5 r10 +#define ARCH_CTX_A6 r12 +#define ARCH_CTX_A7 r13 +#define ARCH_CTX_RET0 rax +#define ARCH_CTX_RET1 rdx diff --git a/arch/x86_64/include/bits/types.h b/arch/x86_64/include/bits/types.h new file mode 100644 index 0000000..010c952 --- /dev/null +++ b/arch/x86_64/include/bits/types.h @@ -0,0 +1,6 @@ +#pragma once + +#include + +typedef uint64_t phys_t; +typedef uint64_t reg_t; diff --git a/arch/x86_64/include/x86_64/apic.h b/arch/x86_64/include/x86_64/apic.h new file mode 100644 index 0000000..ce4757f --- /dev/null +++ b/arch/x86_64/include/x86_64/apic.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +struct local_apic { + uint32_t id; + uint32_t ticks_per_10ms; + struct mmio_view xapic_regs; +}; + +enum apic_delivery_mode { + APIC_DELIVERY_FIXED = 0b000, + APIC_DELIVERY_LOWESTPRIO = 0b001, + APIC_DELIVERY_SMI = 0b010, + APIC_DELIVERY_NMI = 0b100, + APIC_DELIVERY_INIT = 0b101, + APIC_DELIVERY_STARTUP = 0b110, +}; + +enum apic_dest_mode { + APIC_DEST_PHYSICAL = 0, + APIC_DEST_LOGICAL = 1, +}; + +enum apic_delivery_status { + APIC_DELIVERY_STATUS_IDLE = 0, + APIC_DELIVERY_STATUS_PENDING = 1, +}; + +enum apic_level { + APIC_LEVEL_DEASSERT = 0, + APIC_LEVEL_ASSERT = 1, +}; + +enum apic_trigger_mode { + APIC_TRIGGER_EDGE = 0, + APIC_TRIGGER_LEVEL = 1, +}; + +constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20}; +constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80}; +constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0}; +constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0}; +constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0}; +constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0}; +constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280}; +constexpr mmio_reg32 APIC_REG_ICR = {.offset = 0x300}; +constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310}; +constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320}; +constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380}; +constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390}; +constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0}; + +void lapic_init(struct local_apic* lapic); +void lapic_eoi(struct local_apic* lapic); +void pic_disable(); diff --git a/arch/x86_64/include/x86_64/asm.h b/arch/x86_64/include/x86_64/asm.h new file mode 100644 index 0000000..76ddcde --- /dev/null +++ b/arch/x86_64/include/x86_64/asm.h @@ -0,0 +1,135 @@ +#pragma once + +#include + +struct cpuid { + uint32_t eax, ebx, ecx, edx; +}; + +static inline struct cpuid asm_cpuid(uint32_t leaf, uint32_t subleaf) { + struct cpuid out = {0}; + asm volatile("cpuid" : "=a"(out.eax), "=b"(out.ebx), "=c"(out.ecx), "=d"(out.edx) : "0"(leaf), "2"(subleaf)); + return out; +} + +static inline uint64_t asm_rdmsr(uint32_t msr) { + uint32_t eax; + uint32_t edx; + asm volatile("rdmsr" : "=a"(eax), "=d"(edx) : "c"(msr) : "memory"); + return ((uint64_t)edx << 32) | eax; +} + +static inline uint64_t asm_rdtsc() { + uint32_t eax; + uint32_t edx; + asm volatile("rdtsc" : "=a"(eax), "=d"(edx) : : "memory"); + return ((uint64_t)edx << 32) | eax; +} + +// Writes a 64-bit value to a given MSR. +static inline void asm_wrmsr(uint32_t msr, uint64_t val) { + uint32_t eax = (uint32_t)val; + uint32_t edx = val >> 32; + asm volatile("wrmsr" : : "a"(eax), "d"(edx), "c"(msr) : "memory"); +} + +// Writes a 64-bit value to a control register using XSETBV. +static inline void asm_wrxcr(uint32_t reg, uint64_t val) { + uint32_t eax = val; + uint32_t edx = val >> 32; + asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory"); +} + +// Saves the FPU state to a 512-byte region of memory using FXSAVE. +// Pointer must be 16-byte aligned. +static inline void asm_fxsave(void* mem) { + asm volatile("fxsave [%0]" ::"r"(mem) : "memory"); +} + +// Restores the FPU state from a 512-byte region of memory using FXRSTOR. +// Pointer must be 16-byte aligned. +static inline void asm_fxrstor(const void* mem) { + asm volatile("fxrstor [%0]" ::"r"(mem) : "memory"); +} + +// Saves the FPU state to a region of memory using XSAVE. +// Pointer must be 16-byte aligned. +static inline void asm_xsave(void* mem) { + asm volatile("xsave [%0]" ::"r"(mem) : "memory"); +} + +// Restores the FPU state from a region of memory using XRSTOR. +// Pointer must be 16-byte aligned. +static inline void asm_xrstor(const void* mem) { + asm volatile("xrstor [%0]" ::"r"(mem) : "memory"); +} + +static inline uint8_t asm_inb(uint16_t port) { + uint8_t result; + asm volatile("in %0, %1" : "=a"(result) : "Nd"(port)); + return result; +} + +static inline uint16_t asm_inw(uint16_t port) { + uint16_t result; + asm volatile("in %0, %1" : "=a"(result) : "Nd"(port)); + return result; +} + +static inline uint32_t asm_inl(uint16_t port) { + uint32_t result; + asm volatile("in %0, %1" : "=a"(result) : "Nd"(port)); + return result; +} + +static inline void asm_outb(uint16_t port, uint8_t value) { + asm volatile("out %0, %1" : : "Nd"(port), "a"(value)); +} + +static inline void asm_outw(uint16_t port, uint16_t value) { + asm volatile("out %0, %1" : : "Nd"(port), "a"(value)); +} + +static inline void asm_outl(uint16_t port, uint32_t value) { + asm volatile("out %0, %1" : : "Nd"(port), "a"(value)); +} + +static inline uint16_t asm_read_ds() { + uint16_t value = 0; + asm volatile("mov %0, ds" : "=r"(value)); + return value; +} + +static inline uint16_t asm_read_es() { + uint16_t value = 0; + asm volatile("mov %0, es" : "=r"(value)); + return value; +} + +static inline uint16_t asm_read_fs() { + uint16_t value = 0; + asm volatile("mov %0, fs" : "=r"(value)); + return value; +} + +static inline uint16_t asm_read_gs() { + uint16_t value = 0; + asm volatile("mov %0, gs" : "=r"(value)); + return value; +} + +static inline void asm_write_ds(uint16_t value) { + asm volatile("mov ds, %0" ::"r"(value)); +} + +static inline void asm_write_es(uint16_t value) { + asm volatile("mov es, %0" ::"r"(value)); +} + +static inline void asm_write_fs(uint16_t value) { + asm volatile("mov fs, %0" ::"r"(value)); +} + +static inline void asm_write_gs(uint16_t value) { + asm volatile("mov gs, %0" ::"r"(value)); +} diff --git a/arch/x86_64/include/x86_64/defs.h b/arch/x86_64/include/x86_64/defs.h new file mode 100644 index 0000000..2fb6777 --- /dev/null +++ b/arch/x86_64/include/x86_64/defs.h @@ -0,0 +1,201 @@ +#pragma once + +#define CPUID_1C_SSE3 (1 << 0) +#define CPUID_1C_PCLMUL (1 << 1) +#define CPUID_1C_DTES64 (1 << 2) +#define CPUID_1C_MONITOR (1 << 3) +#define CPUID_1C_DS_CPL (1 << 4) +#define CPUID_1C_VMX (1 << 5) +#define CPUID_1C_SMX (1 << 6) +#define CPUID_1C_EST (1 << 7) +#define CPUID_1C_TM2 (1 << 8) +#define CPUID_1C_SSSE3 (1 << 9) +#define CPUID_1C_CID (1 << 10) +#define CPUID_1C_SDBG (1 << 11) +#define CPUID_1C_FMA (1 << 12) +#define CPUID_1C_CX16 (1 << 13) +#define CPUID_1C_XTPR (1 << 14) +#define CPUID_1C_PDCM (1 << 15) +#define CPUID_1C_PCID (1 << 17) +#define CPUID_1C_DCA (1 << 18) +#define CPUID_1C_SSE4_1 (1 << 19) +#define CPUID_1C_SSE4_2 (1 << 20) +#define CPUID_1C_X2APIC (1 << 21) +#define CPUID_1C_MOVBE (1 << 22) +#define CPUID_1C_POPCNT (1 << 23) +#define CPUID_1C_TSC (1 << 24) +#define CPUID_1C_AES (1 << 25) +#define CPUID_1C_XSAVE (1 << 26) +#define CPUID_1C_OSXSAVE (1 << 27) +#define CPUID_1C_AVX (1 << 28) +#define CPUID_1C_F16C (1 << 29) +#define CPUID_1C_RDRAND (1 << 30) +#define CPUID_1C_HYPERVISOR (1 << 31) + +#define CPUID_1D_FPU (1 << 0) +#define CPUID_1D_VME (1 << 1) +#define CPUID_1D_DE (1 << 2) +#define CPUID_1D_PSE (1 << 3) +#define CPUID_1D_TSC (1 << 4) +#define CPUID_1D_MSR (1 << 5) +#define CPUID_1D_PAE (1 << 6) +#define CPUID_1D_MCE (1 << 7) +#define CPUID_1D_CX8 (1 << 8) +#define CPUID_1D_APIC (1 << 9) +#define CPUID_1D_SEP (1 << 11) +#define CPUID_1D_MTRR (1 << 12) +#define CPUID_1D_PGE (1 << 13) +#define CPUID_1D_MCA (1 << 14) +#define CPUID_1D_CMOV (1 << 15) +#define CPUID_1D_PAT (1 << 16) +#define CPUID_1D_PSE36 (1 << 17) +#define CPUID_1D_PSN (1 << 18) +#define CPUID_1D_CLFLUSH (1 << 19) +#define CPUID_1D_DS (1 << 21) +#define CPUID_1D_ACPI (1 << 22) +#define CPUID_1D_MMX (1 << 23) +#define CPUID_1D_FXSR (1 << 24) +#define CPUID_1D_SSE (1 << 25) +#define CPUID_1D_SSE2 (1 << 26) +#define CPUID_1D_SS (1 << 27) +#define CPUID_1D_HTT (1 << 28) +#define CPUID_1D_TM (1 << 29) +#define CPUID_1D_IA64 (1 << 30) +#define CPUID_1D_PBE (1 << 31) + +#define CPUID_7B_FSGSBASE (1 << 0) +#define CPUID_7B_SGX (1 << 2) +#define CPUID_7B_BMI (1 << 3) +#define CPUID_7B_HLE (1 << 4) +#define CPUID_7B_AVX2 (1 << 5) +#define CPUID_7B_SMEP (1 << 7) +#define CPUID_7B_BMI2 (1 << 8) +#define CPUID_7B_RTM (1 << 11) +#define CPUID_7B_AVX512F (1 << 16) +#define CPUID_7B_AVX512DQ (1 << 17) +#define CPUID_7B_RDSEED (1 << 18) +#define CPUID_7B_ADX (1 << 19) +#define CPUID_7B_SMAP (1 << 20) +#define CPUID_7B_AVX512IFMA (1 << 21) +#define CPUID_7B_CLFLUSHOPT (1 << 23) +#define CPUID_7B_CLWB (1 << 24) +#define CPUID_7B_AVX512CD (1 << 28) +#define CPUID_7B_SHA (1 << 29) +#define CPUID_7B_AVX512BW (1 << 30) +#define CPUID_7B_AVX512VL (1 << 31) + +#define CPUID_7C_AVX512VBMI (1 << 1) +#define CPUID_7C_UMIP (1 << 2) +#define CPUID_7C_PKU (1 << 3) +#define CPUID_7C_OSPKE (1 << 4) +#define CPUID_7C_WAITPKG (1 << 5) +#define CPUID_7C_AVX512VBMI2 (1 << 6) +#define CPUID_7C_SHSTK (1 << 7) +#define CPUID_7C_GFNI (1 << 8) +#define CPUID_7C_VAES (1 << 9) +#define CPUID_7C_VPCLMULQDQ (1 << 10) +#define CPUID_7C_AVX512VNNI (1 << 11) +#define CPUID_7C_AVX512BITALG (1 << 12) +#define CPUID_7C_TME_EN (1 << 13) +#define CPUID_7C_AVX512VPOPCNTDQ (1 << 14) +#define CPUID_7C_RDPID (1 << 22) +#define CPUID_7C_KL (1 << 23) +#define CPUID_7C_CLDEMOTE (1 << 25) +#define CPUID_7C_MOVDIRI (1 << 27) +#define CPUID_7C_MOVDIR64B (1 << 28) +#define CPUID_7C_ENQCMD (1 << 29) + +#define CPUID_7D_UINTR (1 << 5) +#define CPUID_7D_AVX512VP2INTERSECT (1 << 8) +#define CPUID_7D_SERIALIZE (1 << 14) +#define CPUID_7D_TSXLDTRK (1 << 16) +#define CPUID_7D_PCONFIG (1 << 18) +#define CPUID_7D_IBT (1 << 20) +#define CPUID_7D_AMX_BF16 (1 << 22) +#define CPUID_7D_AVX512FP16 (1 << 23) +#define CPUID_7D_AMX_TILE (1 << 24) +#define CPUID_7D_AMX_INT8 (1 << 25) + +#define RFLAGS_CF (1u << 0) +#define RFLAGS_PF (1u << 2) +#define RFLAGS_AF (1u << 4) +#define RFLAGS_ZF (1u << 6) +#define RFLAGS_SF (1u << 7) +#define RFLAGS_TF (1u << 8) +#define RFLAGS_IF (1u << 9) +#define RFLAGS_DF (1u << 10) +#define RFLAGS_OF (1u << 11) +#define RFLAGS_IOPL (3u << 12) +#define RFLAGS_NT (1u << 14) +#define RFLAGS_RF (1u << 16) +#define RFLAGS_VM (1u << 17) +#define RFLAGS_AC (1u << 18) +#define RFLAGS_VIF (1u << 19) +#define RFLAGS_VIP (1u << 20) +#define RFLAGS_ID (1u << 21) + +#define XCR0_X87 (1 << 0) +#define XCR0_SSE (1 << 1) +#define XCR0_AVX (1 << 2) +#define XCR0_AXV512_OPMASK (1 << 5) +#define XCR0_AVX512_ZMM_HI256 (1 << 6) +#define XCR0_AVX512_HI16_ZMM (1 << 7) + +#define CR0_PE (1 << 0) +#define CR0_MP (1 << 1) +#define CR0_EM (1 << 2) +#define CR0_TS (1 << 3) +#define CR0_ET (1 << 4) +#define CR0_NE (1 << 5) +#define CR0_WP (1 << 16) +#define CR0_AM (1 << 18) +#define CR0_NW (1 << 29) +#define CR0_CD (1 << 30) +#define CR0_PG (1 << 31) + +#define CR4_VME (1 << 0) +#define CR4_PVI (1 << 1) +#define CR4_TSD (1 << 2) +#define CR4_DE (1 << 3) +#define CR4_PSE (1 << 4) +#define CR4_PAE (1 << 5) +#define CR4_MCE (1 << 6) +#define CR4_PGE (1 << 7) +#define CR4_PCE (1 << 8) +#define CR4_OSFXSR (1 << 9) +#define CR4_OSXMMEXCPT (1 << 10) +#define CR4_UMIP (1 << 11) +#define CR4_VMXE (1 << 13) +#define CR4_SMXE (1 << 14) +#define CR4_FSGSBASE (1 << 16) +#define CR4_PCIDE (1 << 17) +#define CR4_OSXSAVE (1 << 18) +#define CR4_SMEP (1 << 20) +#define CR4_SMAP (1 << 21) +#define CR4_PKE (1 << 22) +#define CR4_CET (1 << 23) +#define CR4_PKS (1 << 24) + +#define MSR_EFER_SCE (1 << 0) +#define MSR_EFER_LME (1 << 8) +#define MSR_EFER_LMA (1 << 10) +#define MSR_EFER_NXE (1 << 11) +#define MSR_EFER_SVME (1 << 12) +#define MSR_EFER_LMSLE (1 << 13) +#define MSR_EFER_FFXSR (1 << 14) +#define MSR_EFER_TCE (1 << 15) + +#define MSR_EFER (0xC0000080) +#define MSR_STAR (0xC0000081) +#define MSR_LSTAR (0xC0000082) +#define MSR_CSTAR (0xC0000083) +#define MSR_SFMASK (0xC0000084) +#define MSR_FS_BASE (0xC0000100) +#define MSR_GS_BASE (0xC0000101) +#define MSR_KERNEL_GS_BASE (0xC0000102) + +#define CPL_USER (0b11) +#define CPL_KERNEL (0b00) + +#define IDT_PF (0xE) +#define IDT_IPI_RESCHED (0xFF) diff --git a/arch/x86_64/include/x86_64/gdt.h b/arch/x86_64/include/x86_64/gdt.h new file mode 100644 index 0000000..0d7ca0c --- /dev/null +++ b/arch/x86_64/include/x86_64/gdt.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +union gdt_desc { + uint64_t val; + struct { + uint16_t limit0; + uint16_t base0; + uint8_t base1; + uint8_t access; + uint8_t limit1_flags; + uint8_t base2; + }; +}; + +static_assert(sizeof(union gdt_desc) == 0x8); + +union gdt_long_desc { + uint64_t val[2]; + struct { + uint16_t limit0; + uint16_t base0; + uint8_t base1; + uint8_t access; + uint8_t limit1_flags; + uint8_t base2; + uint32_t base3; + uint32_t reserved; + }; +}; + +static_assert(sizeof(union gdt_long_desc) == 0x10); + +struct [[__packed]] gdt { + union gdt_desc null; + union gdt_desc kernel_code32; + union gdt_desc kernel_data32; + union gdt_desc kernel_code64; + union gdt_desc kernel_data64; + union gdt_desc user_code32; + union gdt_desc user_data; + union gdt_desc user_code64; + union gdt_long_desc tss; +}; + +struct [[__packed]] gdtr { + uint16_t limit; + struct gdt* base; +}; + +struct percpu; + +// Initializes a GDT on the local CPU. +void gdt_load(); diff --git a/arch/x86_64/include/x86_64/hpet.h b/arch/x86_64/include/x86_64/hpet.h new file mode 100644 index 0000000..1460e2a --- /dev/null +++ b/arch/x86_64/include/x86_64/hpet.h @@ -0,0 +1,3 @@ +#pragma once + +void hpet_init(); diff --git a/arch/x86_64/include/x86_64/idt.h b/arch/x86_64/include/x86_64/idt.h new file mode 100644 index 0000000..27dc0b9 --- /dev/null +++ b/arch/x86_64/include/x86_64/idt.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +[[__init]] +void idt_init(); + +// Loads the IDT on this CPU. +void idt_load(); + +void interrupt_return(); diff --git a/arch/x86_64/include/x86_64/smp.h b/arch/x86_64/include/x86_64/smp.h new file mode 100644 index 0000000..cdd5924 --- /dev/null +++ b/arch/x86_64/include/x86_64/smp.h @@ -0,0 +1,49 @@ +#pragma once + +#define SMP_GDTR_OFFSET 80 +#define SMP_FARJMP_OFFSET 90 +#define SMP_TEMP_STACK_OFFSET 98 +#define SMP_TEMP_CR3_OFFSET 102 +#define SMP_ENTRY_OFFSET 106 +#define SMP_HHDM_OFFSET 114 +#define SMP_KERNEL32_DS 16 +#define SMP_KERNEL64_CS 24 +#define SMP_KERNEL64_DS 32 +#define SMP_INFO_SIZE 127 + +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include +#include + +struct [[__packed]] smp_info { + struct gdt gdt; + struct gdtr gdtr; + uint32_t farjmp_offset; + uint32_t farjmp_segment; + uint32_t temp_stack; + uint32_t temp_cr3; + uint64_t entry; + uint64_t hhdm_offset; + uint32_t lapic_id; + uint8_t booted; +}; + +static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr)); +static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset)); +static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack)); +static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3)); +static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry)); +static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset)); +static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32)); +static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64)); +static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64)); +static_assert(SMP_INFO_SIZE == sizeof(struct smp_info)); + +// Initializes a LAPIC. +void x86_64_smp_init(uint32_t id); + +#endif diff --git a/arch/x86_64/include/x86_64/syscall.h b/arch/x86_64/include/x86_64/syscall.h new file mode 100644 index 0000000..3ac6682 --- /dev/null +++ b/arch/x86_64/include/x86_64/syscall.h @@ -0,0 +1,3 @@ +#pragma once + +void arch_syscall_stub(); diff --git a/arch/x86_64/include/x86_64/tsc.h b/arch/x86_64/include/x86_64/tsc.h new file mode 100644 index 0000000..71dff6f --- /dev/null +++ b/arch/x86_64/include/x86_64/tsc.h @@ -0,0 +1,3 @@ +#pragma once + +void tsc_init(); diff --git a/arch/x86_64/include/x86_64/tss.h b/arch/x86_64/include/x86_64/tss.h new file mode 100644 index 0000000..aa01f30 --- /dev/null +++ b/arch/x86_64/include/x86_64/tss.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +struct [[__packed]] tss { + uint32_t reserved0; + uint64_t rsp0; + uint64_t rsp1; + uint64_t rsp2; + uint32_t reserved1; + uint32_t reserved2; + uint64_t ist1; + uint64_t ist2; + uint64_t ist3; + uint64_t ist4; + uint64_t ist5; + uint64_t ist6; + uint64_t ist7; + uint32_t reserved3; + uint32_t reserved4; + uint16_t reserved5; + uint16_t iopb; +}; + +static_assert(sizeof(struct tss) == 0x68); diff --git a/arch/x86_64/meson.build b/arch/x86_64/meson.build new file mode 100644 index 0000000..b75ec1e --- /dev/null +++ b/arch/x86_64/meson.build @@ -0,0 +1,23 @@ +kernel_c_args += [ + '-mgeneral-regs-only', + '-mno-red-zone', + '-mcmodel=kernel', +] + +kernel_sources += files( + 'apic.c', + 'archctl.c', + 'gdt.c', + 'hpet.c', + 'idt.c', + 'percpu.c', + 'pmap.c', + 'sched.c', + 'serial.c', + 'smp.c', + 'sys.c', + 'syscall.c', + 'trampoline.S', + 'tsc.c', + 'usercopy.c', +) diff --git a/arch/x86_64/percpu.c b/arch/x86_64/percpu.c new file mode 100644 index 0000000..4d050c6 --- /dev/null +++ b/arch/x86_64/percpu.c @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void arch_percpu_bsp_init() { + asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp); + asm_wrmsr(MSR_FS_BASE, 0); + asm_wrmsr(MSR_KERNEL_GS_BASE, 0); + + pic_disable(); + gdt_load(); + idt_init(); + idt_load(); +} + +union ia32_star { + uint64_t value; + struct { + uint32_t rsvd; + uint16_t kernel_cs_ss; + uint16_t user_cs_ss; + }; +}; + +static void setup_cpu(struct percpu* cpu) { + asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu); + asm_wrmsr(MSR_FS_BASE, 0); + asm_wrmsr(MSR_KERNEL_GS_BASE, 0); + + gdt_load(); + idt_load(); + + // Syscall extension. + asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE); + union ia32_star star = {0}; + star.kernel_cs_ss = offsetof(struct gdt, kernel_code64); + star.user_cs_ss = offsetof(struct gdt, user_code32); + asm_wrmsr(MSR_STAR, star.value); + asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub); + asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF); + + uint64_t cr0, cr4; + asm volatile("mov %0, cr0" : "=r"(cr0)); + asm volatile("mov %0, cr4" : "=r"(cr4)); + + struct cpuid cpuid1 = asm_cpuid(1, 0); + struct cpuid cpuid7 = asm_cpuid(7, 0); + struct cpuid cpuid13 = asm_cpuid(13, 0); + + // Enable SSE. + cr0 &= ~CR0_EM; + cr0 |= CR0_MP; + cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT; + + if (cpuid1.ecx & CPUID_1C_XSAVE) { + cr4 |= CR4_OSXSAVE; + asm volatile("mov cr4, %0" ::"r"(cr4)); + + uint64_t xcr0 = XCR0_X87 | XCR0_SSE; + // AVX + if (cpuid1.ecx & CPUID_1C_AVX) + xcr0 |= XCR0_AVX; + + // AVX-512 + if (cpuid7.ebx & CPUID_7B_AVX512F) + xcr0 |= XCR0_AXV512_OPMASK | XCR0_AVX512_ZMM_HI256 | XCR0_AVX512_HI16_ZMM; + + asm_wrxcr(0, xcr0); + + cpu->arch.fpu_size = cpuid13.ecx; + cpu->arch.fpu_save = asm_xsave; + cpu->arch.fpu_restore = asm_xrstor; + } else { + cpu->arch.fpu_size = 512; + cpu->arch.fpu_save = asm_fxsave; + cpu->arch.fpu_restore = asm_fxrstor; + } + + if (cpuid7.ecx & CPUID_7C_UMIP) + cr4 |= CR4_UMIP; + + if (cpuid7.ebx & CPUID_7B_SMEP) + cr4 |= CR4_SMEP; + + if (cpuid7.ebx & CPUID_7B_SMAP) { + cr4 |= CR4_SMAP; + cpu->arch.can_smap = true; + } + + if (cpuid7.ebx & CPUID_7B_FSGSBASE) + cr4 |= CR4_FSGSBASE; + + asm volatile("mov %0, cr0" : "=r"(cr0)); + asm volatile("mov %0, cr4" : "=r"(cr4)); + + lapic_init(&cpu->arch.lapic); + cpu->online = true; +} + +void arch_percpu_init() { + hpet_init(); + tsc_init(); + + // Init the BSP. + setup_cpu(&percpu_bsp); + + // Init any APs. +} diff --git a/arch/x86_64/pmap.c b/arch/x86_64/pmap.c new file mode 100644 index 0000000..f794929 --- /dev/null +++ b/arch/x86_64/pmap.c @@ -0,0 +1,56 @@ +#include +#include + +enum : pte_t { + ARCH_FLAG_PRESENT = 1 << 0, + ARCH_FLAG_READ_WRITE = 1 << 1, + ARCH_FLAG_USER_MODE = 1 << 2, + ARCH_FLAG_WRITE_THROUGH = 1 << 3, + ARCH_FLAG_CACHE_DISABLE = 1 << 4, + ARCH_FLAG_ACCESSED = 1 << 5, + ARCH_FLAG_DIRTY = 1 << 6, + ARCH_FLAG_SIZE = 1 << 7, + ARCH_FLAG_GLOBAL = 1 << 8, + ARCH_FLAG_AVAILABLE = 1 << 9, + ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10, + ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63, + + ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000, +}; + +void pte_clear(pte_t* pte) { + *pte = 0; +} + +pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) { + pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT; + + if (flags & PTE_USER) + result |= ARCH_FLAG_USER_MODE; + + if (flags & PTE_DIR) + result |= ARCH_FLAG_READ_WRITE; + else { + if (flags & PTE_WRITE) + result |= ARCH_FLAG_READ_WRITE; + if (!(flags & PTE_EXEC)) + result |= ARCH_FLAG_EXECUTE_DISABLE; + } + return result; +} + +bool pte_is_present(pte_t* pte) { + return *pte & ARCH_FLAG_PRESENT; +} + +bool pte_is_dir(pte_t* pte) { + return *pte & ARCH_FLAG_SIZE; +} + +phys_t pte_address(pte_t* pte) { + return (phys_t)(*pte & ARCH_PTE_ADDR_MASK); +} + +void pmap_set(struct pmap* pt) { + asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory"); +} diff --git a/arch/x86_64/sched.c b/arch/x86_64/sched.c new file mode 100644 index 0000000..faf82e4 --- /dev/null +++ b/arch/x86_64/sched.c @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// The task frame consists of registers that the C ABI marks as callee-saved. +// If we don't save them, these registers are lost during a context switch. +struct task_frame { + uint64_t rbx, rbp, r12, r13, r14, r15, rip; +}; + +[[__naked]] +static void perform_switch(uint64_t* old_rsp, uint64_t new_rsp) { + asm volatile( + "sub rsp, 0x30\n" // Make room for all regs (except RIP). + "mov [rsp + %c0], rbx\n" + "mov [rsp + %c1], rbp\n" + "mov [rsp + %c2], r12\n" + "mov [rsp + %c3], r13\n" + "mov [rsp + %c4], r14\n" + "mov [rsp + %c5], r15\n" + "mov [rdi], rsp\n" // rdi = old_rsp + "mov rsp, rsi\n" // rsi = new_rsp + "mov rbx, [rsp + %c0]\n" + "mov rbp, [rsp + %c1]\n" + "mov r12, [rsp + %c2]\n" + "mov r13, [rsp + %c3]\n" + "mov r14, [rsp + %c4]\n" + "mov r15, [rsp + %c5]\n" + "add rsp, 0x30\n" + "call %c6\n" + "ret" // This will conveniently move us to the RIP we put at this stack entry. + : + : "i"(offsetof(struct task_frame, rbx)), + "i"(offsetof(struct task_frame, rbp)), + "i"(offsetof(struct task_frame, r12)), + "i"(offsetof(struct task_frame, r13)), + "i"(offsetof(struct task_frame, r14)), + "i"(offsetof(struct task_frame, r15)), + "i"(irq_unlock) + : "memory" + ); +} + +void arch_sched_switch(struct task* from, struct task* to) { + struct arch_percpu* cpu = &percpu_get()->arch; + struct arch_task_context* from_ctx = &from->context; + struct arch_task_context* to_ctx = &to->context; + + cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE; + + if (from->user_stack) { + cpu->fpu_save(from_ctx->fpu_region); + from_ctx->ds = asm_read_ds(); + from_ctx->es = asm_read_es(); + from_ctx->fs = asm_read_fs(); + from_ctx->gs = asm_read_gs(); + from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE); + from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE); + } + + if (to->user_stack) { + cpu->fpu_restore(to_ctx->fpu_region); + asm_write_ds(to_ctx->ds); + asm_write_es(to_ctx->es); + asm_write_fs(to_ctx->fs); + + // If we have to change the GS segment we need to reload the MSR, otherwise we lose its value. + if (to_ctx->gs != asm_read_gs()) { + uint64_t percpu = asm_rdmsr(MSR_GS_BASE); + asm_write_gs(to_ctx->gs); + asm_wrmsr(MSR_GS_BASE, percpu); + } + + asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base); + asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base); + } + + perform_switch(&from_ctx->rsp, to_ctx->rsp); +} + +[[__naked]] +void task_entry_thunk() { + asm volatile( + "mov rdi, rbx\n" + "mov rsi, r12\n" + "jmp %c0" + : + : "i"(task_entry) + : "memory" + ); +} + +errno_t arch_task_init( + struct arch_task_context* context, + void* entry, + uintptr_t arg0, + uintptr_t stack_start, + bool is_user +) { + struct arch_percpu* cpu = &percpu_get()->arch; + + // Prepare a dummy stack with an entry point function to return to. + struct task_frame* frame = (struct task_frame*)stack_start - 1; + frame->rbx = (uint64_t)entry; + frame->r12 = arg0; + frame->rip = (uint64_t)task_entry_thunk; + context->rsp = (uint64_t)frame; + + if (is_user) { + // Allocate FPU space. + const size_t fpu_size = cpu->fpu_size; + void* fpu_base = vm_alloc(fpu_size); + if (!fpu_base) + return ENOMEM; + for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) { + phys_t page; + errno_t s = mem_phys_alloc(1, 0, &page); + if (s) { + vm_free(fpu_base, fpu_size); + return s; + } + + pmap_map(&kernel_vas.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK); + } + context->fpu_region = fpu_base; + + context->ds = asm_read_ds(); + context->es = asm_read_es(); + context->fs = asm_read_fs(); + context->gs = asm_read_gs(); + context->fs_base = asm_rdmsr(MSR_FS_BASE); + context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE); + } + + return 0; +} + +[[__naked]] +void arch_sched_preempt_disable() { + asm volatile( + "inc qword ptr gs:%c0\n" + "ret" + : + : "i"(offsetof(struct percpu, sched.preempt_level)) + ); +} + +[[__naked]] +bool arch_sched_preempt_enable() { + asm volatile( + "dec qword ptr gs:%c0\n" + "setz al\n" + "movzx eax, al\n" + "ret\n" + : + : "i"(offsetof(struct percpu, sched.preempt_level)) + : "cc" + ); +} + +[[__naked]] +void arch_sched_jump_to_context(struct arch_context* context) { + asm volatile( + "mov rsp, rdi\n" + "jmp %c0\n" ::"i"(interrupt_return) + ); +} + +void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) { + struct arch_context context = { + .rip = ip, + .rsp = sp, + .rdi = arg, + .rflags = 0x202, + .cs = offsetof(struct gdt, user_code64) | CPL_USER, + .ss = offsetof(struct gdt, user_data) | CPL_USER, + }; + + uintptr_t percpu = asm_rdmsr(MSR_GS_BASE); + const uint16_t zero = 0; + asm volatile( + "mov ds, %0\n" + "mov es, %0\n" + "mov fs, %0\n" + "mov gs, %0\n" + : + : "r"(zero) + ); + + asm_wrmsr(MSR_FS_BASE, 0); + asm_wrmsr(MSR_GS_BASE, percpu); + asm_wrmsr(MSR_KERNEL_GS_BASE, 0); + + arch_sched_jump_to_context(&context); +} diff --git a/arch/x86_64/serial.c b/arch/x86_64/serial.c new file mode 100644 index 0000000..983da21 --- /dev/null +++ b/arch/x86_64/serial.c @@ -0,0 +1,47 @@ +#include +#include +#include + +constexpr uint16_t COM1_BASE = 0x3F8; + +static bool is_ready() { + return asm_inb(COM1_BASE + 5) & 0x20; +} + +static void serial_write(struct console* con, const char* buf, size_t count) { + for (size_t i = 0; i < count; i++) { + while (!is_ready()) + asm volatile("pause"); + + asm_outb(COM1_BASE, buf[i]); + } +} + +bool serial_setup(struct console* con) { + asm_outb(COM1_BASE + 1, 0x00); // Disable interrupts + asm_outb(COM1_BASE + 3, 0x80); // Enable DLAB (set baud rate divisor) + asm_outb(COM1_BASE, 0x01); // Set divisor low byte (115200 baud if 1) + asm_outb(COM1_BASE + 1, 0x00); // Set divisor high byte + asm_outb(COM1_BASE + 3, 0x03); // 8 bits, no parity, one stop bit (8n1) + asm_outb(COM1_BASE + 2, 0xC7); // Enable FIFO, clear them, with 14-byte threshold + asm_outb(COM1_BASE + 4, 0x0B); // IRQs enabled, RTS/DSR set + + asm_outb(COM1_BASE + 4, 0x1E); // Set to loopback mode. + asm_outb(COM1_BASE, 0xAE); // Send a test byte. + + // If we don't get the same value back, this serial port doesn't work. + if (asm_inb(COM1_BASE) != 0xAE) { + return false; + } + + asm_outb(COM1_BASE + 4, 0x0F); // Disable loopback mode. + + return true; +} + +static struct console serial_con = { + .name = "com1", + .write = serial_write, + .init = serial_setup, +}; +DEFINE_CONSOLE(&serial_con); diff --git a/arch/x86_64/smp.c b/arch/x86_64/smp.c new file mode 100644 index 0000000..fd8aa79 --- /dev/null +++ b/arch/x86_64/smp.c @@ -0,0 +1,17 @@ +#include +#include +#include + +extern uint8_t smp_trampoline_start[]; +extern uint8_t smp_trampoline_end[]; + +[[noreturn]] +static void ap_entry(phys_t info) { + // TODO + + while (1) { + asm volatile("hlt"); + } +} + +void x86_64_smp_init(uint32_t id) {} diff --git a/arch/x86_64/sys.c b/arch/x86_64/sys.c new file mode 100644 index 0000000..f3377ab --- /dev/null +++ b/arch/x86_64/sys.c @@ -0,0 +1,21 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +[[__init, __naked, __used]] +void _start() { + asm volatile( + "lea rsp, [rip + %0]\n" + "jmp %1" + : + : "i"(__ld_stack_top), "r"(kernel_entry) + ); +} + +void arch_panic() {} diff --git a/arch/x86_64/syscall.c b/arch/x86_64/syscall.c new file mode 100644 index 0000000..016d4e2 --- /dev/null +++ b/arch/x86_64/syscall.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include +#include +#include + +[[__naked]] +void arch_syscall_stub() { + asm volatile( + "swapgs\n" + "mov gs:%c0, rsp\n" + "mov rsp, gs:%c1\n" + "cld\n" + // We're pretending to be an interrupt, so fill the bottom fields of `Context`. + "push %c3\n" // SS and CS are not changed during SYSCALL. Use `gdt::user_data | CPL_USER`. + "push gs:%c0\n" + "push r11\n" // RFLAGS is moved into r11 by the CPU. + "push %c2\n" // Same as SS. Use `gdt::user_code64 | CPL_USER` + "push rcx\n" // RIP is moved into rcx by the CPU. + "push 0x00\n" // arch_context::error field + "push 0x00\n" // arch_context::isr field + "push rax\n" + "push rbx\n" + "push rcx\n" + "push rdx\n" + "push rbp\n" + "push rdi\n" + "push rsi\n" + "push r8\n" + "push r9\n" + "push r10\n" + "push r11\n" + "push r12\n" + "push r13\n" + "push r14\n" + "push r15\n" + "xor rbp, rbp\n" + "mov rdi, rsp\n" // Put the trap frame struct as first argument. + "call %c4\n" // Call syscall handler + "cli\n" + "pop r15\n" + "pop r14\n" + "pop r13\n" + "pop r12\n" + "pop r11\n" + "pop r10\n" + "pop r9\n" + "pop r8\n" + "pop rsi\n" + "pop rdi\n" + "pop rbp\n" + "pop rdx\n" + "pop rcx\n" + "pop rbx\n" + "pop rax\n" + "add rsp, 0x10\n" // Skip .error and .isr fields (2 * sizeof(uint64_t)) + "mov rsp, gs:%c0\n" // Load user stack from `percpu::user_stack`. + "swapgs\n" + "sysretq\n" // Return to user mode. + : + : "i"(offsetof(struct percpu, user_stack)), + "i"(offsetof(struct percpu, kernel_stack)), + "i"(offsetof(struct gdt, user_code64) | CPL_USER), + "i"(offsetof(struct gdt, user_data) | CPL_USER), + "i"(syscall_dispatch) + ); +} diff --git a/arch/x86_64/trampoline.S b/arch/x86_64/trampoline.S new file mode 100644 index 0000000..f60b280 --- /dev/null +++ b/arch/x86_64/trampoline.S @@ -0,0 +1,107 @@ +#include +#include + +.intel_syntax noprefix + +.global smp_trampoline_start +.global smp_trampoline_end + +.section .rodata +.code16 + +smp_trampoline_start: + cli + cld + jmp 1f + +smp_trampoline_data: + .skip SMP_INFO_SIZE + +.set data_offset, (smp_trampoline_data - smp_trampoline_start) +.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET) +.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET) +.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET) +.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET) +.set entry_offset, (data_offset + SMP_ENTRY_OFFSET) +.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET) + +1: + mov bx, cs + shl ebx, 4 + +.set idtr_offset, (invalid_idtr - smp_trampoline_start) + lidtd cs:idtr_offset + lgdtd cs:gdtr_offset + +.set mode32_offset, (mode32 - smp_trampoline_start) + lea eax, [ebx + mode32_offset] + mov dword ptr cs:farjmp_offset, eax + + mov eax, CR0_PE | CR0_ET + mov cr0, eax + jmp fword ptr cs:farjmp_offset + +.code32 +mode32: + mov ax, SMP_KERNEL32_DS + mov ds, ax + mov es, ax + mov fs, ax + mov gs, ax + mov ss, ax + + xor eax, eax + lldt ax + + mov esp, dword ptr [ebx + temp_stack_offset] + + xor eax, eax + mov cr4, eax + + or eax, CR4_PAE + mov cr4, eax + + mov ecx, MSR_EFER + mov eax, MSR_EFER_LME + xor edx, edx + wrmsr + + mov eax, dword ptr [ebx + temp_cr3_offset] + mov cr3, eax + + mov eax, cr0 + or eax, CR0_PG + mov cr0, eax + +.set mode64_offset, (mode64 - smp_trampoline_start) + lea eax, [ebx + mode64_offset] + push SMP_KERNEL64_CS + push eax + retf + +.code64 +mode64: + mov ax, SMP_KERNEL64_DS + mov ds, ax + mov es, ax + mov fs, ax + mov gs, ax + mov ss, ax + + mov ecx, MSR_EFER + rdmsr + or eax, MSR_EFER_NXE + wrmsr + + xor ebp, ebp + + lea rdi, [rbx + data_offset] + mov rax, [rbx + entry_offset] + add rsp, [rbx + hhdm_offset] + jmp rax + +invalid_idtr: + .word 0 + .quad 0 + +smp_trampoline_end: diff --git a/arch/x86_64/tsc.c b/arch/x86_64/tsc.c new file mode 100644 index 0000000..fd8200b --- /dev/null +++ b/arch/x86_64/tsc.c @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include + +static bool use_tsc = true; + +void tsc_init() { + if (!use_tsc) + return; + + if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) { + printf("No invariant TSC detected!\n"); + return; + } + + // Check for the TSC info leaf. + bool has_tsc_leaf = false; + struct cpuid tsc_leaf = {0}; + if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) { + tsc_leaf = asm_cpuid(0x15, 0); + } + + uint64_t freq = 0; + if (clock_available()) { + // Calibrate using existing clock. + uint64_t t1 = asm_rdtsc(); + clock_spin_ns(10'000'000); + uint64_t t2 = asm_rdtsc(); + freq = (t2 - t1) * 100; + } else if (has_tsc_leaf) { + // If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency. + // On a normal system, this should usually never be called and is a last resort + // since at this point we have at least the HPET timer. + if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) { + printf("Calibrating TSC using CPUID 0x15"); + freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax; + } else { + printf("Unable to calibrate TSC using CPUID frequency information"); + return; + } + } else { + printf("Unable to calibrate the TSC without a clock or static frequency info!\n"); + return; + } + + printf("Timer frequency is %lu MHz\n", freq / 1'000'000); +} + +static void tsc_option(bool opt) { + use_tsc = opt; +} +CMDLINE_OPTION("tsc", tsc_option); diff --git a/arch/x86_64/usercopy.c b/arch/x86_64/usercopy.c new file mode 100644 index 0000000..8d85829 --- /dev/null +++ b/arch/x86_64/usercopy.c @@ -0,0 +1,137 @@ +#include +#include + +extern void x86_64_read_start(); +extern void x86_64_read_end(); +extern void x86_64_read_fault(); + +static const struct usercopy_region read_region = { + .start_ip = x86_64_read_start, + .end_ip = x86_64_read_end, + .fault_ip = x86_64_read_fault, +}; + +bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) { + asm volatile( + "stac\n" + // Setup regs for `rep movsb`. + "xchg rcx, rdx\n" + "mov rax, [rip + %0]\n" + "mov [rdx], rax\n" + + ".global x86_64_read_start\n" + "x86_64_read_start:\n" + "rep movsb\n" + + ".global x86_64_read_end\n" + "x86_64_read_end:\n" + "xor rax, rax\n" + "mov [rdx], rax\n" + "mov rax, 1\n" + "clac\n" + "ret\n" + + ".global x86_64_read_fault\n" + "x86_64_read_fault:\n" + "xor rax, rax\n" + "mov [rdx], rax\n" + "clac\n" + "ret\n" + : + : "i"(&read_region) + : "memory" + ); + return false; +} + +extern void x86_64_write_start(); +extern void x86_64_write_end(); +extern void x86_64_write_fault(); + +static const struct usercopy_region write_region = { + .start_ip = x86_64_write_start, + .end_ip = x86_64_write_end, + .fault_ip = x86_64_write_fault, +}; + +bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) { + asm volatile( + "stac\n" + // Setup regs for `rep movsb`. + "xchg rcx, rdx\n" + "mov rax, [rip + %0]\n" + "mov [rdx], rax\n" + + ".global x86_64_write_start\n" + "x86_64_write_start:\n" + "rep movsb\n" + + ".global x86_64_write_end\n" + "x86_64_write_end:\n" + "xor rax, rax\n" + "mov [rdx], rax\n" + "mov rax, 1\n" + "clac\n" + "ret\n" + + ".global x86_64_write_fault\n" + "x86_64_write_fault:\n" + "xor rax, rax\n" + "mov [rdx], rax\n" + "clac\n" + "ret\n" + : + : "i"(&read_region) + : "memory" + ); + return false; +} + +extern void x86_64_strlen_start(); +extern void x86_64_strlen_end(); +extern void x86_64_strlen_fault(); + +static const struct usercopy_region strlen_region = { + .start_ip = x86_64_strlen_start, + .end_ip = x86_64_strlen_end, + .fault_ip = x86_64_strlen_fault, +}; + +[[__naked]] +bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) { + asm volatile( + "stac\n" + "mov rax, [rip + %0]\n" + "mov [rcx], rax\n" + + ".global x86_64_strlen_start\n" + "x86_64_strlen_start:\n" + "xor r8, r8\n" + ".Lloop:\n" + "cmp byte ptr [rdi + r8], 0\n" + "je .Lleave\n" + "inc r8\n" + "cmp rsi, r8\n" + "jne .Lloop\n" + ".Lleave:\n" + "mov [rdx], r8\n" + + ".global x86_64_strlen_end\n" + "x86_64_strlen_end:\n" + "xor rax, rax\n" + "mov [rcx], rax\n" + "mov rax, 1\n" + "clac\n" + "ret\n" + + ".global x86_64_strlen_fault\n" + "x86_64_strlen_fault:\n" + "xor rax, rax\n" + "mov [rcx], rax\n" + "clac\n" + "ret\n" + : + : "i"(&strlen_region) + : "memory" + ); +} diff --git a/boot/cmdline.c b/boot/cmdline.c new file mode 100644 index 0000000..f6b9566 --- /dev/null +++ b/boot/cmdline.c @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include + +// Defined in linker script. +extern const struct cmdline_option* __ld_cmdline_start[]; +extern const struct cmdline_option* __ld_cmdline_end[]; + +static char cmdline_buf[0x1000]; + +[[__init]] +void cmdline_parse(const char* cmdline) { + const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1); + size_t idx = 0; + memcpy(cmdline_buf, cmdline, len); + + while (1) { + const struct cmdline_option** opt = __ld_cmdline_start; + char* name = nullptr; + char* value = nullptr; + + // Skip all leading spaces. + while (idx < len && cmdline_buf[idx] == ' ') + idx++; + if (idx >= len) + break; + size_t name_idx = idx; + name = cmdline_buf + name_idx; + + // Find the next equal sign or space. + while (idx < len && cmdline_buf[idx] != '=' && cmdline_buf[idx] != ' ') + idx++; + if (idx > len) + break; + + // Check if the option has a value (=foo). + char seperator = cmdline_buf[idx]; + cmdline_buf[idx++] = 0; + if (seperator == '=') { + // Check if we need to escape the value. + char check; + if (cmdline_buf[idx] == '"') { + check = '"'; + cmdline_buf[idx++] = 0; + } else { + check = ' '; + } + + value = cmdline_buf + idx; + + // Skip the value. + while (idx < len && cmdline_buf[idx] != check) + idx++; + if (idx > len) + break; + cmdline_buf[idx++] = 0; + } + + // Find the corresponding option. + while (opt < __ld_cmdline_end) { + const struct cmdline_option* current = *opt; + if (!strcmp(current->name, name)) { + switch (current->option_type) { + case CMDLINE_STRING: + current->func_str(value); + break; + case CMDLINE_BOOL: + if (!strcmp(value, "on") || !strcmp(value, "yes") || !strcmp(value, "true") || !strcmp(value, "1")) + current->func_bool(true); + else + current->func_bool(false); + break; + case CMDLINE_INTPTR: { + int base = 10; + if (!strncmp(value, "0x", 2)) { + base = 0x10; + value += 2; + } + current->func_intptr(atol(value, base)); + break; + } + case CMDLINE_UINTPTR: { + int base = 10; + if (!strncmp(value, "0x", 2)) { + base = 0x10; + value += 2; + } + current->func_uintptr(atolu(value, base)); + break; + } + } + break; + } + opt++; + } + + if (idx >= len) + break; + } +} diff --git a/boot/kernel.ld b/boot/kernel.ld new file mode 100644 index 0000000..9b940c4 --- /dev/null +++ b/boot/kernel.ld @@ -0,0 +1,97 @@ +ENTRY(_start) + +PHDRS { + text PT_LOAD FLAGS(5); + rodata PT_LOAD FLAGS(4); + data PT_LOAD FLAGS(6); + dynamic PT_DYNAMIC FLAGS(6); + percpu PT_LOAD FLAGS(6); +} + +SECTIONS +{ + . = 0xffffffff80000000; + __ld_kernel_start = .; + + .text ALIGN(CONSTANT(MAXPAGESIZE)) : { + __ld_text_start = .; + + KEEP(*(.init.text)) + *(.text .text.*) + *(.plt .plt.*) + + __ld_text_end = .; + } :text + + .rodata ALIGN(CONSTANT(MAXPAGESIZE)) : { + __ld_rodata_start = .; + + __ld_cmdline_start = .; + KEEP(*(.cmdline)) + __ld_cmdline_end = .; + + *(.rodata .rodata.*) + *(.got .got.*) + + __ld_rodata_end = .; + } :rodata + + .data ALIGN(CONSTANT(MAXPAGESIZE)) : { + __ld_data_start = .; + + __ld_console_start = .; + KEEP(*(.console)) + __ld_console_end = .; + + KEEP(*(SORT_BY_NAME(.init.data.*))) + KEEP(*(.init.data)) + *(.data .data.*) + } :data + + .dynsym : { + __ld_dynsym_start = .; + *(.dynsym) + __ld_dynsym_end = .; + } :data + + .dynstr : { + __ld_dynstr_start = .; + *(.dynstr) + __ld_dynstr_end = .; + } :data + + .dynamic : { + *(.dynamic) + } :data :dynamic + + .bss : { + /* Small stack to bootstrap the allocator */ + __ld_stack_bottom = .; + . += 0x2000; + __ld_stack_top = .; + *(.*bss .*bss.*) + *(COMMON) + } :data + + /* Per-CPU data block. MUST always come last. */ + .percpu ALIGN(CONSTANT(MAXPAGESIZE)) : { + __ld_percpu_start = .; + KEEP(*(.percpu)) + . = ALIGN(CONSTANT(MAXPAGESIZE)); + __ld_percpu_end = .; + + __ld_data_end = .; + } :percpu + + __ld_kernel_end = .; + + /DISCARD/ : + { + *(.comment*) + *(.eh_frame*) + *(.rel.eh_frame*) + *(.gcc_except_table*) + *(.note.gnu.property) + *(.interp) + } +} diff --git a/boot/limine/entry.c b/boot/limine/entry.c new file mode 100644 index 0000000..9a25327 --- /dev/null +++ b/boot/limine/entry.c @@ -0,0 +1,93 @@ +#include +#include +#include +#include +#include +#include +#include "limine.h" + +[[__initdata_sorted("limine.0")]] +static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER; +[[__initdata_sorted("limine.1")]] +static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6); +[[__initdata_sorted("limine.2")]] +static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER; + +#define LIMINE_REQUEST(request, tag, rev) \ + [[__initdata_sorted("limine.1")]] \ + static volatile struct limine_##request request = { \ + .id = tag, \ + .revision = rev, \ + .response = nullptr, \ + } + +LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0); +LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0); +LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0); +LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0); +LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1); +LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0); +LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0); +LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0); + +[[__initdata]] +static struct phys_mem mem[128]; +[[__initdata]] +static struct boot_file files[8]; + +[[__init]] +void kernel_entry() { + kernel_early_init(); + + struct boot_info info; + + struct limine_memmap_response* mm_res = memmap_request.response; + struct limine_module_response* module_res = module_request.response; + struct limine_executable_address_response* exec_res = executable_address_request.response; + struct limine_executable_file_response* exec_file_res = executable_file_request.response; + + info.cmdline = exec_file_res->executable_file->string; + + info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count); + for (size_t i = 0; i < info.num_mem_maps; i++) { + mem[i].address = mm_res->entries[i]->base; + mem[i].length = mm_res->entries[i]->length; + + switch (mm_res->entries[i]->type) { + case LIMINE_MEMMAP_USABLE: + mem[i].usage = PHYS_USABLE; + break; + case LIMINE_MEMMAP_ACPI_RECLAIMABLE: + case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE: + mem[i].usage = PHYS_RECLAIMABLE; + break; + case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES: + mem[i].usage = PHYS_STATIC; + break; + default: + mem[i].usage = PHYS_RESERVED; + break; + } + } + + if (module_res) { + info.files = files; + info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count); + for (size_t i = 0; i < info.num_files; i++) { + files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset; + files[i].length = module_res->modules[i]->size; + strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path)); + } + } + + info.mem_map = mem; + info.phys_base = (phys_t)exec_res->physical_base; + info.virt_base = (uintptr_t)exec_res->virtual_base; + info.hhdm_base = (uintptr_t)hhdm_request.response->offset; + + info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base; + + kernel_main(&info); +} + +phys_t rsdp_addr; diff --git a/boot/limine/limine.h b/boot/limine/limine.h new file mode 100644 index 0000000..e350dbc --- /dev/null +++ b/boot/limine/limine.h @@ -0,0 +1,669 @@ +/* SPDX-License-Identifier: 0BSD */ + +/* Copyright (C) 2022-2026 Mintsuki and contributors. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef LIMINE_H +#define LIMINE_H 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Misc */ + +#ifdef LIMINE_NO_POINTERS +# define LIMINE_PTR(TYPE) uint64_t +#else +# define LIMINE_PTR(TYPE) TYPE +#endif + +#define LIMINE_REQUESTS_START_MARKER { 0xf6b8f4b39de7d1ae, 0xfab91a6940fcb9cf, \ + 0x785c6ed015d3e316, 0x181e920a7852b9d9 } +#define LIMINE_REQUESTS_END_MARKER { 0xadc0e0531bb10d03, 0x9572709f31764c62 } + +#define LIMINE_BASE_REVISION(N) { 0xf9562b2d5c95a6c8, 0x6a7b384944536bdc, (N) } + +#define LIMINE_BASE_REVISION_SUPPORTED(VAR) ((VAR)[2] == 0) + +#define LIMINE_LOADED_BASE_REVISION_VALID(VAR) ((VAR)[1] != 0x6a7b384944536bdc) +#define LIMINE_LOADED_BASE_REVISION(VAR) ((VAR)[1]) + +#define LIMINE_COMMON_MAGIC 0xc7b1dd30df4c8b88, 0x0a82e883a194f07b + +struct limine_uuid { + uint32_t a; + uint16_t b; + uint16_t c; + uint8_t d[8]; +}; + +#define LIMINE_MEDIA_TYPE_GENERIC 0 +#define LIMINE_MEDIA_TYPE_OPTICAL 1 +#define LIMINE_MEDIA_TYPE_TFTP 2 + +struct limine_file { + uint64_t revision; + LIMINE_PTR(void *) address; + uint64_t size; + LIMINE_PTR(char *) path; + LIMINE_PTR(char *) string; + uint32_t media_type; + uint32_t unused; + uint8_t tftp_ipv4[4]; + uint32_t tftp_port; + uint32_t partition_index; + uint32_t mbr_disk_id; + struct limine_uuid gpt_disk_uuid; + struct limine_uuid gpt_part_uuid; + struct limine_uuid part_uuid; +}; + +/* Boot info */ + +#define LIMINE_BOOTLOADER_INFO_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xf55038d8e2a1202f, 0x279426fcf5f59740 } + +struct limine_bootloader_info_response { + uint64_t revision; + LIMINE_PTR(char *) name; + LIMINE_PTR(char *) version; +}; + +struct limine_bootloader_info_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_bootloader_info_response *) response; +}; + +/* Executable command line */ + +#define LIMINE_EXECUTABLE_CMDLINE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x4b161536e598651e, 0xb390ad4a2f1f303a } + +struct limine_executable_cmdline_response { + uint64_t revision; + LIMINE_PTR(char *) cmdline; +}; + +struct limine_executable_cmdline_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_executable_cmdline_response *) response; +}; + +/* Firmware type */ + +#define LIMINE_FIRMWARE_TYPE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8c2f75d90bef28a8, 0x7045a4688eac00c3 } + +#define LIMINE_FIRMWARE_TYPE_X86BIOS 0 +#define LIMINE_FIRMWARE_TYPE_EFI32 1 +#define LIMINE_FIRMWARE_TYPE_EFI64 2 +#define LIMINE_FIRMWARE_TYPE_SBI 3 + +struct limine_firmware_type_response { + uint64_t revision; + uint64_t firmware_type; +}; + +struct limine_firmware_type_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_firmware_type_response *) response; +}; + +/* Stack size */ + +#define LIMINE_STACK_SIZE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x224ef0460a8e8926, 0xe1cb0fc25f46ea3d } + +struct limine_stack_size_response { + uint64_t revision; +}; + +struct limine_stack_size_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_stack_size_response *) response; + uint64_t stack_size; +}; + +/* HHDM */ + +#define LIMINE_HHDM_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x48dcf1cb8ad2b852, 0x63984e959a98244b } + +struct limine_hhdm_response { + uint64_t revision; + uint64_t offset; +}; + +struct limine_hhdm_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_hhdm_response *) response; +}; + +/* Framebuffer */ + +#define LIMINE_FRAMEBUFFER_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9d5827dcd881dd75, 0xa3148604f6fab11b } + +#define LIMINE_FRAMEBUFFER_RGB 1 + +struct limine_video_mode { + uint64_t pitch; + uint64_t width; + uint64_t height; + uint16_t bpp; + uint8_t memory_model; + uint8_t red_mask_size; + uint8_t red_mask_shift; + uint8_t green_mask_size; + uint8_t green_mask_shift; + uint8_t blue_mask_size; + uint8_t blue_mask_shift; +}; + +struct limine_framebuffer { + LIMINE_PTR(void *) address; + uint64_t width; + uint64_t height; + uint64_t pitch; + uint16_t bpp; + uint8_t memory_model; + uint8_t red_mask_size; + uint8_t red_mask_shift; + uint8_t green_mask_size; + uint8_t green_mask_shift; + uint8_t blue_mask_size; + uint8_t blue_mask_shift; + uint8_t unused[7]; + uint64_t edid_size; + LIMINE_PTR(void *) edid; + /* Response revision 1 */ + uint64_t mode_count; + LIMINE_PTR(struct limine_video_mode **) modes; +}; + +struct limine_framebuffer_response { + uint64_t revision; + uint64_t framebuffer_count; + LIMINE_PTR(struct limine_framebuffer **) framebuffers; +}; + +struct limine_framebuffer_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_framebuffer_response *) response; +}; + +/* Flanterm FB init params */ + +#define LIMINE_FLANTERM_FB_INIT_PARAMS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3259399fe7c5f126, 0xe01c1c8c5db9d1a9 } + +#define LIMINE_FLANTERM_FB_ROTATE_0 0 +#define LIMINE_FLANTERM_FB_ROTATE_90 1 +#define LIMINE_FLANTERM_FB_ROTATE_180 2 +#define LIMINE_FLANTERM_FB_ROTATE_270 3 + +struct limine_flanterm_fb_init_params { + LIMINE_PTR(uint32_t *) canvas; + uint64_t canvas_size; + uint32_t ansi_colours[8]; + uint32_t ansi_bright_colours[8]; + uint32_t default_bg; + uint32_t default_fg; + uint32_t default_bg_bright; + uint32_t default_fg_bright; + LIMINE_PTR(void *) font; + uint64_t font_width; + uint64_t font_height; + uint64_t font_spacing; + uint64_t font_scale_x; + uint64_t font_scale_y; + uint64_t margin; + uint64_t rotation; +}; + +struct limine_flanterm_fb_init_params_response { + uint64_t revision; + uint64_t entry_count; + LIMINE_PTR(struct limine_flanterm_fb_init_params **) entries; +}; + +struct limine_flanterm_fb_init_params_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_flanterm_fb_init_params_response *) response; +}; + +/* Paging mode */ + +#define LIMINE_PAGING_MODE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95c1a0edab0944cb, 0xa4e5cb3842f7488a } + +#define LIMINE_PAGING_MODE_X86_64_4LVL 0 +#define LIMINE_PAGING_MODE_X86_64_5LVL 1 +#define LIMINE_PAGING_MODE_X86_64_MIN LIMINE_PAGING_MODE_X86_64_4LVL +#define LIMINE_PAGING_MODE_X86_64_DEFAULT LIMINE_PAGING_MODE_X86_64_4LVL + +#define LIMINE_PAGING_MODE_AARCH64_4LVL 0 +#define LIMINE_PAGING_MODE_AARCH64_5LVL 1 +#define LIMINE_PAGING_MODE_AARCH64_MIN LIMINE_PAGING_MODE_AARCH64_4LVL +#define LIMINE_PAGING_MODE_AARCH64_DEFAULT LIMINE_PAGING_MODE_AARCH64_4LVL + +#define LIMINE_PAGING_MODE_RISCV_SV39 0 +#define LIMINE_PAGING_MODE_RISCV_SV48 1 +#define LIMINE_PAGING_MODE_RISCV_SV57 2 +#define LIMINE_PAGING_MODE_RISCV_MIN LIMINE_PAGING_MODE_RISCV_SV39 +#define LIMINE_PAGING_MODE_RISCV_DEFAULT LIMINE_PAGING_MODE_RISCV_SV48 + +#define LIMINE_PAGING_MODE_LOONGARCH_4LVL 0 +#define LIMINE_PAGING_MODE_LOONGARCH_MIN LIMINE_PAGING_MODE_LOONGARCH_4LVL +#define LIMINE_PAGING_MODE_LOONGARCH_DEFAULT LIMINE_PAGING_MODE_LOONGARCH_4LVL + +struct limine_paging_mode_response { + uint64_t revision; + uint64_t mode; +}; + +struct limine_paging_mode_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_paging_mode_response *) response; + uint64_t mode; + uint64_t max_mode; + uint64_t min_mode; +}; + +/* MP */ + +#define LIMINE_MP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95a67b819a1b857e, 0xa0b61b723b6a73e0 } + +struct limine_mp_info; + +typedef void (*limine_goto_address)(struct limine_mp_info *); + +#if defined (__x86_64__) || defined (__i386__) + +#define LIMINE_MP_RESPONSE_X86_64_X2APIC (1 << 0) + +struct limine_mp_info { + uint32_t processor_id; + uint32_t lapic_id; + uint64_t reserved; + LIMINE_PTR(limine_goto_address) goto_address; + uint64_t extra_argument; +}; + +struct limine_mp_response { + uint64_t revision; + uint32_t flags; + uint32_t bsp_lapic_id; + uint64_t cpu_count; + LIMINE_PTR(struct limine_mp_info **) cpus; +}; + +#elif defined (__aarch64__) + +struct limine_mp_info { + uint32_t processor_id; + uint32_t reserved1; + uint64_t mpidr; + uint64_t reserved; + LIMINE_PTR(limine_goto_address) goto_address; + uint64_t extra_argument; +}; + +struct limine_mp_response { + uint64_t revision; + uint64_t flags; + uint64_t bsp_mpidr; + uint64_t cpu_count; + LIMINE_PTR(struct limine_mp_info **) cpus; +}; + +#elif defined (__riscv) && (__riscv_xlen == 64) + +struct limine_mp_info { + uint64_t processor_id; + uint64_t hartid; + uint64_t reserved; + LIMINE_PTR(limine_goto_address) goto_address; + uint64_t extra_argument; +}; + +struct limine_mp_response { + uint64_t revision; + uint64_t flags; + uint64_t bsp_hartid; + uint64_t cpu_count; + LIMINE_PTR(struct limine_mp_info **) cpus; +}; + +#elif defined (__loongarch__) && (__loongarch_grlen == 64) + +struct limine_mp_info { + uint64_t processor_id; + uint64_t phys_id; + uint64_t reserved; + LIMINE_PTR(limine_goto_address) goto_address; + uint64_t extra_argument; +}; + +struct limine_mp_response { + uint64_t revision; + uint64_t flags; + uint64_t bsp_phys_id; + uint64_t cpu_count; + LIMINE_PTR(struct limine_mp_info **) cpus; +}; + +#else +#error Unknown architecture +#endif + +#define LIMINE_MP_REQUEST_X86_64_X2APIC (1 << 0) + +struct limine_mp_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_mp_response *) response; + uint64_t flags; +}; + +/* Memory map */ + +#define LIMINE_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x67cf3d9d378a806f, 0xe304acdfc50c3c62 } + +#define LIMINE_MEMMAP_USABLE 0 +#define LIMINE_MEMMAP_RESERVED 1 +#define LIMINE_MEMMAP_ACPI_RECLAIMABLE 2 +#define LIMINE_MEMMAP_ACPI_NVS 3 +#define LIMINE_MEMMAP_BAD_MEMORY 4 +#define LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE 5 +#define LIMINE_MEMMAP_EXECUTABLE_AND_MODULES 6 +#define LIMINE_MEMMAP_FRAMEBUFFER 7 +#define LIMINE_MEMMAP_RESERVED_MAPPED 8 + +struct limine_memmap_entry { + uint64_t base; + uint64_t length; + uint64_t type; +}; + +struct limine_memmap_response { + uint64_t revision; + uint64_t entry_count; + LIMINE_PTR(struct limine_memmap_entry **) entries; +}; + +struct limine_memmap_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_memmap_response *) response; +}; + +/* Entry point */ + +#define LIMINE_ENTRY_POINT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x13d86c035a1cd3e1, 0x2b0caa89d8f3026a } + +typedef void (*limine_entry_point)(void); + +struct limine_entry_point_response { + uint64_t revision; +}; + +struct limine_entry_point_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_entry_point_response *) response; + LIMINE_PTR(limine_entry_point) entry; +}; + +/* Executable File */ + +#define LIMINE_EXECUTABLE_FILE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xad97e90e83f1ed67, 0x31eb5d1c5ff23b69 } + +struct limine_executable_file_response { + uint64_t revision; + LIMINE_PTR(struct limine_file *) executable_file; +}; + +struct limine_executable_file_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_executable_file_response *) response; +}; + +/* Module */ + +#define LIMINE_MODULE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3e7e279702be32af, 0xca1c4f3bd1280cee } + +#define LIMINE_INTERNAL_MODULE_REQUIRED (1 << 0) +#define LIMINE_INTERNAL_MODULE_COMPRESSED (1 << 1) + +struct limine_internal_module { + LIMINE_PTR(const char *) path; + LIMINE_PTR(const char *) string; + uint64_t flags; +}; + +struct limine_module_response { + uint64_t revision; + uint64_t module_count; + LIMINE_PTR(struct limine_file **) modules; +}; + +struct limine_module_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_module_response *) response; + + /* Request revision 1 */ + uint64_t internal_module_count; + LIMINE_PTR(struct limine_internal_module **) internal_modules; +}; + +/* RSDP */ + +#define LIMINE_RSDP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xc5e77b6b397e7b43, 0x27637845accdcf3c } + +struct limine_rsdp_response { + uint64_t revision; + LIMINE_PTR(void *) address; +}; + +struct limine_rsdp_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_rsdp_response *) response; +}; + +/* SMBIOS */ + +#define LIMINE_SMBIOS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9e9046f11e095391, 0xaa4a520fefbde5ee } + +struct limine_smbios_response { + uint64_t revision; + LIMINE_PTR(void *) entry_32; + LIMINE_PTR(void *) entry_64; +}; + +struct limine_smbios_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_smbios_response *) response; +}; + +/* EFI system table */ + +#define LIMINE_EFI_SYSTEM_TABLE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x5ceba5163eaaf6d6, 0x0a6981610cf65fcc } + +struct limine_efi_system_table_response { + uint64_t revision; + LIMINE_PTR(void *) address; +}; + +struct limine_efi_system_table_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_efi_system_table_response *) response; +}; + +/* TPM event log */ + +#define LIMINE_TPM_EVENT_LOG_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x98e094fc7e76e979, 0xee8d8775c54e1d1f } + +#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_1_2 1 +#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_2 2 + +struct limine_tpm_event_log_response { + uint64_t revision; + uint64_t format; + uint64_t size; + LIMINE_PTR(void *) address; +}; + +struct limine_tpm_event_log_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_tpm_event_log_response *) response; +}; + +/* EFI memory map */ + +#define LIMINE_EFI_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x7df62a431d6872d5, 0xa4fcdfb3e57306c8 } + +struct limine_efi_memmap_response { + uint64_t revision; + LIMINE_PTR(void *) memmap; + uint64_t memmap_size; + uint64_t desc_size; + uint64_t desc_version; +}; + +struct limine_efi_memmap_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_efi_memmap_response *) response; +}; + +/* Date at boot */ + +#define LIMINE_DATE_AT_BOOT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x502746e184c088aa, 0xfbc5ec83e6327893 } + +struct limine_date_at_boot_response { + uint64_t revision; + int64_t timestamp; +}; + +struct limine_date_at_boot_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_date_at_boot_response *) response; +}; + +/* Executable address */ + +#define LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x71ba76863cc55f63, 0xb2644a48c516a487 } + +struct limine_executable_address_response { + uint64_t revision; + uint64_t physical_base; + uint64_t virtual_base; +}; + +struct limine_executable_address_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_executable_address_response *) response; +}; + +/* Device Tree Blob */ + +#define LIMINE_DTB_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xb40ddb48fb54bac7, 0x545081493f81ffb7 } + +struct limine_dtb_response { + uint64_t revision; + LIMINE_PTR(void *) dtb_ptr; +}; + +struct limine_dtb_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_dtb_response *) response; +}; + +/* RISC-V Boot Hart ID */ + +#define LIMINE_RISCV_BSP_HARTID_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x1369359f025525f9, 0x2ff2a56178391bb6 } + +struct limine_riscv_bsp_hartid_response { + uint64_t revision; + uint64_t bsp_hartid; +}; + +struct limine_riscv_bsp_hartid_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_riscv_bsp_hartid_response *) response; +}; + +/* Bootloader Performance */ + +#define LIMINE_BOOTLOADER_PERFORMANCE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x6b50ad9bf36d13ad, 0xdc4c7e88fc759e17 } + +struct limine_bootloader_performance_response { + uint64_t revision; + uint64_t reset_usec; + uint64_t init_usec; + uint64_t exec_usec; +}; + +struct limine_bootloader_performance_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_bootloader_performance_response *) response; +}; + +#define LIMINE_X86_64_KEEP_IOMMU_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8ebaabe51f490179, 0x2aa86a59ffb4ab0f } + +struct limine_x86_64_keep_iommu_response { + uint64_t revision; +}; + +struct limine_x86_64_keep_iommu_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_x86_64_keep_iommu_response *) response; +}; + +/* TSC (Timestamp Counter) Frequency */ + +#define LIMINE_TSC_FREQUENCY_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x10f2ee1d87d195e4, 0xf747a2b78f6ddb31 } + +struct limine_tsc_frequency_response { + uint64_t revision; + uint64_t frequency; +}; + +struct limine_tsc_frequency_request { + uint64_t id[4]; + uint64_t revision; + LIMINE_PTR(struct limine_tsc_frequency_response *) response; +}; + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/boot/limine/meson.build b/boot/limine/meson.build new file mode 100644 index 0000000..527e9ba --- /dev/null +++ b/boot/limine/meson.build @@ -0,0 +1 @@ +kernel_sources += files('entry.c') diff --git a/boot/main.c b/boot/main.c new file mode 100644 index 0000000..6f8733b --- /dev/null +++ b/boot/main.c @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char zinnia_banner[] = + "Zinnia " ZINNIA_VERSION " (" ZINNIA_ARCH ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")"; + +static void kernel_main_task(uintptr_t arg0) { + struct boot_info* info = (struct boot_info*)arg0; + printf("%s\n", zinnia_banner); // Say hello! + + assert(info->num_files >= 1, "No init executable provided\n"); + + struct vas* init_space; + assert(vma_new(&init_space) == 0, ""); + + struct boot_file* file_data = &info->files[0]; + struct paged_vmo* init_file; + assert(vmo_new_phys(&init_file) == 0, ""); + vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr); + + // Load init. + const char* init_argv[] = {"init", nullptr}; + const char* init_envp[] = {nullptr}; + + struct task* init_task = nullptr; + errno_t status = 0; + assert(status == 0, "Failed to load init process (%i)\n", status); + sched_add_task(&percpu_get()->sched, init_task); + + while (true) { + arch_irq_wait(); + } +} + +[[noreturn]] +void kmain(struct boot_info* info) { + cmdline_parse(info->cmdline); + + console_init(); + mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base); + rsdp_addr = info->rsdp; // TODO + percpu_init(); + sched_init(&percpu_get()->sched); + futex_init(); + + struct task* main_task; + task_create("main", &kernel_vas, nullptr, kernel_main_task, (uintptr_t)info, &main_task); + sched_add_task(&percpu_get()->sched, main_task); + + irq_unlock(); + sched_yield(&percpu_get()->sched); + + unreachable(); +} + +[[__init]] +void kernel_early_init() { + percpu_bsp_init(); + percpu_get()->online = true; + irq_lock(); +} diff --git a/boot/meson.build b/boot/meson.build new file mode 100644 index 0000000..5276801 --- /dev/null +++ b/boot/meson.build @@ -0,0 +1,6 @@ +kernel_sources += files( + 'cmdline.c', + 'main.c', +) + +subdir('limine') diff --git a/core/clock.c b/core/clock.c new file mode 100644 index 0000000..865afe0 --- /dev/null +++ b/core/clock.c @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include +#include + +static struct clock* active_clock = nullptr; +static uint64_t counter_base = 0; +static struct spinlock lock = {}; + +bool clock_available() { + return active_clock != nullptr; +} + +uint64_t clock_get_elapsed_ns() { + struct clock* active = atomic_load(&active_clock); + if (__unlikely(!active)) + return 0; + + return atomic_load(&counter_base) + active->get_elapsed_ns(active); +} + +bool clock_switch(struct clock* clock) { + spin_lock(&lock); + + if (active_clock) { + if (clock->priority <= active_clock->priority) { + spin_unlock(&lock); + return false; + } + } + + printf("Switching to clock source \"%s\"\n", clock->name); + + uint64_t elapsed = clock_get_elapsed_ns(); + counter_base += elapsed; + + active_clock = clock; + spin_unlock(&lock); + return true; +} + +void clock_spin_ns(uint64_t ns) { + if (active_clock == nullptr) { + printf("Unable to sleep for %lu ns, no clock available!\n", ns); + return; + } + + uint64_t target = clock_get_elapsed_ns() + ns; + while (clock_get_elapsed_ns() < target) { + // TODO: Maybe pause? + } +} diff --git a/core/console.c b/core/console.c new file mode 100644 index 0000000..6849ad0 --- /dev/null +++ b/core/console.c @@ -0,0 +1,43 @@ +#include +#include +#include +#include + +extern struct console* const __ld_console_start[]; +extern struct console* const __ld_console_end[]; + +struct console* active = nullptr; + +void console_write(const char* buf, size_t len) { + if (__likely(active)) + active->write(active, buf, len); +} + +static void console_setup(const char* value) { + if (!value) + return; + + // Find a suitable boot console. + struct console* const* cur = __ld_console_start; + while (cur < __ld_console_end) { + if (!strcmp(value, (*cur)->name)) { + active = *cur; + } + cur++; + } +} + +CMDLINE_OPTION("console", console_setup); + +void console_init() { + // If we have no console set and at least one set, use that as a default. + if (!active && &__ld_console_start[0] != &__ld_console_end[0]) + active = __ld_console_start[0]; + + if (active) { + if (active->init(active) == false) + active = nullptr; + else + printf("Logging on console \"%s\"\n", active->name); + } +} diff --git a/core/elf.c b/core/elf.c new file mode 100644 index 0000000..ed3e53f --- /dev/null +++ b/core/elf.c @@ -0,0 +1,200 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +errno_t elf_load(struct exec_info* info, struct task** out) { + errno_t status; + + size_t read; + struct elf_ehdr ehdr = {}; + vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read); + + // Check if the file is an ELF. + if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0) + return EINVAL; + if (ehdr.e_ident[EI_VERSION] != EV_CURRENT) + return EINVAL; + if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS) + return EINVAL; + if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA) + return EINVAL; + if (ehdr.e_machine != ELF_ARCH_MACHINE) + return EINVAL; + if (ehdr.e_type != ET_EXEC) + return EINVAL; + + const size_t page_size = arch_mem_page_size(); + + for (size_t i = 0; i < ehdr.e_phnum; i++) { + struct elf_phdr phdr = {}; + vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read); + if (phdr.p_type == PT_LOAD) { + enum zn_vm_flags prot = 0; + if (phdr.p_flags & PF_R) + prot |= ZN_VM_MAP_READ; + if (phdr.p_flags & PF_W) + prot |= ZN_VM_MAP_WRITE; + if (phdr.p_flags & PF_X) + prot |= ZN_VM_MAP_EXEC; + + assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, ""); + + const uintptr_t misalign = phdr.p_vaddr & (page_size - 1); + const uintptr_t map_address = phdr.p_vaddr - misalign; + const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1); + const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1); + + // Copy the file data into its own mapping. + struct paged_vmo* phdr_obj; + status = vmo_new_phys(&phdr_obj); + if (status) + return status; + + status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr); + if (status) + return status; + + // We map more than we copied so the rest is filled with zeroed pages. + status = vma_map_vmo(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset); + if (status) + return status; + } + } + + const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size; + const size_t stack_size = 2 * 1024 * 1024; // 2MiB + const uintptr_t stack_start = highest - stack_size; + + // Allocate stack. + struct paged_vmo* stack; + status = vmo_new_phys(&stack); + if (status) + return status; + + // Fill in stack info. + uintptr_t stack_off = stack_size; + + // TODO: Instead of allocating, calculate it on the fly. + size_t num_envp = 0; + for (const char** ptr = info->envp; *ptr != nullptr; ptr++) { + num_envp++; + } + size_t num_argv = 0; + for (const char** ptr = info->argv; *ptr != nullptr; ptr++) { + num_argv++; + } + uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0); + uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0); + + for (size_t env = 0; env < num_envp; env++) { + const char nul = 0; + stack_off -= 1; + vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr); + + const size_t len = strlen(info->envp[env]); + stack_off -= len; + vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr); + + envp_offsets[env] = stack_start + stack_off; + } + + for (size_t arg = 0; arg < num_argv; arg++) { + const char nul = 0; + stack_off -= 1; + vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr); + + const size_t len = strlen(info->argv[arg]); + stack_off -= len; + vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr); + + argv_offsets[arg] = stack_start + stack_off; + } + + stack_off = ALIGN_DOWN(stack_off, 16); + + // Align the stack if argc + argv + envp does not add up to 16 byte alignment. + if ((1 + num_argv + num_envp) & 1) { + stack_off -= sizeof(uintptr_t); + uintptr_t zero = 0; + vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); + } + + // Add the init handle to the process namespace. + zn_handle_t init_handle = ZN_HANDLE_INVALID; + if (info->init_handle) { + status = namespace_add_desc(info->ns, *info->init_handle, &init_handle); + if (status) + return status; + } + +#define WRITE_AUXV(auxv, value) \ + do { \ + stack_off -= sizeof(uintptr_t); \ + uintptr_t auxv_val = value; \ + vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ + stack_off -= sizeof(uintptr_t); \ + auxv_val = auxv; \ + vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ + } while (0) + + // Write auxiliary values. + WRITE_AUXV(AT_NULL, 0); + WRITE_AUXV(AT_SECURE, 0); + WRITE_AUXV(AT_PHNUM, ehdr.e_phnum); + WRITE_AUXV(AT_PHENT, ehdr.e_phentsize); + WRITE_AUXV(AT_ENTRY, ehdr.e_entry); + WRITE_AUXV(AT_INIT_HANDLE, init_handle); + + // envp pointers. + stack_off -= sizeof(uintptr_t); + const uintptr_t zero = 0; + vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); + for (size_t env = 0; env < num_envp; env++) { + stack_off -= sizeof(uintptr_t); + vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr); + } + + // argv pointers. + stack_off -= sizeof(uintptr_t); + vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr); + for (size_t arg = 0; arg < num_argv; arg++) { + stack_off -= sizeof(uintptr_t); + vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr); + } + + mem_free(envp_offsets); + mem_free(argv_offsets); + + // argc + stack_off -= sizeof(uintptr_t); + uintptr_t argc = num_argv; + vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr); + + status = vma_map_vmo(info->space, &stack->object, stack_start, stack_size, ZN_VM_MAP_READ | ZN_VM_MAP_WRITE, 0); + if (status) + return status; + + struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0); + start_info->ip = ehdr.e_entry; + start_info->sp = highest - stack_size + stack_off; + start_info->arg = 0; + + struct task* result; + status = task_create(info->argv[0], info->space, info->ns, sched_to_user, (uintptr_t)start_info, &result); + if (status) + return status; + + *out = result; + return 0; +} diff --git a/core/futex.c b/core/futex.c new file mode 100644 index 0000000..4f553ab --- /dev/null +++ b/core/futex.c @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include + +struct futex_bucket { + struct wait_queue wq; + uintptr_t key; // Virtual address used as the futex key. + bool in_use; +}; + +#define FUTEX_BUCKET_COUNT 64 + +static struct futex_bucket futex_table[FUTEX_BUCKET_COUNT]; +static struct spinlock futex_table_lock; + +void futex_init(void) { + futex_table_lock = (struct spinlock){0}; + for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) { + wait_queue_init(&futex_table[i].wq); + futex_table[i].key = 0; + futex_table[i].in_use = false; + } +} + +static size_t futex_hash(uintptr_t addr) { + // Simple hash: mix the address bits and mod by bucket count. + addr = (addr >> 2) ^ (addr >> 12); + return addr % FUTEX_BUCKET_COUNT; +} + +static struct futex_bucket* futex_bucket_get(uintptr_t addr) { + size_t idx = futex_hash(addr); + // Linear probing to find a matching or free bucket. + for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) { + size_t slot = (idx + i) % FUTEX_BUCKET_COUNT; + struct futex_bucket* b = &futex_table[slot]; + if (b->in_use && b->key == addr) + return b; + if (!b->in_use) { + b->key = addr; + b->in_use = true; + return b; + } + } + return nullptr; +} + +errno_t futex_wait(__user int* addr, int expected) { + uintptr_t key = (uintptr_t)addr; + + spin_lock(&futex_table_lock); + + // Read the current value at the userspace address. + int current; + if (!usercopy_read(¤t, addr, sizeof(*addr))) { + spin_unlock(&futex_table_lock); + return EFAULT; + } + + // If the value has already changed, don't block. + if (current != expected) { + spin_unlock(&futex_table_lock); + return 0; + } + + struct futex_bucket* bucket = futex_bucket_get(key); + if (!bucket) { + spin_unlock(&futex_table_lock); + return ENOMEM; + } + + spin_unlock(&futex_table_lock); + + // Block on the bucket's wait queue. + struct sched_percpu* sched = &percpu_get()->sched; + sched_block(sched, &bucket->wq); + + return 0; +} + +errno_t futex_wake(__user int* addr, int count) { + uintptr_t key = (uintptr_t)addr; + + spin_lock(&futex_table_lock); + + size_t idx = futex_hash(key); + struct futex_bucket* bucket = nullptr; + for (size_t i = 0; i < FUTEX_BUCKET_COUNT; i++) { + size_t slot = (idx + i) % FUTEX_BUCKET_COUNT; + struct futex_bucket* b = &futex_table[slot]; + if (b->in_use && b->key == key) { + bucket = b; + break; + } + if (!b->in_use) + break; + } + + if (!bucket) { + spin_unlock(&futex_table_lock); + return 0; // No waiters, nothing to do. + } + + spin_unlock(&futex_table_lock); + + struct sched_percpu* sched = &percpu_get()->sched; + if (count == INT32_MAX) { + sched_wake_all(sched, &bucket->wq); + } else { + for (int i = 0; i < count; i++) { + sched_wake(sched, &bucket->wq); + } + } + + return 0; +} diff --git a/core/irq.c b/core/irq.c new file mode 100644 index 0000000..ba56e44 --- /dev/null +++ b/core/irq.c @@ -0,0 +1,28 @@ +#include +#include +#include + +void irq_lock() { + struct irq_percpu* cpu = &percpu_get()->irq; + if (atomic_load_explicit(&cpu->in_interrupt, memory_order_acquire)) + return; + + arch_irq_set_state(false); + atomic_fetch_add_explicit(&percpu_get()->irq.level, 1, memory_order_acq_rel); +} + +void irq_unlock() { + struct irq_percpu* cpu = &percpu_get()->irq; + if (atomic_load_explicit(&cpu->in_interrupt, memory_order_acquire)) + return; + + uint32_t old_level = atomic_fetch_sub_explicit(&cpu->level, 1, memory_order_acq_rel); + // If it was 1, the new IRQ level is now 0. + if (old_level == 1) { + arch_irq_set_state(true); + } +} + +bool irq_set_interrupted(bool is_interrupted) { + return atomic_exchange_explicit(&percpu_get()->irq.in_interrupt, is_interrupted, memory_order_release); +} diff --git a/core/meson.build b/core/meson.build new file mode 100644 index 0000000..bde4c8f --- /dev/null +++ b/core/meson.build @@ -0,0 +1,16 @@ +kernel_sources += files( + 'clock.c', + 'console.c', + 'elf.c', + 'futex.c', + 'irq.c', + 'panic.c', + 'percpu.c', + 'print.c', + 'spin.c', + 'uacpi_glue.c', +) + +if get_option('sched') == 'simple' + kernel_sources += files('sched_simple.c') +endif diff --git a/core/panic.c b/core/panic.c new file mode 100644 index 0000000..984bfc1 --- /dev/null +++ b/core/panic.c @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include + +[[noreturn]] +void panic(const char* msg, ...) { + // TODO: Stop all other CPUs with IPI. + arch_irq_set_state(false); + + struct percpu* cpu = percpu_get(); + struct task* current = cpu->sched.current; + printf( + "----[ Kernel panic ]----\n" + "In task \"%s\" (TID %zu) on CPU %zu!\n", + current ? current->name : nullptr, + current ? current->id : 0, + cpu->id + ); + + va_list args; + va_start(args, msg); + kvprintf(msg, args); + va_end(args); + + printf("----[ End of panic ]----\n"); + + while (1) { + arch_irq_wait(); + } +} diff --git a/core/percpu.c b/core/percpu.c new file mode 100644 index 0000000..6441d63 --- /dev/null +++ b/core/percpu.c @@ -0,0 +1,14 @@ +#include +#include + +[[__used, __section(".percpu")]] +struct percpu percpu_bsp = { + .self = &percpu_bsp, + .id = 0, + .online = false, +}; + +struct percpu* percpu_new() { + // TODO + return nullptr; +} diff --git a/core/print.c b/core/print.c new file mode 100644 index 0000000..2cfae29 --- /dev/null +++ b/core/print.c @@ -0,0 +1,399 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +static void reverse(char* s) { + int i, j; + char c; + + for (i = 0, j = strlen(s) - 1; i < j; i++, j--) { + c = s[i]; + s[i] = s[j]; + s[j] = c; + } +} + +static int char_to_digit(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'z') + return c - 'a' + 10; + if (c >= 'A' && c <= 'Z') + return c - 'A' + 10; + return -1; +} + +unsigned long atolu(const char* str, int base) { + unsigned long result = 0; + + if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) { + str += 2; + } + + while (*str) { + int digit = char_to_digit(*str++); + if (digit < 0 || digit >= base) + break; + + result = result * base + digit; + } + + return result; +} + +long atol(const char* str, int base) { + int neg = 0; + + if (*str == '-') { + neg = 1; + str++; + } + + unsigned long val = atolu(str, base); + return neg ? -(long)val : (long)val; +} + +char* lutoa(uint64_t value, char* str, int base) { + if (base < 2 || base > 36) + return nullptr; + + size_t i = 0; + + do { + uint64_t digit = value % base; + str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10); + value /= base; + } while (value); + + str[i] = '\0'; + reverse(str); + return str; +} + +char* ltoa(int64_t value, char* str, int base) { + if (base < 2 || base > 36) + return nullptr; + + uint64_t magnitude; + int negative = 0; + + if (value < 0) { + negative = 1; + magnitude = (uint64_t)(-(value + 1)) + 1; + // avoids INT64_MIN overflow + } else { + magnitude = (uint64_t)value; + } + + size_t i = 0; + + do { + uint64_t digit = magnitude % base; + str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10); + magnitude /= base; + } while (magnitude); + + if (negative) + str[i++] = '-'; + + str[i] = '\0'; + reverse(str); + return str; +} + +typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len); + +static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) { + // Amount of bytes written. + int32_t written = 0; + + while (*fmt != '\0') { + size_t maxrem = INT32_MAX - written; + + if (fmt[0] != '%' || fmt[1] == '%') { + if (fmt[0] == '%') + fmt++; + size_t amount = 1; + while (fmt[amount] && fmt[amount] != '%') + amount++; + if (maxrem < amount) { + return; + } + callback(ctx, fmt, amount); + fmt += amount; + written += amount; + continue; + } + + const char* format_begun_at = fmt++; + bool force_sign = false; + bool blank_sign = false; + bool write_prefix = false; + bool pad_zero = false; + bool has_width = false; + size_t width = 0; + bool has_precision = false; + size_t precision = 0; + size_t size = sizeof(uint32_t); + char number[32]; +check_fmt: + memset(number, 0, sizeof(number)); + + switch (*fmt) { + // Flags + case '+': { + force_sign = true; + fmt++; + goto check_fmt; + } + case ' ': { + blank_sign = true; + fmt++; + goto check_fmt; + } + case '#': { + write_prefix = true; + fmt++; + goto check_fmt; + } + case '0': { + pad_zero = true; + fmt++; + goto check_fmt; + } + // Width + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + has_width = true; + char number[10 + 1] = {0}; + size_t idx = 0; + while (*fmt >= '0' && *fmt <= '9') { + number[idx++] = *fmt; + fmt++; + } + width = atolu(number, 10); + + goto check_fmt; + } + // Precision + case '.': { + has_precision = true; + fmt++; + if (*fmt == '*') { + precision = (char)va_arg(args, int32_t); + } else { + char number[10 + 1] = {0}; + size_t idx = 0; + while (*fmt >= '0' && *fmt <= '9') { + number[idx++] = *fmt; + fmt++; + } + precision = atolu(number, 10); + } + goto check_fmt; + } + // Length + case 'h': { + if (fmt[1] == 'h') { + size = sizeof(uint8_t); + fmt++; + } else + size = sizeof(uint16_t); + fmt++; + goto check_fmt; + } + case 'l': { + // There is no difference between `long int` and `long long int`. + if (fmt[1] == 'l') + fmt++; + fmt++; + size = sizeof(uint64_t); + + goto check_fmt; + } + case 'z': { + size = sizeof(size_t); + fmt++; + goto check_fmt; + } + // Character + case 'c': { + char c = (char)va_arg(args, int32_t); + + callback(ctx, &c, sizeof(c)); + + written++; + fmt++; + break; + } + // String of characters + case 's': { + const char* str = va_arg(args, const char*); + if (!str) + str = "(null)"; + size_t len = 0; + if (has_precision) + len = strnlen(str, precision); + else + len = strlen(str); + + callback(ctx, str, len); + + written += len; + fmt++; + break; + } + // Signed decimal integer + case 'i': + case 'd': { + switch (size) { + case sizeof(int8_t): + ltoa((int8_t)va_arg(args, int32_t), number, 10); + break; + case sizeof(int16_t): + ltoa((int16_t)va_arg(args, int32_t), number, 10); + break; + case sizeof(int32_t): + ltoa((int32_t)va_arg(args, int32_t), number, 10); + break; + default: + ltoa(va_arg(args, int64_t), number, 10); + break; + } + goto print_num; + } + // Unsigned decimal integer + case 'u': { + switch (size) { + case sizeof(uint8_t): + lutoa((uint8_t)va_arg(args, uint32_t), number, 10); + break; + case sizeof(uint16_t): + lutoa((uint16_t)va_arg(args, uint32_t), number, 10); + break; + case sizeof(uint32_t): + lutoa(va_arg(args, uint32_t), number, 10); + break; + case sizeof(uint64_t): + lutoa(va_arg(args, uint64_t), number, 10); + break; + } + goto print_num; + } + // Unsigned hexadecimal integer + case 'X': + case 'x': { + switch (size) { + case sizeof(uint8_t): + lutoa((uint8_t)va_arg(args, uint32_t), number, 16); + break; + case sizeof(uint16_t): + lutoa((uint16_t)va_arg(args, uint32_t), number, 16); + break; + case sizeof(uint32_t): + lutoa(va_arg(args, uint32_t), number, 16); + break; + case sizeof(uint64_t): + lutoa(va_arg(args, uint64_t), number, 16); + break; + } +print_num: + size_t len = strlen(number); + if (has_precision) { + for (size_t i = 0; i < precision - len; i++) + callback(ctx, " ", 1); + } + if (has_width) { + char c = pad_zero ? '0' : ' '; + + // We don't have to pad anything. + if (len < width) { + for (size_t i = 0; i < width - len; i++) + callback(ctx, &c, 1); + } + } + if (write_prefix) { + callback(ctx, "0x", 2); + } + if (force_sign) { + char c = '+'; + if (number[0] != '-') + callback(ctx, &c, 1); + } + if (blank_sign) { + char c = ' '; + if (number[0] != '-') + callback(ctx, &c, 1); + } + callback(ctx, number, len); + + written += len; + fmt++; + break; + } + // Pointer address + case 'p': { + const size_t num = va_arg(args, size_t); + const size_t buf_size = sizeof(size_t) * 2 + 1; + char str[buf_size]; + lutoa(num, number, 0x10); + const size_t len = strlen(number); // Get the length of the final number. + for (size_t i = 0; i < buf_size; i++) + str[i] = '0'; + size_t offset = sizeof(str) - len - 1; + + callback(ctx, str, offset); + callback(ctx, number, len); + + fmt++; + written += len; + break; + } + // The number of characters written so far is stored in the pointed location. + case 'n': { + int32_t* const ptr = va_arg(args, int32_t*); + *ptr = written; + fmt++; + break; + } + // No format, just normal text. + default: { + fmt = format_begun_at; + size_t len = strlen(fmt); + + callback(ctx, fmt, len); + + written += len; + fmt += len; + break; + } + } + } +} + +static void printf_console(void*, const char* msg, size_t len) { + console_write(msg, len); +} + +void printf(const char* message, ...) { + va_list args; + va_start(args, message); + do_print(printf_console, nullptr, message, args); + va_end(args); +} + +void kvprintf(const char* message, va_list args) { + do_print(printf_console, nullptr, message, args); +} diff --git a/core/sched_simple.c b/core/sched_simple.c new file mode 100644 index 0000000..dce4582 --- /dev/null +++ b/core/sched_simple.c @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void idle_func(uintptr_t) { + while (1) { + arch_irq_set_state(true); + arch_irq_wait(); + } +} + +void sched_to_user(uintptr_t info_ptr) { + struct task_start_info* info = (struct task_start_info*)info_ptr; + uintptr_t ip = info->ip; + uintptr_t sp = info->sp; + uintptr_t arg = info->arg; + mem_free(info); + arch_sched_jump_to_user(ip, sp, arg); +} + +void sched_to_user_context(uintptr_t ctx) { + arch_sched_jump_to_context((struct arch_context*)ctx); +} + +void sched_init(struct sched_percpu* sched) { + struct task* idle_task; + assert(task_create("idle", &kernel_vas, nullptr, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n"); + sched->idle_task = idle_task; + sched->current = idle_task; + + tailq_init(&sched->run_queue); + + printf("Scheduler initialized\n"); +} + +void sched_add_task(struct sched_percpu* sched, struct task* task) { + assert(task != nullptr, "Tried to add NULL to the run queue\n"); + tailq_insert_tail(&sched->run_queue, task, next); +} + +static struct task* next_task(struct sched_percpu* sched) { + struct task* t = tailq_first(&sched->run_queue); + + if (t == nullptr) + return sched->idle_task; + + tailq_remove(&sched->run_queue, t, next); + + return t; +} + +static void do_reschedule(struct sched_percpu* sched) { + struct task* from = atomic_load_explicit(&sched->current, memory_order_acquire); + struct task* to = next_task(sched); + + if (from == to) { + irq_unlock(); + return; + } + + atomic_store_explicit(&sched->current, to, memory_order_relaxed); + + pmap_set(&to->space->pmap); + + struct percpu* cpu = percpu_get(); + from->kernel_stack = cpu->kernel_stack; + from->user_stack = cpu->user_stack; + cpu->kernel_stack = to->kernel_stack; + cpu->user_stack = to->user_stack; + + arch_sched_switch(from, to); +} + +void sched_reschedule(struct sched_percpu* sched) { + irq_lock(); + if (sched->current->state != TASK_STATE_DEAD) + sched_add_task(sched, sched->current); + do_reschedule(sched); +} + +void sched_yield(struct sched_percpu* sched) { + irq_lock(); + do_reschedule(sched); +} + +errno_t task_create( + const char* name, + struct vas* space, + struct namespace* ns, + task_fn_t entry, + uintptr_t arg0, + struct task** out +) { + if (!out) + return EINVAL; + + struct task* new_task = mem_alloc(sizeof(struct task), 0); + if (new_task == nullptr) + return ENOMEM; + + static size_t last_tid = 0; + new_task->id = atomic_fetch_add(&last_tid, 1); + new_task->space = space; + new_task->namespace = ns; + strncpy(new_task->name, name, sizeof(new_task->name)); + + // Allocate a kernel stack. + new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0); + if (!new_task->kernel_stack) + return ENOMEM; + new_task->user_stack = 0; + + // Initialize the arch-dependent parts. + errno_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true); + if (s != 0) + return s; + + *out = new_task; + return 0; +} + +void task_entry(task_fn_t entry, uintptr_t arg0) { + entry(arg0); + + struct sched_percpu* sched = &percpu_get()->sched; + sched->current->state = TASK_STATE_DEAD; + sched_yield(sched); + panic("unreachable"); +} + +void wait_queue_init(struct wait_queue* wq) { + tailq_init(&wq->waiters); + wq->lock = (struct spinlock){0}; +} + +void sched_block(struct sched_percpu* sched, struct wait_queue* wq) { + irq_lock(); + spin_lock(&wq->lock); + + sched->current->state = TASK_STATE_BLOCKED; + tailq_insert_tail(&wq->waiters, sched->current, next); + + spin_unlock(&wq->lock); + do_reschedule(sched); +} + +void sched_wake(struct sched_percpu* sched, struct wait_queue* wq) { + spin_lock(&wq->lock); + + struct task* t = tailq_first(&wq->waiters); + if (t != nullptr) { + tailq_remove(&wq->waiters, t, next); + t->state = TASK_STATE_READY; + sched_add_task(sched, t); + } + + spin_unlock(&wq->lock); +} + +void sched_wake_all(struct sched_percpu* sched, struct wait_queue* wq) { + spin_lock(&wq->lock); + + struct task* t; + while ((t = tailq_first(&wq->waiters)) != nullptr) { + tailq_remove(&wq->waiters, t, next); + t->state = TASK_STATE_READY; + sched_add_task(sched, t); + } + + spin_unlock(&wq->lock); +} diff --git a/core/spin.c b/core/spin.c new file mode 100644 index 0000000..13d9d90 --- /dev/null +++ b/core/spin.c @@ -0,0 +1,15 @@ +#include +#include + +void spin_lock(struct spinlock* spin) { + uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed); + + while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) { + asm volatile("pause"); + } +} + +void spin_unlock(struct spinlock* spin) { + uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed); + atomic_store_explicit(&spin->owner, val + 1, memory_order_release); +} diff --git a/core/uacpi_glue.c b/core/uacpi_glue.c new file mode 100644 index 0000000..b1a872b --- /dev/null +++ b/core/uacpi_glue.c @@ -0,0 +1,34 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) { + *out_rsdp_address = (uacpi_phys_addr)rsdp_addr; + return UACPI_STATUS_OK; +} + +void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) { + return mmio_new(addr, len); +} + +void uacpi_kernel_unmap(void* addr, uacpi_size len) { + mmio_free(addr, len); +} + +void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) { + va_list args; + va_start(args, msg); + + kvprintf(msg, args); + + va_end(args); +} + +void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) { + kvprintf(msg, args); +} diff --git a/device/acpi/include/uacpi_types.h b/device/acpi/include/uacpi_types.h new file mode 100644 index 0000000..3bfebb1 --- /dev/null +++ b/device/acpi/include/uacpi_types.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +typedef uint8_t uacpi_u8; +typedef uint16_t uacpi_u16; +typedef uint32_t uacpi_u32; +typedef uint64_t uacpi_u64; + +typedef int8_t uacpi_i8; +typedef int16_t uacpi_i16; +typedef int32_t uacpi_i32; +typedef int64_t uacpi_i64; + +#define UACPI_TRUE true +#define UACPI_FALSE false +typedef bool uacpi_bool; + +#define UACPI_NULL nullptr + +typedef uintptr_t uacpi_uintptr; +typedef uacpi_uintptr uacpi_virt_addr; +typedef size_t uacpi_size; + +typedef va_list uacpi_va_list; +#define uacpi_va_start va_start +#define uacpi_va_end va_end +#define uacpi_va_arg va_arg + +typedef char uacpi_char; + +#define uacpi_offsetof offsetof + +#define UACPI_PRIu64 "llu" +#define UACPI_PRIx64 "llx" +#define UACPI_PRIX64 "llX" +#define UACPI_FMT64(val) ((unsigned long long)(val)) diff --git a/device/acpi/meson.build b/device/acpi/meson.build new file mode 100644 index 0000000..a2903fa --- /dev/null +++ b/device/acpi/meson.build @@ -0,0 +1,10 @@ +# uACPI +uacpi = subproject('uacpi') +uacpi_sources = uacpi.get_variable('sources') +uacpi_includes = uacpi.get_variable('includes') +kernel_c_args += [ + '-DUACPI_FORMATTED_LOGGING', + '-DUACPI_BAREBONES_MODE', + '-DUACPI_OVERRIDE_TYPES', +] +kernel_includes += include_directories('include') diff --git a/device/meson.build b/device/meson.build new file mode 100644 index 0000000..1b0ad7c --- /dev/null +++ b/device/meson.build @@ -0,0 +1,3 @@ +if get_option('acpi') + subdir('acpi') +endif diff --git a/fs/meson.build b/fs/meson.build new file mode 100644 index 0000000..7578f28 --- /dev/null +++ b/fs/meson.build @@ -0,0 +1 @@ +subdir('tmpfs') diff --git a/fs/tmpfs/meson.build b/fs/tmpfs/meson.build new file mode 100644 index 0000000..4e4da64 --- /dev/null +++ b/fs/tmpfs/meson.build @@ -0,0 +1 @@ +kernel_sources += files('tmpfs.c') diff --git a/fs/tmpfs/tmpfs.c b/fs/tmpfs/tmpfs.c new file mode 100644 index 0000000..70b786d --- /dev/null +++ b/fs/tmpfs/tmpfs.c @@ -0,0 +1 @@ +// TODO diff --git a/include/assert.h b/include/assert.h new file mode 100644 index 0000000..4b36f1c --- /dev/null +++ b/include/assert.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#define assert(expr, msg, ...) \ + ({ \ + if (__unlikely(!(expr))) { \ + panic( \ + "In function \"%s\" (%s:%u):\n" \ + "Assertion \"%s\" failed!\n" msg, \ + __FUNCTION__, \ + __FILE__, \ + __LINE__, \ + #expr, \ + ##__VA_ARGS__ \ + ); \ + } \ + }) + +#ifndef NDEBUG +#define debug_assert(...) assert(__VA_ARGS__) +#else +#define debug_assert(...) ((void)0) +#endif diff --git a/include/stdarg.h b/include/stdarg.h new file mode 100644 index 0000000..af6bebe --- /dev/null +++ b/include/stdarg.h @@ -0,0 +1,7 @@ +#pragma once + +typedef __builtin_va_list va_list; +#define va_start(v, l) __builtin_va_start(v, l) +#define va_end(v) __builtin_va_end(v) +#define va_arg(v, T) __builtin_va_arg(v, T) +#define va_copy(d, s) __builtin_va_copy(d, s) diff --git a/include/stdatomic.h b/include/stdatomic.h new file mode 100644 index 0000000..a34720f --- /dev/null +++ b/include/stdatomic.h @@ -0,0 +1,36 @@ +#pragma once + +typedef enum memory_order { + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_consume = __ATOMIC_CONSUME, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, + memory_order_seq_cst = __ATOMIC_SEQ_CST +} memory_order; + +#define atomic_store_explicit(ptr, des, ord) __atomic_store_n(ptr, des, ord) +#define atomic_store(ptr, des) atomic_store_explicit(ptr, des, __ATOMIC_SEQ_CST) + +#define atomic_load_explicit(ptr, ord) __atomic_load_n(ptr, ord) +#define atomic_load(ptr) atomic_load_explicit(ptr, __ATOMIC_SEQ_CST) + +#define atomic_exchange_explicit(ptr, des, ord) __atomic_exchange_n(ptr, des, ord) +#define atomic_exchange(ptr, des) atomic_exchange_explicit(ptr, des, __ATOMIC_SEQ_CST) + +#define atomic_fetch_add_explicit(ptr, op, ord) __atomic_fetch_add(ptr, op, ord) +#define atomic_fetch_add(ptr, op) atomic_fetch_add_explicit(ptr, op, __ATOMIC_SEQ_CST) + +#define atomic_fetch_sub_explicit(ptr, op, ord) __atomic_fetch_sub(ptr, op, ord) +#define atomic_fetch_sub(ptr, op) atomic_fetch_sub_explicit(ptr, op, __ATOMIC_SEQ_CST) + +#define atomic_fetch_or_explicit(ptr, op, ord) __atomic_fetch_or(ptr, op, ord) +#define atomic_fetch_or(ptr, op) atomic_fetch_or_explicit(ptr, op, __ATOMIC_SEQ_CST) + +#define atomic_fetch_xor_explicit(ptr, op, ord) __atomic_fetch_xor(ptr, op, ord) +#define atomic_fetch_xor(ptr, op) atomic_fetch_xor_explicit(ptr, op, __ATOMIC_SEQ_CST) + +#define atomic_fetch_and_explicit(ptr, op, ord) __atomic_fetch_and(ptr, op, ord) +#define atomic_fetch_and(ptr, op) atomic_fetch_and_explicit(ptr, op, __ATOMIC_SEQ_CST) + +#define atomic_thread_fence(ord) __atomic_thread_fence(ord) diff --git a/include/stddef.h b/include/stddef.h new file mode 100644 index 0000000..803d089 --- /dev/null +++ b/include/stddef.h @@ -0,0 +1,7 @@ +#pragma once + +typedef unsigned long size_t; +typedef signed long ssize_t; + +#define offsetof(type, member) __builtin_offsetof(type, member) +#define unreachable() __builtin_unreachable() diff --git a/include/stdint.h b/include/stdint.h new file mode 100644 index 0000000..ad5745c --- /dev/null +++ b/include/stdint.h @@ -0,0 +1,26 @@ +#pragma once + +typedef __INT8_TYPE__ int8_t; +typedef __UINT8_TYPE__ uint8_t; +#define INT8_MAX __INT8_MAX__ +#define UINT8_MAX __UINT8_MAX__ + +typedef __INT16_TYPE__ int16_t; +typedef __UINT16_TYPE__ uint16_t; +#define INT16_MAX __INT16_MAX__ +#define UINT16_MAX __UINT16_MAX__ + +typedef __INT32_TYPE__ int32_t; +typedef __UINT32_TYPE__ uint32_t; +#define INT32_MAX __INT32_MAX__ +#define UINT32_MAX __UINT32_MAX__ + +typedef __INT64_TYPE__ int64_t; +typedef __UINT64_TYPE__ uint64_t; +#define INT64_MAX __INT64_MAX__ +#define UINT64_MAX __UINT64_MAX__ + +typedef __INTPTR_TYPE__ intptr_t; +typedef __UINTPTR_TYPE__ uintptr_t; +#define INTPTR_MAX __INTPTR_MAX__ +#define UINTPTR_MAX __UINTPTR_MAX__ diff --git a/include/stdlib.h b/include/stdlib.h new file mode 100644 index 0000000..2b7814e --- /dev/null +++ b/include/stdlib.h @@ -0,0 +1,4 @@ +#pragma once + +long atol(const char* str, int base); +unsigned long atolu(const char* str, int base); diff --git a/include/string.h b/include/string.h new file mode 100644 index 0000000..f4792eb --- /dev/null +++ b/include/string.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +size_t strlen(const char* str); +size_t strnlen(const char* str, size_t len); +int memcmp(const void* s1, const void* s2, size_t size); +void* memcpy(void* restrict dest, const void* restrict src, size_t n); +void* memmove(void* dstptr, const void* srcptr, size_t size); +void* memset(void* dest, int value, size_t n); +char* strncpy(char* restrict dst, const char* restrict src, size_t len); +int strcmp(const char* str1, const char* str2); +int strncmp(const char* str1, const char* str2, size_t len); +char* strchr(const char* c, int s); diff --git a/include/uapi/archctl.h b/include/uapi/archctl.h new file mode 100644 index 0000000..fcfee9e --- /dev/null +++ b/include/uapi/archctl.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_ARCHCTL_H +#define __ZINNIA_UAPI_ARCHCTL_H + +#ifdef __x86_64__ +#define __ARCHCTL_SET_FSBASE 1 +#define __ARCHCTL_SET_GSBASE 2 +#endif + +#endif diff --git a/include/uapi/epoll.h b/include/uapi/epoll.h new file mode 100644 index 0000000..e4b07b0 --- /dev/null +++ b/include/uapi/epoll.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_EPOLL_H +#define __ZINNIA_UAPI_EPOLL_H + +#include + +#define __EPOLLIN (1 << 0) +#define __EPOLLOUT (1 << 1) +#define __EPOLLPRI (1 << 2) +#define __EPOLLHUP (1 << 3) +#define __EPOLLERR (1 << 4) +#define __EPOLLRDHUP (1 << 5) +#define __EPOLLMSG (1 << 6) +#define __EPOLLRDNORM (1 << 7) +#define __EPOLLRDBAND (1 << 8) +#define __EPOLLWRNORM (1 << 9) +#define __EPOLLWRBAND (1 << 10) +#define __EPOLLEXCLUSIVE (1 << 11) +#define __EPOLLWAKEUP (1 << 12) +#define __EPOLLONESHOT (1 << 13) +#define __EPOLLET (1 << 14) + +#define __EPOLL_CTL_ADD 1 +#define __EPOLL_CTL_DEL 2 +#define __EPOLL_CTL_MOD 3 + +union epoll_data { + void* ptr; + int fd; + uint32_t u32; + uint64_t u64; +}; + +struct epoll_event { + uint32_t events; + union epoll_data data; +}; + +#endif diff --git a/include/uapi/errno.h b/include/uapi/errno.h new file mode 100644 index 0000000..1246007 --- /dev/null +++ b/include/uapi/errno.h @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_ERRNO_H +#define __ZINNIA_UAPI_ERRNO_H + +#define __EPERM 1 +#define __ENOENT 2 +#define __ESRCH 3 +#define __EINTR 4 +#define __EIO 5 +#define __ENXIO 6 +#define __E2BIG 7 +#define __ENOEXEC 8 +#define __EBADF 9 +#define __ECHILD 10 +#define __EAGAIN 11 +#define __ENOMEM 12 +#define __EACCES 13 +#define __EFAULT 14 +#define __ENOTBLK 15 +#define __EBUSY 16 +#define __EEXIST 17 +#define __EXDEV 18 +#define __ENODEV 19 +#define __ENOTDIR 20 +#define __EISDIR 21 +#define __EINVAL 22 +#define __ENFILE 23 +#define __EMFILE 24 +#define __ENOTTY 25 +#define __ETXTBSY 26 +#define __EFBIG 27 +#define __ENOSPC 28 +#define __ESPIPE 29 +#define __EROFS 30 +#define __EMLINK 31 +#define __EPIPE 32 +#define __EDOM 33 +#define __ERANGE 34 +#define __EDEADLK 35 +#define __ENAMETOOLONG 36 +#define __ENOLCK 37 +#define __ENOSYS 38 +#define __ENOTEMPTY 39 +#define __ELOOP 40 +#define __EWOULDBLOCK __EAGAIN +#define __ENOMSG 42 +#define __EIDRM 43 +#define __ECHRNG 44 +#define __EL2NSYNC 45 +#define __EL3HLT 46 +#define __EL3RST 47 +#define __ELNRNG 48 +#define __EUNATCH 49 +#define __ENOCSI 50 +#define __EL2HLT 51 +#define __EBADE 52 +#define __EBADR 53 +#define __EXFULL 54 +#define __ENOANO 55 +#define __EBADRQC 56 +#define __EBADSLT 57 +#define __EDEADLOCK __EDEADLK +#define __EBFONT 59 +#define __ENOSTR 60 +#define __ENODATA 61 +#define __ETIME 62 +#define __ENOSR 63 +#define __ENONET 64 +#define __ENOPKG 65 +#define __EREMOTE 66 +#define __ENOLINK 67 +#define __EADV 68 +#define __ESRMNT 69 +#define __ECOMM 70 +#define __EPROTO 71 +#define __EMULTIHOP 72 +#define __EDOTDOT 73 +#define __EBADMSG 74 +#define __EOVERFLOW 75 +#define __ENOTUNIQ 76 +#define __EBADFD 77 +#define __EREMCHG 78 +#define __ELIBACC 79 +#define __ELIBBAD 80 +#define __ELIBSCN 81 +#define __ELIBMAX 82 +#define __ELIBEXEC 83 +#define __EILSEQ 84 +#define __ERESTART 85 +#define __ESTRPIPE 86 +#define __EUSERS 87 +#define __ENOTSOCK 88 +#define __EDESTADDRREQ 89 +#define __EMSGSIZE 90 +#define __EPROTOTYPE 91 +#define __ENOPROTOOPT 92 +#define __EPROTONOSUPPORT 93 +#define __ESOCKTNOSUPPORT 94 +#define __EOPNOTSUPP 95 +#define __ENOTSUP __EOPNOTSUPP +#define __EPFNOSUPPORT 96 +#define __EAFNOSUPPORT 97 +#define __EADDRINUSE 98 +#define __EADDRNOTAVAIL 99 +#define __ENETDOWN 100 +#define __ENETUNREACH 101 +#define __ENETRESET 102 +#define __ECONNABORTED 103 +#define __ECONNRESET 104 +#define __ENOBUFS 105 +#define __EISCONN 106 +#define __ENOTCONN 107 +#define __ESHUTDOWN 108 +#define __ETOOMANYREFS 109 +#define __ETIMEDOUT 110 +#define __ECONNREFUSED 111 +#define __EHOSTDOWN 112 +#define __EHOSTUNREACH 113 +#define __EALREADY 114 +#define __EINPROGRESS 115 +#define __ESTALE 116 +#define __EUCLEAN 117 +#define __ENOTNAM 118 +#define __ENAVAIL 119 +#define __EISNAM 120 +#define __EREMOTEIO 121 +#define __EDQUOT 122 +#define __ENOMEDIUM 123 +#define __EMEDIUMTYPE 124 +#define __ECANCELED 125 +#define __ENOKEY 126 +#define __EKEYEXPIRED 127 +#define __EKEYREVOKED 128 +#define __EKEYREJECTED 129 +#define __EOWNERDEAD 130 +#define __ENOTRECOVERABLE 131 +#define __ERFKILL 132 +#define __EHWPOISON 133 + +#endif diff --git a/include/uapi/fcntl.h b/include/uapi/fcntl.h new file mode 100644 index 0000000..293c8f9 --- /dev/null +++ b/include/uapi/fcntl.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_FCNTL_H +#define __ZINNIA_UAPI_FCNTL_H + +#define __F_OK 0 +#define __X_OK 1 +#define __W_OK 2 +#define __R_OK 4 + +#define __O_RDONLY (1 << 0) +#define __O_WRONLY (1 << 1) +#define __O_RDWR (__O_RDONLY | __O_WRONLY) +#define __O_CREAT (1 << 6) +#define __O_EXCL (1 << 7) +#define __O_NOCTTY (1 << 8) +#define __O_TRUNC (1 << 9) +#define __O_APPEND (1 << 10) +#define __O_NONBLOCK (1 << 11) +#define __O_DSYNC (1 << 12) +#define __O_ASYNC (1 << 13) +#define __O_DIRECT (1 << 14) +#define __O_LARGEFILE (1 << 15) +#define __O_DIRECTORY (1 << 16) +#define __O_NOFOLLOW (1 << 17) +#define __O_NOATIME (1 << 18) +#define __O_CLOEXEC (1 << 19) +#define __O_PATH (1 << 21) +#define __O_TMPFILE (1 << 22) +#define __O_SYNC (__O_DIRECTORY | __O_TMPFILE) +#define __O_RSYNC (__O_SYNC) +#define __O_EXEC (__O_PATH) +#define __O_SEARCH (__O_PATH) +#define __O_ACCMODE (__O_RDWR | __O_PATH) + +#define __AT_FDCWD (-100) + +#define __AT_SYMLINK_NOFOLLOW (1 << 8) +#define __AT_SYMLINK_FOLLOW (1 << 9) +#define __AT_REMOVEDIR (1 << 10) +#define __AT_EACCESS (1 << 11) +#define __AT_EMPTY_PATH (1 << 12) + +#endif diff --git a/include/uapi/mman.h b/include/uapi/mman.h new file mode 100644 index 0000000..5474159 --- /dev/null +++ b/include/uapi/mman.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_MMAN_H +#define __ZINNIA_UAPI_MMAN_H + +#define __PROT_NONE 0x00 +#define __PROT_READ 0x01 +#define __PROT_WRITE 0x02 +#define __PROT_EXEC 0x04 + +#define __MAP_FAILED ((void*)-1) +#define __MAP_FILE 0x00 +#define __MAP_SHARED 0x01 +#define __MAP_PRIVATE 0x02 +#define __MAP_FIXED 0x10 +#define __MAP_ANON 0x20 +#define __MAP_ANONYMOUS 0x20 + +#define __MS_ASYNC 0x1 +#define __MS_INVALIDATE 0x2 +#define __MS_SYNC 0x4 + +#endif diff --git a/include/uapi/mount.h b/include/uapi/mount.h new file mode 100644 index 0000000..79c4ca7 --- /dev/null +++ b/include/uapi/mount.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_MOUNT_H +#define __ZINNIA_UAPI_MOUNT_H + +#define __MNT_RDONLY (1 << 0) +#define __MNT_NOSUID (1 << 1) +#define __MNT_NOEXEC (1 << 2) +#define __MNT_RELATIME (1 << 3) +#define __MNT_NOATIME (1 << 4) +#define __MNT_REMOUNT (1 << 5) +#define __MNT_FORCE (1 << 6) + +#endif diff --git a/include/uapi/poll.h b/include/uapi/poll.h new file mode 100644 index 0000000..b2d5fce --- /dev/null +++ b/include/uapi/poll.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_POLL_H +#define __ZINNIA_UAPI_POLL_H + +#include + +#define __POLLIN 0x01 +#define __POLLOUT 0x02 +#define __POLLPRI 0x04 +#define __POLLHUP 0x08 +#define __POLLERR 0x10 +#define __POLLRDHUP 0x20 +#define __POLLNVAL 0x40 +#define __POLLRDNORM 0x80 +#define __POLLRDBAND 0x100 +#define __POLLWRNORM 0x200 +#define __POLLWRBAND 0x400 + +typedef intptr_t nfds_t; + +struct pollfd { + int fd; + int16_t events; + int16_t revents; +}; + +#endif diff --git a/include/uapi/stat.h b/include/uapi/stat.h new file mode 100644 index 0000000..4a89ccc --- /dev/null +++ b/include/uapi/stat.h @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_STAT_H +#define __ZINNIA_UAPI_STAT_H + +#include +#include + +#define __S_IFMT 0x0F000 +#define __S_IFBLK 0x06000 +#define __S_IFCHR 0x02000 +#define __S_IFIFO 0x01000 +#define __S_IFREG 0x08000 +#define __S_IFDIR 0x04000 +#define __S_IFLNK 0x0A000 +#define __S_IFSOCK 0x0C000 +#define __S_IRWXU 0700 +#define __S_IRUSR 0400 +#define __S_IWUSR 0200 +#define __S_IXUSR 0100 +#define __S_IRWXG 070 +#define __S_IRGRP 040 +#define __S_IWGRP 020 +#define __S_IXGRP 010 +#define __S_IRWXO 07 +#define __S_IROTH 04 +#define __S_IWOTH 02 +#define __S_IXOTH 01 +#define __S_ISUID 04000 +#define __S_ISGID 02000 +#define __S_ISVTX 01000 +#define __S_IREAD __S_IRUSR +#define __S_IWRITE __S_IWUSR +#define __S_IEXEC __S_IXUSR + +struct stat { + dev_t st_dev; + ino_t st_ino; + mode_t st_mode; + nlink_t st_nlink; + uid_t st_uid; + gid_t st_gid; + dev_t st_rdev; + off_t st_size; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; + blksize_t st_blksize; + blkcnt_t st_blocks; +}; + +#endif diff --git a/include/uapi/syscall_numbers.h b/include/uapi/syscall_numbers.h new file mode 100644 index 0000000..b7a7d7f --- /dev/null +++ b/include/uapi/syscall_numbers.h @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_SYSCALL_NUMBERS_H +#define __ZINNIA_UAPI_SYSCALL_NUMBERS_H + +#define __SYS_exit 0 +#define __SYS_syslog 1 +#define __SYS_getuname 2 +#define __SYS_setuname 3 +#define __SYS_archctl 4 +#define __SYS_reboot 5 +#define __SYS_mmap 6 +#define __SYS_munmap 7 +#define __SYS_mprotect 8 +#define __SYS_madvise 9 +#define __SYS_sigprocmask 10 +#define __SYS_sigsuspend 11 +#define __SYS_sigpending 12 +#define __SYS_sigaction 13 +#define __SYS_sigtimedwait 14 +#define __SYS_sigaltstack 15 +#define __SYS_sigreturn 16 +#define __SYS_sigqueue 17 +#define __SYS_execve 18 +#define __SYS_fexecve 19 +#define __SYS_fork 20 +#define __SYS_kill 21 +#define __SYS_gettid 22 +#define __SYS_getpid 23 +#define __SYS_getppid 24 +#define __SYS_waitid 25 +#define __SYS_waitpid 26 +#define __SYS_pread 27 +#define __SYS_readv 28 +#define __SYS_pwrite 29 +#define __SYS_writev 30 +#define __SYS_seek 31 +#define __SYS_ioctl 32 +#define __SYS_openat 33 +#define __SYS_close 34 +#define __SYS_fstat 35 +#define __SYS_fstatat 36 +#define __SYS_statvfs 37 +#define __SYS_fstatvfs 38 +#define __SYS_faccessat 39 +#define __SYS_fcntl 40 +#define __SYS_ftruncate 41 +#define __SYS_truncate 42 +#define __SYS_fallocate 43 +#define __SYS_fadvise 44 +#define __SYS_utimensat 45 +#define __SYS_mknodat 46 +#define __SYS_getcwd 47 +#define __SYS_chdir 48 +#define __SYS_fchdir 49 +#define __SYS_mkdirat 50 +#define __SYS_rmdirat 51 +#define __SYS_getdents 52 +#define __SYS_renameat 53 +#define __SYS_fchmod 54 +#define __SYS_fchmodat 55 +#define __SYS_fchownat 56 +#define __SYS_linkat 57 +#define __SYS_symlinkat 58 +#define __SYS_unlinkat 59 +#define __SYS_readlinkat 60 +#define __SYS_flock 61 +#define __SYS_ppoll 62 +#define __SYS_dup 63 +#define __SYS_dup3 64 +#define __SYS_sync 65 +#define __SYS_fsync 66 +#define __SYS_fdatasync 67 +#define __SYS_getgroups 68 +#define __SYS_setgroups 69 +#define __SYS_getsid 70 +#define __SYS_setsid 71 +#define __SYS_setuid 72 +#define __SYS_getuid 73 +#define __SYS_setgid 74 +#define __SYS_getgid 75 +#define __SYS_geteuid 76 +#define __SYS_seteuid 77 +#define __SYS_getegid 78 +#define __SYS_setegid 79 +#define __SYS_getpgid 80 +#define __SYS_setpgid 81 +#define __SYS_getresuid 82 +#define __SYS_setresuid 83 +#define __SYS_getresgid 84 +#define __SYS_setresgid 85 +#define __SYS_setreuid 86 +#define __SYS_setregid 87 +#define __SYS_umask 88 +#define __SYS_pipe 89 +#define __SYS_futex_wait 90 +#define __SYS_futex_wake 91 +#define __SYS_thread_create 92 +#define __SYS_thread_kill 93 +#define __SYS_thread_exit 94 +#define __SYS_thread_setname 95 +#define __SYS_thread_getname 96 +#define __SYS_timer_create 97 +#define __SYS_timer_set 98 +#define __SYS_timer_get 99 +#define __SYS_timer_getoverrun 100 +#define __SYS_timer_delete 101 +#define __SYS_itimer_get 102 +#define __SYS_itimer_set 103 +#define __SYS_clock_get 104 +#define __SYS_clock_getres 105 +#define __SYS_clock_set 106 +#define __SYS_sleep 107 +#define __SYS_yield 108 +#define __SYS_chroot 109 +#define __SYS_mount 110 +#define __SYS_umount 111 +#define __SYS_socket 112 +#define __SYS_socketpair 113 +#define __SYS_shutdown 114 +#define __SYS_bind 115 +#define __SYS_connect 116 +#define __SYS_accept 117 +#define __SYS_listen 118 +#define __SYS_getpeername 119 +#define __SYS_getsockname 120 +#define __SYS_getsockopt 121 +#define __SYS_setsockopt 122 +#define __SYS_sendmsg 123 +#define __SYS_recvmsg 124 +#define __SYS_getentropy 125 +#define __SYS_getrusage 126 +#define __SYS_getrlimit 127 +#define __SYS_setrlimit 128 +#define __SYS_getpriority 129 +#define __SYS_setpriority 130 +#define __SYS_sched_getparam 131 +#define __SYS_sched_setparam 132 +#define __SYS_module_insert 133 +#define __SYS_module_remove 134 +#define __SYS_epoll_create 135 +#define __SYS_epoll_ctl 136 +#define __SYS_epoll_pwait 137 +#define __SYS_timerfd_create 138 +#define __SYS_timerfd_gettime 139 +#define __SYS_timerfd_settime 140 +#define __SYS_signalfd_create 141 +#define __SYS_eventfd_create 142 +#define __SYS_msync 143 +#define __SYS_sysconf 144 + +#endif diff --git a/include/uapi/time.h b/include/uapi/time.h new file mode 100644 index 0000000..ffc057c --- /dev/null +++ b/include/uapi/time.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_TIME_H +#define __ZINNIA_UAPI_TIME_H + +#include + +typedef int64_t time_t; +typedef int64_t suseconds_t; + +struct timespec { + time_t tv_sec; + int64_t tv_nsec; +}; + +struct timeval { + time_t tv_sec; + suseconds_t tv_usec; +}; + +struct itimerval { + struct timeval it_interval; + struct timeval it_value; +}; + +struct itimerspec { + struct timespec it_interval; + struct timespec it_value; +}; + +#define __ITIMER_REAL 0 +#define __ITIMER_VIRTUAL 1 +#define __ITIMER_PROF 2 + +#define __CLOCK_REALTIME 0 +#define __CLOCK_MONOTONIC 1 +#define __CLOCK_PROCESS_CPUTIME_ID 2 +#define __CLOCK_THREAD_CPUTIME_ID 3 +#define __CLOCK_MONOTONIC_RAW 4 +#define __CLOCK_REALTIME_COARSE 5 +#define __CLOCK_MONOTONIC_COARSE 6 +#define __CLOCK_BOOTTIME 7 + +#define __TFD_CLOEXEC __O_CLOEXEC +#define __TFD_NONBLOCK __O_NONBLOCK +#define __TFD_TIMER_ABSTIME 1 + +#endif diff --git a/include/uapi/types.h b/include/uapi/types.h new file mode 100644 index 0000000..494e966 --- /dev/null +++ b/include/uapi/types.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_TYPES_H +#define __ZINNIA_UAPI_TYPES_H + +#include + +typedef int64_t off_t; +typedef off_t off64_t; +typedef uintptr_t blksize_t; +typedef uintptr_t blkcnt_t; +typedef uintptr_t clockid_t; +typedef uintptr_t dev_t; +typedef uint32_t gid_t; +typedef uintptr_t ino_t; +typedef uint32_t mode_t; +typedef uintptr_t nlink_t; +typedef int32_t pid_t; +typedef uintptr_t rlim_t; +typedef uint32_t uid_t; +typedef uintptr_t fsblkcnt_t; +typedef uintptr_t fsfilcnt_t; +typedef uintptr_t pthread_attr_t; + +#endif diff --git a/include/uapi/uio.h b/include/uapi/uio.h new file mode 100644 index 0000000..8428862 --- /dev/null +++ b/include/uapi/uio.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_UIO_H +#define __ZINNIA_UAPI_UIO_H + +#include + +struct iovec { + void* base; + size_t len; +}; + +#endif diff --git a/include/uapi/utsname.h b/include/uapi/utsname.h new file mode 100644 index 0000000..20fdf3f --- /dev/null +++ b/include/uapi/utsname.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ZINNIA_UAPI_UTSNAME_H +#define __ZINNIA_UAPI_UTSNAME_H + +struct utsname { + char sysname[65]; + char nodename[65]; + char release[65]; + char version[65]; + char machine[65]; + char domainname[65]; +}; + +#endif diff --git a/include/zinnia/archctl.h b/include/zinnia/archctl.h new file mode 100644 index 0000000..07d15ba --- /dev/null +++ b/include/zinnia/archctl.h @@ -0,0 +1,3 @@ +#pragma once + +#include diff --git a/include/zinnia/bits/elf.h b/include/zinnia/bits/elf.h new file mode 100644 index 0000000..371326f --- /dev/null +++ b/include/zinnia/bits/elf.h @@ -0,0 +1,3 @@ +#pragma once + +#include diff --git a/include/zinnia/bits/irq.h b/include/zinnia/bits/irq.h new file mode 100644 index 0000000..3935e69 --- /dev/null +++ b/include/zinnia/bits/irq.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +void arch_irq_set_state(bool state); +bool arch_irq_get_state(); +void arch_irq_wait(); diff --git a/include/zinnia/bits/mem.h b/include/zinnia/bits/mem.h new file mode 100644 index 0000000..d0d60f9 --- /dev/null +++ b/include/zinnia/bits/mem.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +size_t arch_mem_page_size(); +size_t arch_mem_page_bits(); +size_t arch_mem_level_bits(); +size_t arch_mem_num_levels(); +uintptr_t arch_mem_hhdm_addr(); +uintptr_t arch_vm_pfndb_addr(); +uintptr_t arch_mem_mapping_addr(); diff --git a/include/zinnia/bits/usercopy.h b/include/zinnia/bits/usercopy.h new file mode 100644 index 0000000..417e58d --- /dev/null +++ b/include/zinnia/bits/usercopy.h @@ -0,0 +1,9 @@ +#pragma once + +#include +#include +#include + +bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region); +bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region); +bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region); diff --git a/include/zinnia/clock.h b/include/zinnia/clock.h new file mode 100644 index 0000000..33de343 --- /dev/null +++ b/include/zinnia/clock.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +struct clock { + const char* name; + uint8_t priority; + + void (*reset)(struct clock* c); + uint64_t (*get_elapsed_ns)(struct clock* c); +}; + +// Attempts to set a new clock. Returns true if the clock was changed. +bool clock_switch(struct clock* clock); + +// Returns true, if a clock is available. +bool clock_available(); + +// Gets the current timestamp in nanoseconds since the last reset. +uint64_t clock_get_elapsed_ns(); + +// Spin for `ns` nanoseconds. +void clock_spin_ns(uint64_t ns); diff --git a/include/zinnia/cmdline.h b/include/zinnia/cmdline.h new file mode 100644 index 0000000..307c732 --- /dev/null +++ b/include/zinnia/cmdline.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +#define _CMDLINE_OPTION(x, opt_name, opt_func) \ + static const struct cmdline_option x = { \ + .name = opt_name, \ + .option_type = _Generic( \ + (opt_func), \ + void (*)(const char*): CMDLINE_STRING, \ + void (*)(bool): CMDLINE_BOOL, \ + void (*)(intptr_t): CMDLINE_INTPTR, \ + void (*)(uintptr_t): CMDLINE_UINTPTR \ + ), \ + .func = (void*)(opt_func), \ + }; \ + [[__used, __section(".cmdline")]] \ + static const struct cmdline_option* CONCAT(x, _ptr) = &x; + +#define CMDLINE_OPTION(opt_name, opt_func) _CMDLINE_OPTION(UNIQUE_IDENT(cmdline_option), opt_name, opt_func) + +enum cmdline_type { + CMDLINE_STRING, + CMDLINE_BOOL, + CMDLINE_INTPTR, + CMDLINE_UINTPTR, +}; + +struct cmdline_option { + // The name of this option. + const char* name; + // Gets called if this option is present on the command line. + union { + void (*func_str)(const char* value); + void (*func_bool)(bool value); + void (*func_intptr)(intptr_t value); + void (*func_uintptr)(uintptr_t value); + void* func; + }; + enum cmdline_type option_type; +}; + +// Parses the command line and invokes all options. +void cmdline_parse(const char* cmdline); diff --git a/include/zinnia/compiler.h b/include/zinnia/compiler.h new file mode 100644 index 0000000..c467a05 --- /dev/null +++ b/include/zinnia/compiler.h @@ -0,0 +1,27 @@ +#pragma once + +// Attributes that affect data layout. +#define __section(x) gnu::section(x) +#define __aligned(x) gnu::aligned(x) +#define __packed gnu::packed +#define __naked gnu::naked + +// These hints may improve code generation. +#define __cold gnu::cold +#define __hot gnu::hot +#define __used gnu::used +#define __unused gnu::unused +#define __format(like, ...) gnu::format(like, __VA_ARGS__) +#define __likely(x) __builtin_expect(!!(x), 1) +#define __unlikely(x) __builtin_expect(!!(x), 0) +#define __always_inline gnu::always_inline + +// Only clang has these hints, GCC will throw a warning here. +// They also don't support the C23 attribute format. +#ifdef __clang__ +#define __user __attribute__((noderef, address_space(1))) +#define __phys __attribute__((noderef, address_space(2))) +#else +#define __user +#define __phys +#endif diff --git a/include/zinnia/console.h b/include/zinnia/console.h new file mode 100644 index 0000000..c3ff63f --- /dev/null +++ b/include/zinnia/console.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + +struct console { + const char* name; + void* private; + + // Initializes the console. + bool (*init)(struct console* con); + // Writes to the console. + void (*write)(struct console* con, const char* buf, size_t count); +}; + +// Initializes a console for logging. +void console_init(); + +// Writes a message to a console. +void console_write(const char* buf, size_t len); + +// Defines a new early console which can be used to log early boot messages. +// An early console must not make any heap allocations. +#define DEFINE_CONSOLE(con) \ + [[__used, __section(".console")]] \ + static struct console* UNIQUE_IDENT(console) = (con) diff --git a/include/zinnia/elf.h b/include/zinnia/elf.h new file mode 100644 index 0000000..6c796cb --- /dev/null +++ b/include/zinnia/elf.h @@ -0,0 +1,340 @@ +#pragma once + +#include + +constexpr char ELF_MAG[4] = {0x7F, 'E', 'L', 'F'}; + +typedef uint64_t elf64_addr_t; +typedef uint64_t elf64_off_t; +typedef uint32_t elf32_addr_t; +typedef uint32_t elf32_off_t; + +enum : uint8_t { + EI_MAG0 = 0, + EI_MAG1 = 1, + EI_MAG2 = 2, + EI_MAG3 = 3, + EI_CLASS = 4, + EI_DATA = 5, + EI_VERSION = 6, + EI_OSABI = 7, + EI_ABIVERSION = 8, + EI_PAD = 9, + EI_NIDENT = 16, +}; + +enum : uint16_t { + EM_X86_64 = 62, + EM_AARCH64 = 183, + EM_RISCV = 243, + EM_LOONGARCH = 258, +}; + +enum : uint32_t { + ELFCLASS32 = 1, + ELFCLASS64 = 2, +}; + +enum : uint32_t { + ELFDATA2LSB = 1, + ELFDATA2MSB = 2, +}; + +enum : uint32_t { + EV_NONE = 0, + EV_CURRENT = 1, + EV_NUM = 2, +}; + +enum : uint32_t { + ELFOSABI_SYSV = 0, + ELFOSABI_HPUX = 1, + ELFOSABI_STANDALONE = 255, +}; + +enum : uint32_t { + ET_NONE = 0, + ET_REL = 1, + ET_EXEC = 2, + ET_DYN = 3, + ET_CORE = 4, +}; + +enum : uint32_t { + PT_NULL = 0x00000000, + PT_LOAD = 0x00000001, + PT_DYNAMIC = 0x00000002, + PT_INTERP = 0x00000003, + PT_NOTE = 0x00000004, + PT_SHLIB = 0x00000005, + PT_PHDR = 0x00000006, + PT_TLS = 0x00000007, +}; + +enum : uint32_t { + PF_X = 0x00000001, + PF_W = 0x00000002, + PF_R = 0x00000004, +}; + +enum : uint32_t { + DT_NULL = 0, + DT_NEEDED = 1, + DT_PLTRELSZ = 2, + DT_PLTGOT = 3, + DT_HASH = 4, + DT_STRTAB = 5, + DT_SYMTAB = 6, + DT_RELA = 7, + DT_RELASZ = 8, + DT_RELAENT = 9, + DT_STRSZ = 10, + DT_SYMENT = 11, + DT_INIT = 12, + DT_FINI = 13, + DT_SONAME = 14, + DT_RPATH = 15, + DT_SYMBOLIC = 16, + DT_REL = 17, + DT_RELSZ = 18, + DT_RELENT = 19, + DT_PLTREL = 20, + DT_DEBUG = 21, + DT_TEXTREL = 22, + DT_JMPREL = 23, + DT_BIND_NOW = 24, + DT_INIT_ARRAY = 25, + DT_FINI_ARRAY = 26, + DT_INIT_ARRAYSZ = 27, + DT_FINI_ARRAYSZ = 28, + DT_LOOS = 0x60000000, + DT_HIOS = 0x6FFFFFFF, + DT_LOPROC = 0x70000000, + DT_HIPROC = 0x7FFFFFFF, +}; + +enum : uint32_t { + AT_NULL = 0, + AT_IGNORE = 1, + AT_EXECFD = 2, + AT_PHDR = 3, + AT_PHENT = 4, + AT_PHNUM = 5, + AT_PAGESZ = 6, + AT_BASE = 7, + AT_FLAGS = 8, + AT_ENTRY = 9, + AT_NOTELF = 10, + AT_UID = 11, + AT_EUID = 12, + AT_GID = 13, + AT_EGID = 14, + AT_HWCAP = 16, + AT_SECURE = 23, + AT_RANDOM = 25, + AT_EXECFN = 31, + AT_SYSINFO_EHDR = 33, + AT_INIT_HANDLE = 0x1000, +}; + +struct elf64_ehdr { + uint8_t e_ident[EI_NIDENT]; + uint16_t e_type; + uint16_t e_machine; + uint32_t e_version; + elf64_addr_t e_entry; + elf64_off_t e_phoff; + elf64_off_t e_shoff; + uint32_t e_flags; + uint16_t e_ehsize; + uint16_t e_phentsize; + uint16_t e_phnum; + uint16_t e_shentsize; + uint16_t e_shnum; + uint16_t e_shstrndx; +}; + +struct elf32_ehdr { + uint8_t e_ident[EI_NIDENT]; + uint16_t e_type; + uint16_t e_machine; + uint32_t e_version; + elf32_addr_t e_entry; + elf32_off_t e_phoff; + elf32_off_t e_shoff; + uint32_t e_flags; + uint16_t e_ehsize; + uint16_t e_phentsize; + uint16_t e_phnum; + uint16_t e_shentsize; + uint16_t e_shnum; + uint16_t e_shstrndx; +}; + +struct elf64_phdr { + uint32_t p_type; + uint32_t p_flags; + elf64_off_t p_offset; + elf64_addr_t p_vaddr; + elf64_addr_t p_paddr; + uint64_t p_filesz; + uint64_t p_memsz; + uint64_t p_align; +}; + +struct elf32_phdr { + uint32_t p_type; + elf32_off_t p_offset; + elf32_addr_t p_vaddr; + elf32_addr_t p_paddr; + uint32_t p_filesz; + uint32_t p_memsz; + uint32_t p_flags; + uint32_t p_align; +}; + +struct elf64_dyn { + int64_t d_tag; + union { + uint64_t d_val; + elf64_addr_t d_ptr; + } d_un; +}; + +struct elf32_dyn { + int32_t r_offset; + union { + uint32_t d_val; + elf32_addr_t d_ptr; + } d_un; +}; + +struct elf64_nhdr { + uint64_t n_namesz; + uint64_t n_descsz; + uint64_t n_type; +}; + +struct elf32_nhdr { + uint32_t n_namesz; + uint32_t n_descsz; + uint32_t n_type; +}; + +struct elf64_auxv { + uint64_t atype; + uint64_t avalue; +}; + +struct elf32_auxv { + uint32_t atype; + uint32_t avalue; +}; + +struct elf_hash { + uint32_t n_buckets; + uint32_t n_chain; +}; + +struct elf32_sym { + elf32_off_t st_name; + elf32_addr_t st_value; + elf32_off_t st_size; + uint8_t st_info; + uint8_t st_other; + uint16_t st_shndx; +}; + +struct elf64_sym { + uint32_t st_name; + uint8_t st_info; + uint8_t st_other; + uint16_t st_shndx; + uint64_t st_value; + uint64_t st_size; +}; + +static uint8_t ELF64_ST_BIND(uint8_t info) { + return info >> 4; +} + +static inline uint8_t ELF64_ST_TYPE(uint8_t info) { + return info & 0x0F; +} + +static inline uint8_t ELF64_ST_INFO(uint8_t bind, uint8_t type) { + return (bind << 4) | type; +} + +typedef struct { + uint16_t si_boundto; + uint16_t si_flags; +} elf64_syminfo; + +static inline uint8_t ELF32_ST_BIND(uint8_t info) { + return info >> 4; +} + +static inline uint8_t ELF32_ST_TYPE(uint8_t info) { + return info & 0xF; +} + +static inline uint8_t ELF32_ST_INFO(uint8_t bind, uint8_t type) { + return (bind << 4) | (type & 0xF); +} + +enum { + STB_GLOBAL = 1, + STB_WEAK = 2, + STB_GNU_UNIQUE = 10, + STB_LOPROC = 13, + STB_HIPROC = 15 +}; + +enum { + STT_NOTYPE = 0, + STT_OBJECT = 1, + STT_FUNC = 2, + STT_TLS = 6, + STT_GNU_IFUNC = 10, + STT_LOPROC = 13, + STT_HIPROC = 15 +}; + +struct elf32_rela { + elf32_addr_t r_offset; + uint32_t r_info; + int32_t r_addend; +}; + +struct elf64_rela { + elf64_addr_t r_offset; + uint64_t r_info; + int64_t r_addend; +}; + +struct elf32_rel { + elf32_addr_t r_offset; + uint32_t r_info; +}; + +struct elf64_rel { + elf64_addr_t r_offset; + uint64_t r_info; +}; + +static inline uint32_t ELF32_R_SYM(uint32_t info) { + return info >> 8; +} + +static inline uint32_t ELF32_R_TYPE(uint32_t info) { + return info & 0xF; +} + +static inline uint32_t ELF64_R_SYM(uint64_t info) { + return info >> 32; +} + +static inline uint32_t ELF64_R_TYPE(uint64_t info) { + return info & 0xFFFFFFFF; +} diff --git a/include/zinnia/errno.h b/include/zinnia/errno.h new file mode 100644 index 0000000..aebb84f --- /dev/null +++ b/include/zinnia/errno.h @@ -0,0 +1,142 @@ +#pragma once + +#include +#include + +typedef enum : int32_t { + EOK = 0, // Not really an error. + EPERM = __EPERM, + ENOENT = __ENOENT, + ESRCH = __ESRCH, + EINTR = __EINTR, + EIO = __EIO, + ENXIO = __ENXIO, + E2BIG = __E2BIG, + ENOEXEC = __ENOEXEC, + EBADF = __EBADF, + ECHILD = __ECHILD, + EAGAIN = __EAGAIN, + ENOMEM = __ENOMEM, + EACCES = __EACCES, + EFAULT = __EFAULT, + ENOTBLK = __ENOTBLK, + EBUSY = __EBUSY, + EEXIST = __EEXIST, + EXDEV = __EXDEV, + ENODEV = __ENODEV, + ENOTDIR = __ENOTDIR, + EISDIR = __EISDIR, + EINVAL = __EINVAL, + ENFILE = __ENFILE, + EMFILE = __EMFILE, + ENOTTY = __ENOTTY, + ETXTBSY = __ETXTBSY, + EFBIG = __EFBIG, + ENOSPC = __ENOSPC, + ESPIPE = __ESPIPE, + EROFS = __EROFS, + EMLINK = __EMLINK, + EPIPE = __EPIPE, + EDOM = __EDOM, + ERANGE = __ERANGE, + EDEADLK = __EDEADLK, + ENAMETOOLONG = __ENAMETOOLONG, + ENOLCK = __ENOLCK, + ENOSYS = __ENOSYS, + ENOTEMPTY = __ENOTEMPTY, + ELOOP = __ELOOP, + EWOULDBLOCK = __EWOULDBLOCK, + ENOMSG = __ENOMSG, + EIDRM = __EIDRM, + ECHRNG = __ECHRNG, + EL2NSYNC = __EL2NSYNC, + EL3HLT = __EL3HLT, + EL3RST = __EL3RST, + ELNRNG = __ELNRNG, + EUNATCH = __EUNATCH, + ENOCSI = __ENOCSI, + EL2HLT = __EL2HLT, + EBADE = __EBADE, + EBADR = __EBADR, + EXFULL = __EXFULL, + ENOANO = __ENOANO, + EBADRQC = __EBADRQC, + EBADSLT = __EBADSLT, + EDEADLOCK = __EDEADLOCK, + EBFONT = __EBFONT, + ENOSTR = __ENOSTR, + ENODATA = __ENODATA, + ETIME = __ETIME, + ENOSR = __ENOSR, + ENONET = __ENONET, + ENOPKG = __ENOPKG, + EREMOTE = __EREMOTE, + ENOLINK = __ENOLINK, + EADV = __EADV, + ESRMNT = __ESRMNT, + ECOMM = __ECOMM, + EPROTO = __EPROTO, + EMULTIHOP = __EMULTIHOP, + EDOTDOT = __EDOTDOT, + EBADMSG = __EBADMSG, + EOVERFLOW = __EOVERFLOW, + ENOTUNIQ = __ENOTUNIQ, + EBADFD = __EBADFD, + EREMCHG = __EREMCHG, + ELIBACC = __ELIBACC, + ELIBBAD = __ELIBBAD, + ELIBSCN = __ELIBSCN, + ELIBMAX = __ELIBMAX, + ELIBEXEC = __ELIBEXEC, + EILSEQ = __EILSEQ, + ERESTART = __ERESTART, + ESTRPIPE = __ESTRPIPE, + EUSERS = __EUSERS, + ENOTSOCK = __ENOTSOCK, + EDESTADDRREQ = __EDESTADDRREQ, + EMSGSIZE = __EMSGSIZE, + EPROTOTYPE = __EPROTOTYPE, + ENOPROTOOPT = __ENOPROTOOPT, + EPROTONOSUPPORT = __EPROTONOSUPPORT, + ESOCKTNOSUPPORT = __ESOCKTNOSUPPORT, + EOPNOTSUPP = __EOPNOTSUPP, + ENOTSUP = __ENOTSUP, + EPFNOSUPPORT = __EPFNOSUPPORT, + EAFNOSUPPORT = __EAFNOSUPPORT, + EADDRINUSE = __EADDRINUSE, + EADDRNOTAVAIL = __EADDRNOTAVAIL, + ENETDOWN = __ENETDOWN, + ENETUNREACH = __ENETUNREACH, + ENETRESET = __ENETRESET, + ECONNABORTED = __ECONNABORTED, + ECONNRESET = __ECONNRESET, + ENOBUFS = __ENOBUFS, + EISCONN = __EISCONN, + ENOTCONN = __ENOTCONN, + ESHUTDOWN = __ESHUTDOWN, + ETOOMANYREFS = __ETOOMANYREFS, + ETIMEDOUT = __ETIMEDOUT, + ECONNREFUSED = __ECONNREFUSED, + EHOSTDOWN = __EHOSTDOWN, + EHOSTUNREACH = __EHOSTUNREACH, + EALREADY = __EALREADY, + EINPROGRESS = __EINPROGRESS, + ESTALE = __ESTALE, + EUCLEAN = __EUCLEAN, + ENOTNAM = __ENOTNAM, + ENAVAIL = __ENAVAIL, + EISNAM = __EISNAM, + EREMOTEIO = __EREMOTEIO, + EDQUOT = __EDQUOT, + ENOMEDIUM = __ENOMEDIUM, + EMEDIUMTYPE = __EMEDIUMTYPE, + ECANCELED = __ECANCELED, + ENOKEY = __ENOKEY, + EKEYEXPIRED = __EKEYEXPIRED, + EKEYREVOKED = __EKEYREVOKED, + EKEYREJECTED = __EKEYREJECTED, + EOWNERDEAD = __EOWNERDEAD, + ENOTRECOVERABLE = __ENOTRECOVERABLE, + ERFKILL = __ERFKILL, + EHWPOISON = __EHWPOISON, +} errno_t; diff --git a/include/zinnia/futex.h b/include/zinnia/futex.h new file mode 100644 index 0000000..04b9d3e --- /dev/null +++ b/include/zinnia/futex.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +struct futex {}; + +// Initializes the global futex table. +void futex_init(void); + +// Blocks the calling task if *addr == expected. +errno_t futex_wait(__user int* addr, int expected); + +// Wakes up to `count` tasks waiting on the given address. +// Returns the number of tasks woken. +errno_t futex_wake(__user int* addr, int count); diff --git a/include/zinnia/hashmap.h b/include/zinnia/hashmap.h new file mode 100644 index 0000000..687088b --- /dev/null +++ b/include/zinnia/hashmap.h @@ -0,0 +1,223 @@ +#pragma once + +#include +#include +#include +#include +#include + +// FNV-1a hash over raw bytes. +static inline uint64_t hashmap_hash_default(const void* key, size_t len) { + const uint8_t* data = (const uint8_t*)key; + uint64_t hash = 0xcbf29ce484222325; + for (size_t i = 0; i < len; i++) { + hash ^= data[i]; + hash *= 0x100000001b3; + } + return hash; +} + +// Byte-wise equality. +static inline bool hashmap_eq_default(const void* a, const void* b, size_t len) { + return memcmp(a, b, len) == 0; +} + +// Ensure a hash is never zero (zero is the empty sentinel). +#define _HASHMAP_FIXHASH(h) ((h) == 0 ? (uint64_t)1 : (h)) + +#define HASHMAP(K, V) \ + struct { \ + K* keys; \ + V* values; \ + uint64_t* hashes; \ + size_t capacity; \ + size_t count; \ + } + +#define HASHMAP_INIT(map) \ + do { \ + (map)->keys = nullptr; \ + (map)->values = nullptr; \ + (map)->hashes = nullptr; \ + (map)->capacity = 0; \ + (map)->count = 0; \ + } while (0) + +#define HASHMAP_DESTROY(map) \ + do { \ + mem_free((map)->keys); \ + mem_free((map)->values); \ + mem_free((map)->hashes); \ + (map)->keys = nullptr; \ + (map)->values = nullptr; \ + (map)->hashes = nullptr; \ + (map)->capacity = 0; \ + (map)->count = 0; \ + } while (0) + +#define HASHMAP_COUNT(map) ((map)->count) + +// Internal: probe distance from ideal position. +#define _HASHMAP_PROBE_DIST(hash, pos, cap) (((pos) - ((hash) & ((cap) - 1))) & ((cap) - 1)) + +// Internal: grow and rehash. +#define _HASHMAP_GROW(map, hash_fn, eq_fn) \ + do { \ + size_t _old_cap = (map)->capacity; \ + size_t _new_cap = _old_cap == 0 ? 16 : _old_cap * 2; \ + typeof((map)->keys) _old_keys = (map)->keys; \ + typeof((map)->values) _old_values = (map)->values; \ + uint64_t* _old_hashes = (map)->hashes; \ + (map)->keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \ + (map)->values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \ + (map)->hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \ + (map)->capacity = _new_cap; \ + (map)->count = 0; \ + for (size_t _i = 0; _i < _old_cap; _i++) { \ + if (_old_hashes[_i] != 0) { \ + _HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \ + } \ + } \ + mem_free(_old_keys); \ + mem_free(_old_values); \ + mem_free(_old_hashes); \ + } while (0) + +// Internal: insert with a precomputed hash. Used by grow and the public INSERT macro. +#define _HASHMAP_INSERT_INNER(map, k, v, h) \ + do { \ + size_t _cap = (map)->capacity; \ + size_t _pos = (h) & (_cap - 1); \ + typeof(*(map)->keys) _ik = (k); \ + typeof(*(map)->values) _iv = (v); \ + uint64_t _ih = (h); \ + for (;;) { \ + if ((map)->hashes[_pos] == 0) { \ + (map)->keys[_pos] = _ik; \ + (map)->values[_pos] = _iv; \ + (map)->hashes[_pos] = _ih; \ + (map)->count++; \ + break; \ + } \ + size_t _existing_dist = _HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap); \ + size_t _insert_dist = _HASHMAP_PROBE_DIST(_ih, _pos, _cap); \ + if (_insert_dist > _existing_dist) { \ + typeof(*(map)->keys) _tk = (map)->keys[_pos]; \ + typeof(*(map)->values) _tv = (map)->values[_pos]; \ + uint64_t _th = (map)->hashes[_pos]; \ + (map)->keys[_pos] = _ik; \ + (map)->values[_pos] = _iv; \ + (map)->hashes[_pos] = _ih; \ + _ik = _tk; \ + _iv = _tv; \ + _ih = _th; \ + } \ + _pos = (_pos + 1) & (_cap - 1); \ + } \ + } while (0) + +// Insert or update a key-value pair. Returns 0 or ENOMEM. +#define HASHMAP_INSERT(map, key, val, hash_fn, eq_fn) \ + ({ \ + errno_t _status = 0; \ + typeof(*(map)->keys) _k = (key); \ + uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \ + /* Check for existing key first. */ \ + bool _found = false; \ + if ((map)->capacity > 0) { \ + size_t _cap = (map)->capacity; \ + size_t _pos = _h & (_cap - 1); \ + for (size_t _d = 0; _d < _cap; _d++) { \ + if ((map)->hashes[_pos] == 0) \ + break; \ + if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \ + break; \ + if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \ + (map)->values[_pos] = (val); \ + _found = true; \ + break; \ + } \ + _pos = (_pos + 1) & (_cap - 1); \ + } \ + } \ + if (!_found) { \ + if ((map)->count * 4 >= (map)->capacity * 3 || (map)->capacity == 0) { \ + _HASHMAP_GROW(map, hash_fn, eq_fn); \ + if ((map)->keys == nullptr || (map)->values == nullptr || (map)->hashes == nullptr) { \ + _status = ENOMEM; \ + } \ + } \ + if (_status == 0) { \ + _HASHMAP_INSERT_INNER(map, _k, (val), _h); \ + } \ + } \ + _status; \ + }) + +// Look up a key. Returns a pointer to the value, or nullptr if not found. +#define HASHMAP_GET(map, key, hash_fn, eq_fn) \ + ({ \ + typeof(*(map)->values)* _result = nullptr; \ + if ((map)->capacity > 0) { \ + typeof(*(map)->keys) _k = (key); \ + uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \ + size_t _cap = (map)->capacity; \ + size_t _pos = _h & (_cap - 1); \ + for (size_t _d = 0; _d < _cap; _d++) { \ + if ((map)->hashes[_pos] == 0) \ + break; \ + if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \ + break; \ + if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \ + _result = &(map)->values[_pos]; \ + break; \ + } \ + _pos = (_pos + 1) & (_cap - 1); \ + } \ + } \ + _result; \ + }) + +// Remove a key. Returns true if the key was found and removed. +// Uses backward-shift deletion to avoid tombstones. +#define HASHMAP_REMOVE(map, key, hash_fn, eq_fn) \ + ({ \ + bool _removed = false; \ + if ((map)->capacity > 0) { \ + typeof(*(map)->keys) _k = (key); \ + uint64_t _h = _HASHMAP_FIXHASH((hash_fn)(&_k, sizeof(_k))); \ + size_t _cap = (map)->capacity; \ + size_t _pos = _h & (_cap - 1); \ + for (size_t _d = 0; _d < _cap; _d++) { \ + if ((map)->hashes[_pos] == 0) \ + break; \ + if (_HASHMAP_PROBE_DIST((map)->hashes[_pos], _pos, _cap) < _d) \ + break; \ + if ((map)->hashes[_pos] == _h && (eq_fn)(&(map)->keys[_pos], &_k, sizeof(_k))) { \ + (map)->hashes[_pos] = 0; \ + (map)->count--; \ + /* Backward-shift: move subsequent displaced entries back. */ \ + size_t _empty = _pos; \ + size_t _next = (_pos + 1) & (_cap - 1); \ + while ((map)->hashes[_next] != 0 && _HASHMAP_PROBE_DIST((map)->hashes[_next], _next, _cap) > 0) { \ + (map)->keys[_empty] = (map)->keys[_next]; \ + (map)->values[_empty] = (map)->values[_next]; \ + (map)->hashes[_empty] = (map)->hashes[_next]; \ + (map)->hashes[_next] = 0; \ + _empty = _next; \ + _next = (_next + 1) & (_cap - 1); \ + } \ + _removed = true; \ + break; \ + } \ + _pos = (_pos + 1) & (_cap - 1); \ + } \ + } \ + _removed; \ + }) + +// Iterate over all occupied entries. `idx` is a size_t loop variable. +// Access keys and values via (map)->keys[idx] and (map)->values[idx]. +#define HASHMAP_FOREACH(map, idx) \ + for (size_t idx = 0; idx < (map)->capacity; idx++) \ + if ((map)->hashes[idx] != 0) diff --git a/include/zinnia/init.h b/include/zinnia/init.h new file mode 100644 index 0000000..a5cf695 --- /dev/null +++ b/include/zinnia/init.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include +#include + +#define __init __used, __section(".init.text"), __cold +#define __initdata __used, __section(".init.data") +#define __initdata_sorted(name) __used, __section(".init.data." name) + +struct boot_file { + phys_t data; + size_t length; + char path[128]; +}; + +struct boot_info { + char* cmdline; + struct phys_mem* mem_map; + size_t num_mem_maps; + phys_t phys_base; + uintptr_t virt_base; + uintptr_t hhdm_base; + struct boot_file* files; + size_t num_files; + phys_t rsdp; +}; + +extern uint8_t __ld_stack_top[]; +extern uint8_t __ld_stack_bottom[]; + +extern phys_t rsdp_addr; + +// Entry point for the kernel after arch-specific setup has finished. +void kernel_entry(); + +// Initializes the early parts of the kernel. +void kernel_early_init(); + +[[noreturn]] +void kernel_main(struct boot_info* info); diff --git a/include/zinnia/iovec.h b/include/zinnia/iovec.h new file mode 100644 index 0000000..72d2e1a --- /dev/null +++ b/include/zinnia/iovec.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +struct iovec_iter { + struct iovec* iovecs; + size_t iovecs_len; + size_t total_len; + size_t total_offset; + size_t current_idx; + size_t current_offset; +}; diff --git a/include/zinnia/irq.h b/include/zinnia/irq.h new file mode 100644 index 0000000..71ff500 --- /dev/null +++ b/include/zinnia/irq.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +enum irq_status : uint8_t { + IRQ_STATUS_HANDLED, + IRQ_STATUS_IGNORED, +}; + +enum irq_polarity : uint8_t { + IRQ_POLARITY_LOW, + IRQ_POLARITY_HIGH, +}; + +enum irq_trigger_mode : uint8_t { + IRQ_TRIGGER_EDGE, + IRQ_TRIGGER_LEVEL, +}; + +struct irq_handler { + enum irq_status (*handler)(void* self); + list_link(struct irq_handler*) next; +}; + +struct irq_line { + int (*set_config)(enum irq_trigger_mode mode, enum irq_polarity polarity); + void (*mask)(struct irq_line* self); + void (*unmask)(struct irq_line* self); + void (*eoi)(struct irq_line* self); + + list_head(struct irq_handler*) handlers; + bool is_busy; +}; + +void irq_line_attach(struct irq_line* self, struct irq_handler* handler); + +struct irq_percpu { + uint32_t level; + bool in_interrupt; +}; + +void irq_lock(); +void irq_unlock(); +bool irq_set_interrupted(bool is_interrupted); +void irq_raise(); diff --git a/include/zinnia/list.h b/include/zinnia/list.h new file mode 100644 index 0000000..d7c2114 --- /dev/null +++ b/include/zinnia/list.h @@ -0,0 +1,29 @@ +#pragma once + +#define list_head(type) \ + struct { \ + type _sl_head; \ + } + +#define list_link(type) \ + struct { \ + type _sl_link; \ + } + +#define list_is_empty(head) ((head)->_sl_head == NULL) +#define list_first(head) ((head)->_sl_head) +#define list_next(elm, field) ((elm)->field._sl_link) + +#define list_insert_end(slistelm, elm, field) \ + do { \ + list_next((elm), field) = list_next((slistelm), field); \ + list_next((slistelm), field) = (elm); \ + } while (0) + +#define list_insert_head(head, elm, field) \ + do { \ + list_next((elm), field) = list_first((head)); \ + list_first((head)) = (elm); \ + } while (0) + +#define list_foreach(var, head, field) for ((var) = list_first((head)); (var); (var) = list_next((var), field)) diff --git a/include/zinnia/mem/alloc.h b/include/zinnia/mem/alloc.h new file mode 100644 index 0000000..c05ce5f --- /dev/null +++ b/include/zinnia/mem/alloc.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +enum alloc_flags { + ALLOC_NOZERO = 1 << 0, // Don't zero out the allocated memory. + ALLOC_MEM32 = 1 << 1, // Allocated memory needs to fit inside 32 bits. + ALLOC_MEM20 = 1 << 2, // Allocated memory needs to fit inside 20 bits. +}; + +enum phys_mem_usage { + PHYS_RESERVED, + PHYS_USABLE, + PHYS_STATIC, + PHYS_RECLAIMABLE, +}; + +struct phys_mem { + phys_t address; + size_t length; + enum phys_mem_usage usage; +}; + +// Base address of the HHDM. +extern uintptr_t mem_hhdm_base; +#define HHDM_PTR(paddr) (void*)(mem_hhdm_base + (paddr)) + +// Allocates a region of memory which can be smaller than the page size. +// Returns `nullptr` if the allocator cannot provide an allocation for the given `length` + `flags`. +// Always returns `nullptr` if `length` is 0. +void* mem_alloc(size_t length, enum alloc_flags flags); + +// Frees an allocation created by `mem_alloc`. +// Passing `nullptr` is a no-op. +void mem_free(void* mem); + +errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out); +errno_t mem_phys_free(phys_t start, size_t num_pages); + +void mem_init(struct phys_mem* map, size_t length, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm); +void slab_init(); +void mem_phys_bootstrap(struct phys_mem* mem); +void mem_phys_init(struct phys_mem* map, size_t length); + +uintptr_t arch_mem_hhdm_addr(); +uintptr_t arch_vm_pfndb_addr(); +uintptr_t arch_mem_mapping_addr(); +size_t arch_mem_page_bits(); +size_t arch_mem_page_size(); +size_t arch_mem_level_bits(); +size_t arch_mem_num_levels(); + +static inline size_t mem_high_shift() { + return arch_mem_level_bits() * arch_mem_num_levels() + arch_mem_page_bits(); +} diff --git a/include/zinnia/mem/object.h b/include/zinnia/mem/object.h new file mode 100644 index 0000000..20f86d0 --- /dev/null +++ b/include/zinnia/mem/object.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +struct vmo { + errno_t (*get_page)(struct vmo* vmo, uintptr_t offset_idx, struct page** out); +}; + +errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read); +errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written); +errno_t vmo_copy( + struct vmo* target, + uintptr_t target_offset, + struct vmo* src, + uintptr_t src_offset, + size_t len, + size_t* actual_copied +); + +struct pager_ops { + errno_t (*get_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page** out); + errno_t (*put_page)(struct pager_ops* pager, uintptr_t offset_idx, struct page* page); +}; + +struct page_list { + uintptr_t offset; + struct page* value; + list_link(struct page_list*) next; +}; + +struct paged_vmo { + struct vmo object; + struct pager_ops source; + list_head(struct page_list*) cache; + struct spinlock lock; +}; + +errno_t vmo_new_paged(struct pager_ops ops, struct paged_vmo** out); +errno_t vmo_new_phys(struct paged_vmo** out); + +// Creates a VMO backed by a specific physical address range (e.g. for MMIO). +errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out); diff --git a/include/zinnia/mem/page.h b/include/zinnia/mem/page.h new file mode 100644 index 0000000..6b3c0dc --- /dev/null +++ b/include/zinnia/mem/page.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +enum page_type : uint32_t { + PAGE_PHYS = 0, // Regular physical memory. +}; + +struct page { + enum page_type type; + uint32_t flags; + size_t refcount; + union { + struct { + struct page* next; // Pointer to the next chunk. + size_t count; // Amount of free pages. + } freelist; + }; +}; +static_assert(0x1000 % sizeof(struct page) == 0, "struct must fit cleanly inside a page!"); +static_assert(sizeof(struct page) <= 64, "struct must be smaller than 64 bytes!"); + +void* vm_alloc(size_t length); +void vm_free(void* addr, size_t length); + +// Base address of the `struct page` array. +extern struct page* vm_pfndb; + +uintptr_t vm_page_addr(struct page* page); diff --git a/include/zinnia/mem/pager.h b/include/zinnia/mem/pager.h new file mode 100644 index 0000000..c73b083 --- /dev/null +++ b/include/zinnia/mem/pager.h @@ -0,0 +1,8 @@ +#pragma once + +#include +#include + +struct pager { + errno_t (*try_get_page)(struct pager* pager, uintptr_t offset); +}; diff --git a/include/zinnia/mem/pmap.h b/include/zinnia/mem/pmap.h new file mode 100644 index 0000000..12b499f --- /dev/null +++ b/include/zinnia/mem/pmap.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include + +enum pte_flags { + PTE_READ = 1 << 0, // Can read from this page. + PTE_WRITE = 1 << 1, // Can write to this page. + PTE_EXEC = 1 << 2, // Can execute code on this page. + PTE_USER = 1 << 3, // Can be accessed by the user. + PTE_DIR = 1 << 4, // Is a non-leaf page. +}; + +enum cache_mode { + CACHE_NONE, // None means "nothing explicit" here. + CACHE_WRITE_COMBINE, + CACHE_WRITE_THROUGH, + CACHE_WRITE_BACK, + CACHE_MMIO, +}; + +struct pmap { + phys_t root; + struct spinlock lock; + bool is_user; +}; + +errno_t pmap_kernel(struct pmap* pt, enum alloc_flags flags); +errno_t pmap_user(struct pmap* pt, enum alloc_flags flags); +errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache); +errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags); +errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr); +void pmap_set(struct pmap* pt); + +pte_t* pmap_pte_resolve(struct pmap* pt, virt_t virt); + +void pte_clear(pte_t* pte); +pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache); +bool pte_is_present(pte_t* pte); +bool pte_is_dir(pte_t* pte); +phys_t pte_address(pte_t* pte); diff --git a/include/zinnia/mmio.h b/include/zinnia/mmio.h new file mode 100644 index 0000000..0e0fa29 --- /dev/null +++ b/include/zinnia/mmio.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include + +struct mmio_view { + volatile void* base; + size_t len; +}; + +#define _MMIO_REG(x) \ + typedef struct { \ + size_t offset; \ + bool swapped; \ + } mmio_reg##x; + +_MMIO_REG(8) +_MMIO_REG(16) +_MMIO_REG(32) +_MMIO_REG(64) + +#define bswap_16(x) (uint16_t)x +#define bswap_32(x) (uint32_t)x +#define bswap_64(x) (uint64_t)x + +#define mmio_load(_view, _reg) \ + ({ \ + auto _mmio_ptr = _Generic( \ + (_reg), \ + mmio_reg8: (volatile uint8_t*)((_view)->base + (_reg).offset), \ + mmio_reg16: (volatile uint16_t*)((_view)->base + (_reg).offset), \ + mmio_reg32: (volatile uint32_t*)((_view)->base + (_reg).offset), \ + mmio_reg64: (volatile uint64_t*)((_view)->base + (_reg).offset) \ + ); \ + auto _mmio_val = *_mmio_ptr; \ + if ((_reg).swapped) \ + _Generic( \ + (_mmio_val), \ + char: (void)(_mmio_val), \ + uint8_t: (void)(_mmio_val), \ + uint16_t: (_mmio_val = bswap_16(_mmio_val)), \ + uint32_t: (_mmio_val = bswap_32(_mmio_val)), \ + uint64_t: (_mmio_val = bswap_64(_mmio_val)) \ + ); \ + _mmio_val; \ + }) + +#define mmio_store(_view, _reg, _val) \ + ({ \ + auto _mmio_val = _val; \ + if ((_reg).swapped) \ + _Generic( \ + (_mmio_val), \ + char: (void)(_mmio_val), \ + uint8_t: (void)(_mmio_val), \ + uint16_t: (_mmio_val = bswap_16(_mmio_val)), \ + uint32_t: (_mmio_val = bswap_32(_mmio_val)), \ + uint64_t: (_mmio_val = bswap_64(_mmio_val)) \ + ); \ + auto _mmio_ptr = _Generic( \ + (_reg), \ + mmio_reg8: (volatile uint8_t*)((_view)->base + (_reg).offset), \ + mmio_reg16: (volatile uint16_t*)((_view)->base + (_reg).offset), \ + mmio_reg32: (volatile uint32_t*)((_view)->base + (_reg).offset), \ + mmio_reg64: (volatile uint64_t*)((_view)->base + (_reg).offset) \ + ); \ + *_mmio_ptr = _mmio_val; \ + }) + +struct mmio_view mmio_new(phys_t addr, size_t length); +void mmio_free(struct mmio_view* view); diff --git a/include/zinnia/mutex.h b/include/zinnia/mutex.h new file mode 100644 index 0000000..1a48e90 --- /dev/null +++ b/include/zinnia/mutex.h @@ -0,0 +1,11 @@ +#pragma once + +struct task; + +struct mutex { + struct task* owner; + bool flag; +}; + +void mutex_lock(struct mutex* mutex); +void mutex_unlock(struct mutex* mutex); diff --git a/include/zinnia/panic.h b/include/zinnia/panic.h new file mode 100644 index 0000000..60fd18a --- /dev/null +++ b/include/zinnia/panic.h @@ -0,0 +1,6 @@ +#pragma once +#include + +// Stop all execution upon panic. +[[noreturn, __format(printf, 1, 2)]] +void panic(const char* msg, ...); diff --git a/include/zinnia/percpu.h b/include/zinnia/percpu.h new file mode 100644 index 0000000..6a4764f --- /dev/null +++ b/include/zinnia/percpu.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +ASSERT_TYPE(struct arch_percpu); + +// CPU-relative information. +struct percpu { + struct percpu* self; // A pointer to this structure. + size_t id; // The virtual ID of this CPU. + bool online; // Whether this CPU is initialized and active. + uintptr_t kernel_stack; // The kernel mode stack. + uintptr_t user_stack; // The user mode stack. + struct arch_percpu arch; // Architecture-specific fields. + struct irq_percpu irq; // IRQ information. + struct sched_percpu sched; // Scheduler information. +}; + +// Per-CPU data for the bootstrap processor. +extern struct percpu percpu_bsp; + +// Allocates a block of memory for a new CPU. +struct percpu* percpu_new(); + +// Gets the per-CPU data on the current CPU. +static inline struct percpu* percpu_get() { + return arch_percpu_get(); +} + +// Initializes the bootstrap processor. +static inline void percpu_bsp_init() { + arch_percpu_bsp_init(); +} + +// Initializes all processors. +static inline void percpu_init() { + arch_percpu_init(); +} diff --git a/include/zinnia/poll.h b/include/zinnia/poll.h new file mode 100644 index 0000000..197daed --- /dev/null +++ b/include/zinnia/poll.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +enum poll_flags : uint16_t { + POLLIN = __POLLIN, + POLLOUT = __POLLOUT, + POLLPRI = __POLLPRI, + POLLHUP = __POLLHUP, + POLLERR = __POLLERR, + POLLRDHUP = __POLLRDHUP, + POLLNVAL = __POLLNVAL, + POLLRDNORM = __POLLRDNORM, + POLLRDBAND = __POLLRDBAND, + POLLWRNORM = __POLLWRNORM, + POLLWRBAND = __POLLWRBAND, +}; diff --git a/include/zinnia/print.h b/include/zinnia/print.h new file mode 100644 index 0000000..c5044c5 --- /dev/null +++ b/include/zinnia/print.h @@ -0,0 +1,9 @@ +#pragma once + +#include +#include + +[[__format(printf, 1, 2)]] +void printf(const char* message, ...); + +void kvprintf(const char* message, va_list args); diff --git a/include/zinnia/refcount.h b/include/zinnia/refcount.h new file mode 100644 index 0000000..c4b8f4e --- /dev/null +++ b/include/zinnia/refcount.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +struct refcount { + size_t count; +}; + +static inline void refcount_inc(struct refcount* rc) { + rc->count++; +} + +struct foo { + int a; + struct refcount rc; +}; + +static void do_foo() { + struct foo f = {.a = 4, .rc = {0}}; + + refcount_inc(&f.rc); +} diff --git a/include/zinnia/sched.h b/include/zinnia/sched.h new file mode 100644 index 0000000..5246e3f --- /dev/null +++ b/include/zinnia/sched.h @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +ASSERT_TYPE(struct arch_context); +ASSERT_TYPE(struct arch_task_context); + +enum task_state { + TASK_STATE_RUNNING, // Task is active and running. + TASK_STATE_READY, // Task is active, but not currently scheduled. + TASK_STATE_BLOCKED, // Task is waiting on another object. + TASK_STATE_DEAD, // Task is killed. +}; + +#define TASK_NAME_MAX 128 +#define KERNEL_STACK_SIZE 0x4000 + +// A task is the smallest unit of the scheduler. +struct task { + size_t id; + char name[TASK_NAME_MAX]; + struct vas* space; + struct namespace* namespace; + enum task_state state; + struct arch_task_context context; + struct usercopy_region* usercopy_region; + uintptr_t kernel_stack; + uintptr_t user_stack; + size_t time_slice; + int8_t priority; + + tailq_link(struct task) next; +}; + +// Per-CPU data for scheduling. +struct sched_percpu { + struct task* current; + struct task* idle_task; + size_t preempt_level; + tailq_head(struct task) run_queue; +}; + +// Blocks the current task on the given wait queue. +// The caller must NOT hold any spinlocks. Returns after the task is woken. +void sched_block(struct sched_percpu* sched, struct wait_queue* wq); + +// Wakes the first task blocked on the given wait queue, if any. +void sched_wake(struct sched_percpu* sched, struct wait_queue* wq); + +// Wakes all tasks blocked on the given wait queue. +void sched_wake_all(struct sched_percpu* sched, struct wait_queue* wq); + +typedef void (*task_fn_t)(uintptr_t arg0); + +errno_t task_create( + const char* name, + struct vas* space, + struct namespace* ns, + task_fn_t entry, + uintptr_t arg0, + struct task** out +); +[[noreturn]] +void task_entry(task_fn_t entry, uintptr_t arg0); + +void sched_init(struct sched_percpu* sched); +void sched_reschedule(struct sched_percpu* sched); +void sched_yield(struct sched_percpu* sched); +void sched_add_task(struct sched_percpu* sched, struct task* task); +[[noreturn]] +void sched_to_user(uintptr_t ctx); +[[noreturn]] +void sched_to_user_context(uintptr_t ctx); + +// Info block for sched_to_user. Heap-allocated by the caller, freed by sched_to_user_arg. +struct task_start_info { + uintptr_t ip; + uintptr_t sp; + uintptr_t arg; +}; + +void arch_sched_preempt_disable(); +bool arch_sched_preempt_enable(); +void arch_sched_switch(struct task* from, struct task* to); +errno_t arch_task_init( + struct arch_task_context* context, + void* entry, + uintptr_t arg0, + uintptr_t stack_start, + bool is_user +); +[[noreturn]] +void arch_sched_jump_to_context(struct arch_context* context); +[[noreturn]] +void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg); diff --git a/include/zinnia/spin.h b/include/zinnia/spin.h new file mode 100644 index 0000000..0372df8 --- /dev/null +++ b/include/zinnia/spin.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +// Busy-waits in a loop until the lock is freed. +// Does not put the CPU to sleep. +struct spinlock { + uint32_t owner; + uint32_t next; +}; + +// Attempts to lock a spinlock. If it's already locked, waits until it's freed. +void spin_lock(struct spinlock* spin); + +// Unlocks a spinlock. +void spin_unlock(struct spinlock* spin); diff --git a/include/zinnia/syscall.h b/include/zinnia/syscall.h new file mode 100644 index 0000000..74765d7 --- /dev/null +++ b/include/zinnia/syscall.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +typedef void (*syscall_fn_t)(struct arch_context* ctx); + +void syscall_dispatch(struct arch_context* ctx); + +#define __SYSCALL_DEFINE(_idx, name) void syscall_##name(struct arch_context* ctx); +#include +#undef __SYSCALL_DEFINE diff --git a/include/zinnia/tailq.h b/include/zinnia/tailq.h new file mode 100644 index 0000000..bd7e4f3 --- /dev/null +++ b/include/zinnia/tailq.h @@ -0,0 +1,52 @@ +#pragma once + +#define tailq_head(type) \ + struct { \ + type* tqh_first; \ + type** tqh_last; \ + } + +#define tailq_link(type) \ + struct { \ + type* tqe_next; \ + type** tqe_prev; \ + } + +#define tailq_init(head) \ + do { \ + (head)->tqh_first = nullptr; \ + (head)->tqh_last = &(head)->tqh_first; \ + } while (0) + +#define tailq_first(head) ((head)->tqh_first) + +#define tailq_next(elm, field) ((elm)->field.tqe_next) + +#define tailq_empty(head) (tailq_first(head) == nullptr) + +#define tailq_insert_tail(head, elm, field) \ + do { \ + (elm)->field.tqe_next = nullptr; \ + (elm)->field.tqe_prev = (head)->tqh_last; \ + *(head)->tqh_last = (elm); \ + (head)->tqh_last = &(elm)->field.tqe_next; \ + } while (0) + +#define tailq_insert_head(head, elm, field) \ + do { \ + if (((elm)->field.tqe_next = (head)->tqh_first) != nullptr) \ + (head)->tqh_first->field.tqe_prev = &(elm)->field.tqe_next; \ + else \ + (head)->tqh_last = &(elm)->field.tqe_next; \ + (head)->tqh_first = (elm); \ + (elm)->field.tqe_prev = &(head)->tqh_first; \ + } while (0) + +#define tailq_remove(head, elm, field) \ + do { \ + if (((elm)->field.tqe_next) != nullptr) \ + (elm)->field.tqe_next->field.tqe_prev = (elm)->field.tqe_prev; \ + else \ + (head)->tqh_last = (elm)->field.tqe_prev; \ + *(elm)->field.tqe_prev = (elm)->field.tqe_next; \ + } while (0) diff --git a/include/zinnia/time.h b/include/zinnia/time.h new file mode 100644 index 0000000..e4ebd31 --- /dev/null +++ b/include/zinnia/time.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +struct timespec time_from_sec(uint64_t sec); +struct timespec time_from_nano(uint64_t nsec); +struct timespec time_from_milli(uint64_t msec); diff --git a/include/zinnia/types.h b/include/zinnia/types.h new file mode 100644 index 0000000..0bd2a7b --- /dev/null +++ b/include/zinnia/types.h @@ -0,0 +1,8 @@ +#pragma once + +#include +#include + +ASSERT_TYPE(phys_t); +ASSERT_TYPE(uintptr_t); +ASSERT_TYPE(reg_t); diff --git a/include/zinnia/usercopy.h b/include/zinnia/usercopy.h new file mode 100644 index 0000000..3a9b48d --- /dev/null +++ b/include/zinnia/usercopy.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +struct usercopy_region { + void (*start_ip)(); + void (*end_ip)(); + void (*fault_ip)(); +}; + +// Copies a block of data from user to kernel memory. +bool usercopy_read(void* dst, const __user void* src, size_t len); + +// Copies a block of data from kernel to user memory. +bool usercopy_write(__user void* dst, const void* src, size_t len); + +// Performs a strlen() on a user string. +bool usercopy_strlen(const __user char* str, size_t max, size_t* len); diff --git a/include/zinnia/utils.h b/include/zinnia/utils.h new file mode 100644 index 0000000..8c225d0 --- /dev/null +++ b/include/zinnia/utils.h @@ -0,0 +1,40 @@ +#pragma once + +#define ARRAY_SIZE(array) (sizeof(array) / sizeof(array[0])) + +#define ROUND_UP(value, to) (((value) + ((to) - 1)) / (to)) +#define ALIGN_DOWN(value, align) (((value) / (align)) * (align)) +#define ALIGN_UP(value, align) (ROUND_UP(value, align) * align) + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + +#define BIT_TEST(value, bit) (((value) & (1 << bit)) == (1 << bit)) + +#if defined(__has_builtin) +#if __has_builtin(__builtin_types_compatible_p) +// Get pointer to parent object given pointer to a struct or union member. +#define containerof(ptr, type, member) \ + ({ \ + static_assert( \ + __builtin_types_compatible_p(typeof(*(ptr)), typeof(((type*)0)->member)), \ + "Incompatible types for containerof" \ + ); \ + (type*)((size_t)(ptr) - offsetof(type, member)); \ + }) +#endif +#endif +#ifndef containerof +// Get pointer to parent object given pointer to a struct or union member. +#define containerof(ptr, type, member) (type*)((size_t)(ptr) - offsetof(type, member)) +#endif + +#define CONCAT(A, B) _CONCAT(A, B) +#define _CONCAT(A, B) A##B + +// Creates a unique identifier for a compilation unit. +#define UNIQUE_IDENT(ident) CONCAT(__unique_##ident, __COUNTER__) + +// Ensures that a type is properly defined at this point. +// This is used to ensure that e.g. architecture-specific structures are defined. +#define ASSERT_TYPE(type) static_assert(sizeof(type) == sizeof(type)) diff --git a/include/zinnia/vas.h b/include/zinnia/vas.h new file mode 100644 index 0000000..ea33ac0 --- /dev/null +++ b/include/zinnia/vas.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include +#include + +// Virtual address space. +struct vas { + // Architecture-specific virtual memory map implementation. + struct pmap pmap; +}; + +// Try to create a new, empty virtual address space. +errno_t vma_new(struct vas** out); + +// Delete a virtual address space. +void vma_delete(struct vas* vm); + +errno_t vma_map_vmo(struct vas* vm, struct vmo* vmo, uintptr_t addr, size_t len, enum flags, uintptr_t vmo_offset); + +// Remove any mappings in the given address range. +errno_t vma_unmap(struct vas* vm, uintptr_t vaddr, size_t size); + +// Change the protection of an existing mapping. +errno_t vma_protect(struct vas* vm, uintptr_t vaddr, size_t size, enum zn_vm_flags flags); + +// Try to handle a page fault. +bool vma_page_fault(struct vas* vm, uintptr_t vaddr, enum zn_vm_flags access_type); + +extern struct vas kernel_vas; diff --git a/include/zinnia/vec.h b/include/zinnia/vec.h new file mode 100644 index 0000000..ea6aada --- /dev/null +++ b/include/zinnia/vec.h @@ -0,0 +1,105 @@ +#pragma once + +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include +#include + +#define vec(T) \ + struct { \ + T* data; \ + size_t length; \ + size_t capacity; \ + } + +#define vec_init(vec) \ + do { \ + (vec)->data = nullptr; \ + (vec)->length = 0; \ + (vec)->capacity = 0; \ + } while (0) + +#define vec_destroy(vec) \ + do { \ + mem_free((vec)->data); \ + (vec)->data = nullptr; \ + (vec)->length = 0; \ + (vec)->capacity = 0; \ + } while (0) + +#define vec_length(vec) ((vec)->length) +#define vec_capacity(vec) ((vec)->capacity) +#define vec_empty(vec) ((vec)->length == 0) +#define vec_at(vec, i) ((vec)->data[i]) + +// Internal: grow to at least `needed` capacity. +#define _vec_grow(vec, needed) \ + do { \ + size_t _new_cap = (vec)->capacity == 0 ? 8 : (vec)->capacity; \ + while (_new_cap < (needed)) \ + _new_cap *= 2; \ + typeof((vec)->data) _new = mem_alloc(_new_cap * sizeof(*(vec)->data), ALLOC_NOZERO); \ + if (_new != nullptr && (vec)->length > 0) \ + memcpy(_new, (vec)->data, (vec)->length * sizeof(*(vec)->data)); \ + mem_free((vec)->data); \ + (vec)->data = _new; \ + (vec)->capacity = _new_cap; \ + } while (0) + +// Append an element. Returns 0 or ENOMEM. +#define vec_push(vec, val) \ + ({ \ + errno_t _status = 0; \ + if ((vec)->length >= (vec)->capacity) { \ + _vec_grow(vec, (vec)->length + 1); \ + if ((vec)->data == nullptr) \ + _status = ENOMEM; \ + } \ + if (_status == 0) \ + (vec)->data[(vec)->length++] = (val); \ + _status; \ + }) + +// Remove and return the last element. Undefined if empty. +#define vec_pop(vec) ((vec)->data[--(vec)->length]) + +// Insert at index, shifting subsequent elements right. Returns 0 or ENOMEM. +#define vec_insert(vec, i, val) \ + ({ \ + errno_t _status = 0; \ + size_t _idx = (i); \ + if ((vec)->length >= (vec)->capacity) { \ + _vec_grow(vec, (vec)->length + 1); \ + if ((vec)->data == nullptr) \ + _status = ENOMEM; \ + } \ + if (_status == 0) { \ + memmove(&(vec)->data[_idx + 1], &(vec)->data[_idx], ((vec)->length - _idx) * sizeof(*(vec)->data)); \ + (vec)->data[_idx] = (val); \ + (vec)->length++; \ + } \ + _status; \ + }) + +// Remove element at index, shifting subsequent elements left. +#define vec_remove(vec, i) \ + do { \ + size_t _idx = (i); \ + (vec)->length--; \ + if (_idx < (vec)->length) \ + memmove(&(vec)->data[_idx], &(vec)->data[_idx + 1], ((vec)->length - _idx) * sizeof(*(vec)->data)); \ + } while (0) + +// Remove element at index by swapping with the last element (O(1), unordered). +#define vec_swap_remove(vec, i) \ + do { \ + size_t _idx = (i); \ + (vec)->length--; \ + if (_idx < (vec)->length) \ + (vec)->data[_idx] = (vec)->data[(vec)->length]; \ + } while (0) + +// Clear all elements without freeing memory. +#define vec_clear(vec) ((vec)->length = 0) + +#define vec_foreach(vec, idx) for (size_t idx = 0; idx < (vec)->length; idx++) diff --git a/include/zinnia/vfs.h b/include/zinnia/vfs.h new file mode 100644 index 0000000..c6f8ad2 --- /dev/null +++ b/include/zinnia/vfs.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +struct vfs_mount { + struct refcount rc; +}; + +struct vfs_entry { + struct refcount rc; +}; + +struct vfs_path { + struct vfs_mount* mount; + struct vfs_entry* entry; +}; + +struct vfs_node; +struct vfs_regular_ops { + errno_t (*read)(struct vfs_node* file, struct iovec_iter* buffer, uint64_t offset, ssize_t* out_read); + errno_t (*write)(struct vfs_node* file, struct iovec_iter* buffer, uint64_t offset, ssize_t* out_written); + errno_t (*devctl)(struct vfs_node* file, ); +}; + +struct vfs_node { + struct refcount rc; + const struct vfs_regular_ops* ops; + struct vfs_vnode* vnode; + struct vfs_path path; + struct string abs_path; + bool has_path; +}; + +enum open_flags : uint32_t { + O_RDONLY = __O_RDONLY, + O_WRONLY = __O_WRONLY, + O_RDWR = __O_RDWR, + O_CREAT = __O_CREAT, + O_EXCL = __O_EXCL, + O_NOCTTY = __O_NOCTTY, + O_TRUNC = __O_TRUNC, + O_APPEND = __O_APPEND, + O_NONBLOCK = __O_NONBLOCK, + O_DSYNC = __O_DSYNC, + O_ASYNC = __O_ASYNC, + O_DIRECT = __O_DIRECT, + O_LARGEFILE = __O_LARGEFILE, + O_DIRECTORY = __O_DIRECTORY, + O_NOFOLLOW = __O_NOFOLLOW, + O_NOATIME = __O_NOATIME, + O_CLOEXEC = __O_CLOEXEC, + O_PATH = __O_PATH, + O_TMPFILE = __O_TMPFILE, + O_SYNC = __O_SYNC, + O_RSYNC = __O_RSYNC, + O_EXEC = __O_EXEC, + O_SEARCH = __O_SEARCH, + O_ACCMODE = __O_ACCMODE, +}; diff --git a/include/zinnia/wait.h b/include/zinnia/wait.h new file mode 100644 index 0000000..83d14f6 --- /dev/null +++ b/include/zinnia/wait.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +struct task; + +// A queue of tasks waiting for an event. +struct wait_queue { + tailq_head(struct task) waiters; + struct spinlock lock; +}; + +void wait_queue_init(struct wait_queue* wq); diff --git a/lib/meson.build b/lib/meson.build new file mode 100644 index 0000000..efb9e3f --- /dev/null +++ b/lib/meson.build @@ -0,0 +1,5 @@ +# Freestanding libc functions that are needed in the kernel + +kernel_sources += files( + 'string.c', +) diff --git a/lib/string.c b/lib/string.c new file mode 100644 index 0000000..69ae21f --- /dev/null +++ b/lib/string.c @@ -0,0 +1,164 @@ +#include +#include +#include +#include +#include + +[[__hot]] +size_t strlen(const char* str) { + size_t result = 0; + while (*str++) { + result++; + } + return result; +} + +[[__hot]] +size_t strnlen(const char* str, size_t len) { + size_t result = 0; + while (result < len && *str++) { + result++; + } + return result; +} + +[[__hot]] +int memcmp(const void* s1, const void* s2, size_t size) { + int diff = 0; + char* s1ptr = (char*)s1; + char* s2ptr = (char*)s2; + + for (size_t i = 0; i < size; i++) { + if (s1ptr[i] != s2ptr[i]) + diff++; + } + return diff; +} + +[[__hot]] +void* memcpy(void* restrict dest, const void* restrict src, size_t n) { + if (__unlikely(n == 0)) + return dest; + + size_t d = (size_t)dest; + size_t s = (size_t)src; + + if (d % sizeof(size_t) != 0 || s % sizeof(size_t) != 0) { + while (n && (d % sizeof(size_t) != 0) && (s % sizeof(size_t) != 0)) { + *((uint8_t*)d) = *((uint8_t*)s); + d++; + s++; + n--; + } + } + + size_t* qword_dest = (size_t*)d; + const size_t* qword_src = (const size_t*)s; + const size_t word_count = n / sizeof(size_t); + + for (size_t i = 0; i < word_count; i++) { + qword_dest[i] = qword_src[i]; + } + + size_t remaining_bytes = n % sizeof(size_t); + d = (size_t)(qword_dest + word_count); + s = (size_t)(qword_src + word_count); + + while (remaining_bytes--) { + *((uint8_t*)d) = *((uint8_t*)s); + d++; + s++; + } + + return dest; +} + +[[__hot]] +void* memmove(void* dstptr, const void* srcptr, size_t size) { + uint8_t* dst = (uint8_t*)dstptr; + const uint8_t* src = (const uint8_t*)srcptr; + if (dst < src) { + for (size_t i = 0; i < size; i++) + dst[i] = src[i]; + } else { + for (size_t i = size; i != 0; i--) + dst[i - 1] = src[i - 1]; + } + return dstptr; +} + +[[__hot]] +void* memset(void* dest, int value, size_t n) { + if (__unlikely(n == 0)) + return dest; + + size_t d = (size_t)dest; + + if (d % sizeof(size_t) != 0) { + while (n && (d % sizeof(size_t) != 0)) { + *((uint8_t*)d) = value; + d++; + n--; + } + } + + size_t* qword_dest = (size_t*)d; + size_t qword_value = value; + qword_value |= (qword_value << 8); + qword_value |= (qword_value << 16); + qword_value |= (qword_value << 32); + + const size_t word_count = n / sizeof(size_t); + + for (size_t i = 0; i < word_count; i++) { + qword_dest[i] = qword_value; + } + + size_t remaining_bytes = n % sizeof(size_t); + d = (size_t)(qword_dest + word_count); + + while (remaining_bytes--) { + *((uint8_t*)d) = value; + d++; + } + + return dest; +} + +char* strncpy(char* restrict dst, const char* restrict src, size_t len) { + size_t src_len = strnlen(src, len) + 1; + return memcpy(dst, src, MIN(len, src_len)); +} + +int strncmp(const char* str1, const char* str2, size_t len) { + while (len && *str1 && (*str1 == *str2)) { + ++str1; + ++str2; + --len; + } + if (len == 0) { + return 0; + } else { + return (*(unsigned char*)str1 - *(unsigned char*)str2); + } +} + +int strcmp(const char* str1, const char* str2) { + while (*str1 && (*str1 == *str2)) { + ++str1; + ++str2; + } + return (*(unsigned char*)str1 - *(unsigned char*)str2); +} + +char* strchr(const char* s, int c) { + char ch = (char)c; + while (true) { + char cur = *s; + if (cur == ch) + return (char*)s; + if (cur == 0) + return nullptr; + s++; + } +} diff --git a/logo.svg b/logo.svg new file mode 100644 index 0000000..f844c21 --- /dev/null +++ b/logo.svg @@ -0,0 +1,110 @@ + + + + + Zinnia + + + + + + + + + + + + + + + Zinnia + + + conkkerxd + + + A logo for Zinnia. + + + + diff --git a/mem/init.c b/mem/init.c new file mode 100644 index 0000000..26999a6 --- /dev/null +++ b/mem/init.c @@ -0,0 +1,167 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern uint8_t __ld_text_start[]; +extern uint8_t __ld_text_end[]; +extern uint8_t __ld_rodata_start[]; +extern uint8_t __ld_rodata_end[]; +extern uint8_t __ld_data_start[]; +extern uint8_t __ld_data_end[]; +extern uint8_t __ld_kernel_start[]; + +struct vas kernel_vas = {0}; + +[[__init]] +void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) { + // This function creates a kernel page table and initializes all memory managers. + + const size_t pgsz = arch_mem_page_size(); + + printf("Memory map:\n"); + for (size_t i = 0; i < map_len; i++) { + if (map[i].length == 0) + continue; + + const char* label = nullptr; + switch (map[i].usage) { + case PHYS_RESERVED: + label = "Reserved"; + break; + case PHYS_USABLE: + label = "Usable"; + break; + case PHYS_STATIC: + label = "Static"; + break; + case PHYS_RECLAIMABLE: + label = "Reclaimable"; + break; + } + + printf("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label); + } + + // Set up the bootstrap allocator. + struct phys_mem* largest = map; + for (size_t i = 0; i < map_len; i++) { + if (map[i].length > largest->length && map[i].usage == PHYS_USABLE) + largest = &map[i]; + } + mem_phys_bootstrap(largest); + printf( + "Using region [%p - %p] for bootstrap allocator\n", + (void*)largest->address, + (void*)(largest->address + largest->length - 1) + ); + + // Set the HHDM base address to the address given by the loader. + // We must not keep any virtual addresses to this region, + // since we're likely going to map it at a different base address. + mem_hhdm_base = tmp_hhdm; + + assert(pmap_new_kernel(&kernel_vas.pmap, 0) == 0, "Unable to allocate the kernel page table\n"); + + // text + printf("Mapping text segment at %p\n", __ld_text_start); + for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) { + errno_t status = pmap_map( + &kernel_vas.pmap, + (uintptr_t)p, + (phys_t)(p - __ld_kernel_start + kernel_phys), + PTE_READ | PTE_EXEC, + CACHE_NONE + ); + assert(!status, "Failed to map %p with error %i\n", p, status); + } + + // rodata + printf("Mapping rodata segment at %p\n", __ld_rodata_start); + for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) { + errno_t status = pmap_map( + &kernel_vas.pmap, + (uintptr_t)p, + (phys_t)(p - __ld_kernel_start + kernel_phys), + PTE_READ, + CACHE_NONE + ); + assert(!status, "Failed to map %p with error %i\n", p, status); + } + + // data + printf("Mapping data segment at %p\n", __ld_data_start); + for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) { + errno_t status = pmap_map( + &kernel_vas.pmap, + (uintptr_t)p, + (phys_t)(p - __ld_kernel_start + kernel_phys), + PTE_READ | PTE_WRITE, + CACHE_NONE + ); + assert(!status, "Failed to map %p with error %i\n", p, status); + } + + // The kernel address space is divided into 4 segments. + // For example, they could live at: + // HHDM FFFF'8000'0000'0000 + // PFNDB FFFF'A000'0000'0000 + // Mappings FFFF'C000'0000'0000 + // Kernel FFFF'FFFF'8000'0000 + + // Map all physical memory to the HHDM address. + tmp_hhdm = arch_mem_hhdm_addr(); + for (size_t i = 0; i < map_len; i++) { + if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC) + continue; + for (size_t p = 0; p <= map[i].length; p += pgsz) { + uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm); + phys_t paddr = (phys_t)(map[i].address + p); + errno_t status = pmap_map(&kernel_vas.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_NONE); + assert(!status, "Failed to map HHDM page %p to %p with error %i\n", (void*)vaddr, (void*)paddr, status); + } + } + mem_hhdm_base = tmp_hhdm; + + // Switch to our own page table. + pmap_set(&kernel_vas.pmap); + + // We record metadata for every single page of available memory in a large array. + // This array is contiguous in virtual memory, but is sparsely populated. + // Only those array entries which represent usable memory are mapped. + for (size_t i = 0; i < map_len; i++) { + if (map[i].length == 0 || map[i].usage != PHYS_USABLE) + continue; + + const size_t num_pages = (map[i].length + pgsz - 1) / pgsz; + const size_t metadata_size = num_pages * sizeof(struct page); + const uintptr_t metadata_start_vaddr = arch_vm_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page)); + const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size; + + // Find the page-aligned boundaries that contain this metadata. + const uintptr_t aligned_start = ALIGN_DOWN(metadata_start_vaddr, pgsz); + const uintptr_t aligned_end = ALIGN_UP(metadata_end_vaddr, pgsz); + const size_t length = aligned_end - aligned_start; + const uintptr_t vaddr = aligned_start; + + for (size_t page = 0; page < length; page += pgsz) { + phys_t paddr; + assert(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!\n"); + pmap_map(&kernel_vas.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_NONE); + } + } + vm_pfndb = (struct page*)arch_vm_pfndb_addr(); + + // We don't need the bootstrap allocator from this point on. + // Initialize the real page allocator. + mem_phys_init(map, map_len); + + slab_init(); + + printf("Memory initialization complete\n"); +} diff --git a/mem/meson.build b/mem/meson.build new file mode 100644 index 0000000..c12d046 --- /dev/null +++ b/mem/meson.build @@ -0,0 +1,11 @@ +kernel_sources += files( + 'init.c', + 'mmio.c', + 'pmap.c', + 'pmm.c', + 'slab.c', + 'usercopy.c', + 'virt.c', + 'vas.c', + 'vmo.c', +) diff --git a/mem/mmio.c b/mem/mmio.c new file mode 100644 index 0000000..ca3c47c --- /dev/null +++ b/mem/mmio.c @@ -0,0 +1,22 @@ +#include +#include +#include +#include +#include + +void* mmio_new(phys_t addr, size_t length) { + void* ptr = vm_alloc(length); + for (size_t i = 0; i < length; i += arch_mem_page_size()) { + + if (pmap_map(&kernel_vas.pmap, (uintptr_t)ptr + i, addr + i, PTE_READ | PTE_WRITE, CACHE_MMIO)) { + vm_free(ptr, length); + return nullptr; + } + } + + return ptr; +} + +void mmio_free(void* ptr, size_t length) { + // TODO +} diff --git a/mem/pmap.c b/mem/pmap.c new file mode 100644 index 0000000..a5ab52f --- /dev/null +++ b/mem/pmap.c @@ -0,0 +1,104 @@ +#include +#include +#include +#include +#include +#include + +errno_t pmap_kernel(struct pmap* pt, enum alloc_flags flags) { + flags &= ~ALLOC_NOZERO; + + phys_t addr; + errno_t status = mem_phys_alloc(1, flags, &addr); + if (status) + return status; + + struct pmap result = { + .root = addr, + .lock = (struct spinlock){0}, + .is_user = false, + }; + + *pt = result; + return 0; +} + +errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) { + phys_t user_l1; + errno_t status = mem_phys_alloc(1, 0, &user_l1); + if (status) + return status; + + void* user_l1_ptr = HHDM_PTR(user_l1); + void* kernel_l1_ptr = HHDM_PTR(kernel_vas.pmap.root); + memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size()); + + struct pmap result = { + .root = user_l1, + .lock = (struct spinlock){0}, + .is_user = true, + }; + + *pt = result; + return 0; +} + +// Gets a reference to the PTE at the given virtual address. +// If `check_only` is set, only checks if the PTE exists, +// and doesn't allocate new levels if they don't already exist. +// If it can't allocate a page if it has to, returns `nullptr`. +static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) { + pte_t* current_head = HHDM_PTR(pt->root); + size_t index = 0; + + for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) { + const size_t addr_mask = (1 << arch_mem_level_bits()) - 1; + const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level); + const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0); + + index = (vaddr >> addr_shift) & addr_mask; + + // The last level is used to access the actual PTE, so break the loop then. + // We still need to update the index beforehand, that's why we can't just end early. + if (level == 0) + break; + + pte_t* pte = ¤t_head[index]; + + if (pte_is_present(pte)) { + // Get the next level. + *pte = pte_build(pte_address(pte), level_flags, CACHE_NONE); + current_head = HHDM_PTR(pte_address(pte)); + } else { + // If the current level isn't present, we can skip the rest. + if (check_only) + return nullptr; + + phys_t addr; + if (mem_phys_alloc(1, 0, &addr) != 0) + return nullptr; + + *pte = pte_build(addr, level_flags, CACHE_NONE); + current_head = HHDM_PTR(addr); + } + } + + return ¤t_head[index]; +} + +errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) { + spin_lock(&pt->lock); + + errno_t status = 0; + pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false); + if (pte == nullptr) { + status = ENOMEM; + goto fail; + } + + *pte = pte_build(paddr, flags, cache); + +fail: + spin_unlock(&pt->lock); + return status; +} diff --git a/mem/pmm.c b/mem/pmm.c new file mode 100644 index 0000000..f2a3ea0 --- /dev/null +++ b/mem/pmm.c @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +uintptr_t mem_hhdm_base = 0; + +static errno_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out); +static errno_t (*free_fn)(phys_t start, size_t num_pages); + +errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) { + return alloc_fn(num_pages, flags, out); +} + +errno_t mem_phys_free(phys_t start, size_t num_pages) { + return free_fn(start, num_pages); +} + +static struct phys_mem bump_mem = {0}; +static struct phys_mem* bump_region = nullptr; + +static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) { + const size_t bytes = num_pages * arch_mem_page_size(); + if (((intptr_t)bump_mem.length - bytes) <= 0) + return ENOMEM; + + *out = bump_mem.address; + bump_mem.length -= bytes; + bump_mem.address += bytes; + + if (!(flags & ALLOC_NOZERO)) { + memset(HHDM_PTR(*out), 0, bytes); + } + + return 0; +} + +static errno_t bump_free(phys_t addr, size_t num_pages) { + // We don't free bootstrap memory. + panic("Attempted to free bootstrap memory!"); +} + +void mem_phys_bootstrap(struct phys_mem* mem) { + alloc_fn = bump_alloc; + free_fn = bump_free; + + // Save the region by reference and value. + // We can't directly modify the region by reference, because that + // interferes with e.g. HHDM mapping logic. + bump_mem = *mem; + bump_region = mem; +} + +static struct page* pmm_head = nullptr; +static struct spinlock pmm_lock = {0}; +static size_t pmm_total_free = 0; + +static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) { + spin_lock(&pmm_lock); + + const size_t bytes = num_pages * arch_mem_page_size(); + phys_t limit = UINTPTR_MAX; + if (flags & ALLOC_MEM20) { + limit = (phys_t)1 << 20; + } else if (flags & ALLOC_MEM32) { + limit = (phys_t)1 << 32; + } + + struct page* it = pmm_head; + struct page* prev_it = nullptr; + + while (it) { + const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size(); + // If the region is too high up or doesn't have enough pages left, continue searching. + if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) { + prev_it = it; + it = it->freelist.next; + continue; + } + + // If the current block has exactly enough pages left, consume it entirely. + if (__unlikely(it->freelist.count == num_pages)) { + if (prev_it) { + prev_it->freelist.next = it->freelist.next; + } else { + pmm_head = it->freelist.next; + } + + it->freelist.next = nullptr; + it->freelist.count = 0; + + if (__likely(out)) + *out = page_addr; + } + // If not, we can just shrink the block. + else { + it->freelist.count -= num_pages; + + if (__likely(out)) + *out = page_addr + (it->freelist.count * arch_mem_page_size()); + } + goto success; + } + + spin_unlock(&pmm_lock); + return ENOMEM; + +success: + // If the NOZERO flag is *not* specified, zero memory. + if (__likely(out) && __likely(!(flags & ALLOC_NOZERO))) { + memset(HHDM_PTR(*out), 0, bytes); + } + pmm_total_free -= num_pages; + spin_unlock(&pmm_lock); + return 0; +} + +static errno_t freelist_free(phys_t addr, size_t num_pages) { + spin_lock(&pmm_lock); + + struct page* page = &vm_pfndb[addr / arch_mem_page_size()]; + page->freelist.count = num_pages; + page->freelist.next = pmm_head; + pmm_head = page; + + spin_unlock(&pmm_lock); + return 0; +} + +void mem_phys_init(struct phys_mem* map, size_t length) { + alloc_fn = freelist_alloc; + free_fn = freelist_free; + bump_region->address = bump_mem.address; + bump_region->length = bump_mem.length; + + for (size_t i = 0; i < length; i++) { + // Regions smaller than a page are useless. + if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE) + continue; + + struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()]; + page->freelist.count = map[i].length / arch_mem_page_size(); + page->freelist.next = pmm_head; + pmm_head = page; + pmm_total_free += page->freelist.count; + } + + const size_t free_bytes = (pmm_total_free * arch_mem_page_size()); + printf("Total available memory: %zu MiB\n", free_bytes / 1024 / 1024); +} diff --git a/mem/slab.c b/mem/slab.c new file mode 100644 index 0000000..8bd6a11 --- /dev/null +++ b/mem/slab.c @@ -0,0 +1,143 @@ +#include +#include +#include +#include +#include + +typedef struct { + size_t num_pages; // Amount of pages connected to this slab. + size_t size; // Size of this slab. +} slab_info; + +struct slab { + struct spinlock lock; // Access lock. + size_t ent_size; // Size of one entry. + void** head; +}; + +typedef struct { + struct slab* slab; +} slab_header; + +// Initializes the SLAB structures. + +static struct slab slabs[8] = {0}; + +static errno_t slab_new(struct slab* slab, size_t size) { + slab->lock = (struct spinlock){0}; + + // Allocate a new page for the head. + phys_t addr; + errno_t status = mem_phys_alloc(1, 0, &addr); + if (status) + return status; + + slab->head = (void**)(addr + arch_mem_hhdm_addr()); + slab->ent_size = size; + + const size_t offset = ALIGN_UP(sizeof(slab_header), size); + const size_t available_size = arch_mem_page_size() - offset; + + slab_header* ptr = (slab_header*)slab->head; + ptr->slab = slab; + slab->head = (void**)((void*)slab->head + offset); + + void** arr = slab->head; + const size_t max = available_size / size - 1; + const size_t fact = size / sizeof(void*); + + for (size_t i = 0; i < max; i++) { + arr[i * fact] = &arr[(i + 1) * fact]; + } + arr[max * fact] = nullptr; + + return 0; +} + +void slab_init(void) { + slab_new(&slabs[0], 16); + slab_new(&slabs[1], 32); + slab_new(&slabs[2], 64); + slab_new(&slabs[3], 128); + slab_new(&slabs[4], 256); + slab_new(&slabs[5], 512); + slab_new(&slabs[6], 1024); + slab_new(&slabs[7], 2048); +} + +static void* slab_do_alloc(struct slab* slab) { + spin_lock(&slab->lock); + + if (__unlikely(slab->head == nullptr)) + slab_new(slab, slab->ent_size); + + void** old_free = slab->head; + slab->head = *old_free; + memset(old_free, 0, slab->ent_size); + + spin_unlock(&slab->lock); + return old_free; +} + +static void slab_do_free(struct slab* slab, void* addr) { + spin_lock(&slab->lock); + + if (__unlikely(addr == nullptr)) + goto cleanup; + + void** new_head = addr; + *new_head = slab->head; + slab->head = new_head; + +cleanup: + spin_unlock(&slab->lock); +} + +static inline struct slab* slab_find_size(size_t size) { + for (size_t i = 0; i < ARRAY_SIZE(slabs); i++) { + if (slabs[i].ent_size >= size) + return &slabs[i]; + } + return nullptr; +} + +void* mem_alloc(size_t size, enum alloc_flags flags) { + if (__unlikely(size == 0)) + return nullptr; + + struct slab* slab = slab_find_size(size); + if (slab != nullptr) { + return slab_do_alloc(slab); + } + + size_t num_pages = ROUND_UP(size, arch_mem_page_size()); + + // Allocate the pages plus an additional page for metadata. + phys_t ret; + errno_t status = mem_phys_alloc(num_pages + 1, 0, &ret); + if (__unlikely(status != 0)) + return nullptr; + + ret = ret + (phys_t)arch_mem_hhdm_addr(); + // Write metadata into the first page. + slab_info* info = (slab_info*)ret; + info->num_pages = num_pages; + info->size = size; + // Skip the first page and return the next one. + return (void*)(ret + arch_mem_page_size()); +} + +void mem_free(void* addr) { + if (__unlikely(addr == nullptr)) + return; + + // If the address is page aligned. + if ((size_t)addr == ALIGN_DOWN((size_t)addr, arch_mem_page_size())) { + slab_info* info = (slab_info*)(addr - arch_mem_page_size()); + mem_phys_free(((phys_t)info - (phys_t)arch_mem_hhdm_addr()), info->num_pages + 1); + return; + } + + slab_header* header = (slab_header*)(ALIGN_DOWN((size_t)addr, arch_mem_page_size())); + slab_do_free(header->slab, addr); +} diff --git a/mem/usercopy.c b/mem/usercopy.c new file mode 100644 index 0000000..595103c --- /dev/null +++ b/mem/usercopy.c @@ -0,0 +1,15 @@ +#include +#include +#include + +bool usercopy_read(void* dst, const __user void* src, size_t len) { + return arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region); +} + +bool usercopy_write(__user void* dst, const void* src, size_t len) { + return arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region); +} + +bool usercopy_strlen(const __user char* str, size_t max, size_t* len) { + return arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region); +} diff --git a/mem/vas.c b/mem/vas.c new file mode 100644 index 0000000..0073df9 --- /dev/null +++ b/mem/vas.c @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include + +// Try to create a new, empty virtual address space. +errno_t vma_new(struct vas** out) { + debug_assert(out != nullptr, "Missing out"); + + errno_t res; + if (!out) + return EINVAL; + + struct vas* vmspace = mem_alloc(sizeof(struct vas), 0); + if (!vmspace) + return ENOMEM; + + res = pmap_new_user(&vmspace->pmap, 0); + if (res) + goto err1; + + *out = vmspace; + return 0; + +err1: + mem_free(vmspace); +err0: + return res; +} + +// Delete a virtual address space. +void vma_delete(struct vas* vm) {} + +// Remove any mappings in the given address range. +// May fail if there is no memory and a mapping needs to be split. +errno_t vma_unmap(struct vas* vm, uintptr_t vaddr, size_t size); + +// Change the protection of an existing mapping. +// May fail if there is no memory and a mapping needs to be split. +errno_t vma_protect(struct vas* vm, uintptr_t vaddr, size_t size, enum zn_vm_flags flags); + +// Try to handle a page fault. +bool vma_page_fault(struct vas* vm, uintptr_t vaddr, enum zn_vm_flags access_type) { + // TODO + return false; +} + +errno_t vma_map_vmo( + struct vas* vas, + struct vmo* vmo, + uintptr_t addr, + size_t len, + enum zn_vm_flags flags, + uintptr_t vmo_offset +) { + const size_t page_size = arch_mem_page_size(); + + if (addr % page_size != vmo_offset % page_size) + return EINVAL; + + size_t start_page = addr / page_size; + const size_t addr_offset = addr % page_size; + const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size; + size_t end_page = start_page + pages; + + if (!vas || !vmo) + return EINVAL; + + const size_t first_vmo_page = vmo_offset / page_size; + + for (size_t p = start_page; p < end_page; ++p) { + const size_t page_index = p - start_page; + struct page* page_obj; + errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj); + if (status != 0) + return status; + + phys_t paddr = (phys_t)vm_page_addr(page_obj); + + enum pte_flags pf = 0; + if (flags & ZN_VM_MAP_READ) + pf |= PTE_READ; + if (flags & ZN_VM_MAP_WRITE) + pf |= PTE_WRITE; + if (flags & ZN_VM_MAP_EXEC) + pf |= PTE_EXEC; + if (vas->pmap.is_user) + pf |= PTE_USER; + + status = pmap_map(&vas->pmap, p * page_size, paddr, pf, CACHE_NONE); + if (status != 0) + return status; + } + + return 0; +} diff --git a/mem/virt.c b/mem/virt.c new file mode 100644 index 0000000..ce8d293 --- /dev/null +++ b/mem/virt.c @@ -0,0 +1,26 @@ +#include +#include +#include +#include +#include + +struct page* vm_pfndb = nullptr; + +// TODO: Replace with actual allocator. + +static size_t mapping_offset = 0; + +void* vm_alloc(size_t length) { + uintptr_t base = arch_mem_mapping_addr(); + size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size())); + + return (void*)(base + offset); +} + +void vm_free(void* ptr, size_t length) { + // TODO +} + +uintptr_t vm_page_addr(struct page* page) { + return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size(); +} diff --git a/mem/vmo.c b/mem/vmo.c new file mode 100644 index 0000000..17a38f2 --- /dev/null +++ b/mem/vmo.c @@ -0,0 +1,196 @@ +#include +#include +#include +#include +#include +#include +#include + +errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) { + const size_t page_size = arch_mem_page_size(); + size_t progress = 0; + errno_t status = 0; + + while (progress < len) { + const size_t misalign = (progress + offset) % page_size; + const size_t page_index = (progress + offset) / page_size; + const size_t copy_size = MIN(page_size - misalign, len - progress); + + struct page* p; + status = obj->get_page(obj, page_index, &p); + if (status != 0) + break; + + uintptr_t page_addr = vm_page_addr(p); + + memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size); + progress += copy_size; + } + + if (actual_read) + *actual_read = progress; + return status; +} + +errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) { + const size_t page_size = arch_mem_page_size(); + size_t progress = 0; + errno_t status = 0; + + while (progress < len) { + const size_t misalign = (progress + offset) % page_size; + const size_t page_index = (progress + offset) / page_size; + const size_t copy_size = MIN(page_size - misalign, len - progress); + + struct page* p; + status = obj->get_page(obj, page_index, &p); + if (status != 0) + break; + + uintptr_t page_addr = vm_page_addr(p); + + memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size); + progress += copy_size; + } + + if (actual_written) + *actual_written = progress; + return status; +} + +errno_t vmo_copy( + struct vmo* target, + uintptr_t target_offset, + struct vmo* src, + uintptr_t src_offset, + size_t len, + size_t* actual_copied +) { + const size_t page_size = arch_mem_page_size(); + size_t progress = 0; + errno_t status = 0; + + while (progress < len) { + const size_t target_misalign = (progress + target_offset) % page_size; + const size_t src_misalign = (progress + src_offset) % page_size; + const size_t target_page_index = (progress + target_offset) / page_size; + const size_t src_page_index = (progress + src_offset) / page_size; + + const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress); + + struct page* tpage; + status = target->get_page(target, target_page_index, &tpage); + if (status != 0) + break; + + struct page* spage; + status = src->get_page(src, src_page_index, &spage); + if (status != 0) + break; + + uintptr_t taddr = vm_page_addr(tpage); + uintptr_t saddr = vm_page_addr(spage); + + memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size); + progress += copy_size; + } + + if (actual_copied) + *actual_copied = progress; + return status; +} + +static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) { + struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object); + + // Try to find the requested page in the cache. + struct page_list* iter; + list_foreach(iter, &paged->cache, next) { + if (iter->offset != offset_idx) + continue; + + // We found a cached page! + *out = iter->value; + return 0; + } + + // Get page from out pager if we don't have one already. + struct page* new_page; + errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page); + if (status) + return status; + + struct page_list* entry = mem_alloc(sizeof(struct page_list), 0); + if (!entry) + return ENOMEM; + + entry->offset = offset_idx; + entry->value = new_page; + + // Add page to the object cache. + list_insert_head(&paged->cache, entry, next); + + *out = new_page; + return 0; +} + +errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) { + struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0); + if (!result) + return ENOMEM; + + result->source = pager; + result->object.get_page = paged_get_page; + + *out = result; + return 0; +} + +static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) { + phys_t addr; + errno_t status = mem_phys_alloc(1, 0, &addr); + if (status != 0) + return status; + size_t idx = addr / arch_mem_page_size(); + *out = &vm_pfndb[idx]; + return 0; +} + +static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) { + // We do nothing here. + return 0; +} + +static const struct pager_ops phys_pager = { + .get_page = phys_get_page, + .put_page = phys_put_page, +}; + +errno_t vmo_new_phys(struct paged_vmo** out) { + return vmo_new_paged(phys_pager, out); +} + +struct phys_range_vmo { + struct vmo object; + uintptr_t start; + size_t len; +}; + +static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) { + struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object); + *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx]; + return 0; +} + +errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) { + struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0); + if (!vmo) + return ENOMEM; + + vmo->object.get_page = phys_range_get_page; + vmo->start = ALIGN_DOWN(addr, arch_mem_page_size()); + vmo->len = ALIGN_UP(length, arch_mem_page_size()); + + *out = &vmo->object; + return 0; +} diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..194c8ce --- /dev/null +++ b/meson.build @@ -0,0 +1,119 @@ +project( + 'zinnia', + license: 'MIT', + version: '0.1.0', + meson_version: '>=1.3.0', + default_options: [ + 'c_std=gnu23', + 'werror=true', + 'warning_level=2', + 'strip=true', + ], +) + +if get_option('headers_only') + install_subdir('include/uapi', install_dir: get_option('includedir')) + pkg = import('pkgconfig') + pkg.generate(name: 'zinnia', description: 'Zinnia user API') + subdir_done() +endif + +add_languages('c', native: false) + +# Check if we're using a sane compiler. +c_compiler = meson.get_compiler('c') + +# Check for LP64 +assert(c_compiler.sizeof('short') == 2) +assert(c_compiler.sizeof('int') == 4) +assert(c_compiler.sizeof('long') == 8) +assert(c_compiler.sizeof('long long') == 8) + +# We require a compiler that supports the GNU C dialect. +assert(c_compiler.has_define('__GNUC__')) + +add_project_arguments( + '-Wno-unused', + '-Wno-unused-parameter', + '-fno-stack-protector', + '-ffunction-sections', + '-fdata-sections', + language: 'c', +) + +add_project_link_arguments( + '-znoexecstack', + '-Wl,--gc-sections', + '-Wl,--build-id=none', + language: 'c', +) + +if target_machine.cpu_family() == 'x86_64' + add_project_arguments('-masm=intel', language: 'c') +endif + +common_includes = include_directories('include') + +cfg = configuration_data() +cfg.set_quoted('ZINNIA_VERSION', meson.project_version()) +cfg.set_quoted('ZINNIA_ARCH', target_machine.cpu_family()) +cfg.set_quoted('ZINNIA_COMPILER_ID', c_compiler.get_id() + ' ' + c_compiler.version()) +cfg.set_quoted('ZINNIA_LINKER_ID', c_compiler.get_linker_id()) + +kernel_c_args = [ + '-ffreestanding', + '-nostdinc', # We provide our own headers. + '-fno-stack-protector', + '-fno-omit-frame-pointer', + '-fno-pie', +] + +kernel_lds = meson.current_source_dir() / 'boot' / 'kernel.ld' + +kernel_link_args = [ + '-ffreestanding', + '-nostdlib', + '-static', + '-T' + kernel_lds, +] + +kernel_includes = [ + include_directories( + 'arch' / target_machine.cpu_family() / 'include', + ), +] + +kernel_sources = [] + +configure_file(configuration: cfg, output: 'config.h') + +# Goes first so we can add arch-specific flags to drivers. +subdir('arch' / target_machine.cpu_family()) + +subdir('device') +subdir('fs') + +subdir('boot') +subdir('mem') +subdir('core') +subdir('syscalls') + +subdir('lib') + +executable( + 'zinnia', + include_directories: [ + common_includes, + kernel_includes, + uacpi_includes, + ], + c_args: kernel_c_args, + link_args: kernel_link_args, + link_depends: kernel_lds, + sources: [ + kernel_sources, + uacpi_sources, + ], + install: true, + install_dir: '/boot', +) diff --git a/meson.options b/meson.options new file mode 100644 index 0000000..1531305 --- /dev/null +++ b/meson.options @@ -0,0 +1,14 @@ +# Meta options +option( + 'headers_only', + type: 'boolean', + value: false, + description: 'Only install headers', +) + +# Core options +option('sched', type: 'combo', choices: ['simple'], value: 'simple') + +# Firmware +option('acpi', type: 'boolean', value: true) +option('open_firmware', type: 'boolean', value: true) diff --git a/subprojects/uacpi b/subprojects/uacpi new file mode 160000 index 0000000..9c9b26d --- /dev/null +++ b/subprojects/uacpi @@ -0,0 +1 @@ +Subproject commit 9c9b26d6291a1cdd9014cc5bb6b03e596697cbfd diff --git a/syscalls/archctl.c b/syscalls/archctl.c new file mode 100644 index 0000000..b1ab46b --- /dev/null +++ b/syscalls/archctl.c @@ -0,0 +1,8 @@ +#include +#include + +errno_t arch_archctl(zn_archctl_t op, uintptr_t value); + +errno_t syscall_archctl(struct arch_context* ctx) { + return arch_archctl((zn_archctl_t)ctx->ARCH_CTX_A0, (uintptr_t)ctx->ARCH_CTX_A1); +} diff --git a/syscalls/dispatch.c b/syscalls/dispatch.c new file mode 100644 index 0000000..5c3b802 --- /dev/null +++ b/syscalls/dispatch.c @@ -0,0 +1,25 @@ +#include +#include +#include +#include +#include +#include +#include + +static const syscall_fn_t dispatch_table[] = { +#undef __ZINNIA_UAPI_SYSCALL_NUMBERS_H // We want to include the header again +#define __SYSCALL_DEFINE(idx, name) [idx] = syscall_##name, +#include +#undef __SYSCALL_DEFINE +}; + +void syscall_dispatch(struct arch_context* ctx) { + const size_t num = ctx->ARCH_CTX_NUM; + + if (num >= ARRAY_SIZE(dispatch_table)) { + ctx->ARCH_CTX_RET0 = ENOSYS; + return; + } + + dispatch_table[num](ctx); +} diff --git a/syscalls/futex.c b/syscalls/futex.c new file mode 100644 index 0000000..13d02f9 --- /dev/null +++ b/syscalls/futex.c @@ -0,0 +1,17 @@ +#include +#include +#include + +errno_t syscall_futex_wait(struct arch_context* ctx) { + __user int* addr = (__user int*)ctx->ARCH_CTX_A0; + int expected = (int)ctx->ARCH_CTX_A1; + + return futex_wait(addr, expected); +} + +errno_t syscall_futex_wake(struct arch_context* ctx) { + __user int* addr = (__user int*)ctx->ARCH_CTX_A0; + int count = (int)ctx->ARCH_CTX_A1; + + return futex_wake(addr, count); +} diff --git a/syscalls/handle.c b/syscalls/handle.c new file mode 100644 index 0000000..213556b --- /dev/null +++ b/syscalls/handle.c @@ -0,0 +1,48 @@ +#include +#include +#include +#include +#include +#include +#include + +errno_t syscall_handle_clone(struct arch_context* ctx) { + zn_handle_t handle = ctx->ARCH_CTX_A0; + zn_rights_t cloned_rights = ctx->ARCH_CTX_A1; + __user zn_handle_t* cloned = (__user zn_handle_t*)ctx->ARCH_CTX_A2; + + struct task* current = percpu_get()->sched.current; + struct namespace* ns = current->namespace; + + struct namespace_desc desc; + errno_t status = namespace_get(ns, handle, &desc); + if (status != 0) + return status; + + zn_handle_t new_handle; + status = namespace_clone_desc(ns, &desc, cloned_rights, &new_handle); + if (status != 0) + return status; + + if (!usercopy_write(cloned, &new_handle, sizeof(new_handle))) { + namespace_del_desc(ns, new_handle); + return ZN_ERR_BAD_BUFFER; + } + + return 0; +} + +errno_t syscall_handle_drop(struct arch_context* ctx) { + zn_handle_t handle = ctx->ARCH_CTX_A0; + struct task* current = percpu_get()->sched.current; + + return namespace_del_desc(current->namespace, handle); +} + +errno_t syscall_handle_validate(struct arch_context* ctx) { + zn_handle_t handle = ctx->ARCH_CTX_A0; + struct task* current = percpu_get()->sched.current; + + struct namespace_desc desc; + return namespace_get(current->namespace, handle, &desc); +} diff --git a/syscalls/meson.build b/syscalls/meson.build new file mode 100644 index 0000000..b1120c4 --- /dev/null +++ b/syscalls/meson.build @@ -0,0 +1,4 @@ +kernel_sources += files( + 'archctl.c', + 'dispatch.c', +) diff --git a/syscalls/random.c b/syscalls/random.c new file mode 100644 index 0000000..89d5ff3 --- /dev/null +++ b/syscalls/random.c @@ -0,0 +1,10 @@ +#include +#include + +errno_t syscall_random_entropy(struct arch_context* ctx) { + return ZN_ERR_UNSUPPORTED; +} + +errno_t syscall_random_get(struct arch_context* ctx) { + return ZN_ERR_UNSUPPORTED; +} diff --git a/syscalls/system.c b/syscalls/system.c new file mode 100644 index 0000000..b981e14 --- /dev/null +++ b/syscalls/system.c @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include +#include + +errno_t syscall_log(struct arch_context* ctx) { + __user const char* msg = (__user const char*)ctx->ARCH_CTX_A0; + size_t len = ctx->ARCH_CTX_A1; + + char* buf = mem_alloc(len, ALLOC_NOZERO); + if (!buf) { + return ENOMEM; + } + + if (usercopy_read(buf, msg, len)) { + console_write(buf, len); + + mem_free(buf); + return 0; + } + + mem_free(buf); + return ZN_ERR_BAD_BUFFER; +} + +errno_t syscall_page_size(struct arch_context* ctx) { + __user size_t* out = (__user size_t*)ctx->ARCH_CTX_A0; + + size_t size = arch_mem_page_size(); + if (!usercopy_write(out, &size, sizeof(size))) + return ZN_ERR_BAD_BUFFER; + + return 0; +} + +errno_t syscall_time(struct arch_context* ctx) { + __user uint64_t* out = (__user uint64_t*)ctx->ARCH_CTX_A0; + + uint64_t time = clock_get_elapsed_ns(); + if (!usercopy_write(out, &time, sizeof(time))) + return ZN_ERR_BAD_BUFFER; + + return 0; +} diff --git a/syscalls/thread.c b/syscalls/thread.c new file mode 100644 index 0000000..f6d46cb --- /dev/null +++ b/syscalls/thread.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +errno_t syscall_thread_create(struct arch_context* ctx) { + zn_handle_t ns_handle = ctx->ARCH_CTX_A0; + __user const char* name = (__user const char*)ctx->ARCH_CTX_A1; + size_t name_len = ctx->ARCH_CTX_A2; + enum zn_thread_flags flags = ctx->ARCH_CTX_A3; + __user zn_handle_t* out = (__user zn_handle_t*)ctx->ARCH_CTX_A4; + + struct task* current = percpu_get()->sched.current; + // Resolve namespace. + struct namespace* ns; + if (ns_handle == ZN_HANDLE_THIS_NS) { + ns = current->namespace; + } else { + struct namespace_desc ns_desc; + errno_t status = namespace_get(current->namespace, ns_handle, &ns_desc); + if (status != 0) + return status; + if (ns_desc.type != NAMESPACE_DESC_NS) + return ZN_ERR_BAD_HANDLE; + ns = ns_desc.value.namespace; + } + + // Copy name from userspace. + if (name_len == 0 || name_len >= TASK_NAME_MAX) + return EINVAL; + + char task_name[TASK_NAME_MAX] = {}; + if (!usercopy_read(task_name, name, name_len)) + return ZN_ERR_BAD_BUFFER; + + // Create the task sharing the caller's address space. + // Uses sched_to_user as a placeholder entry — the real IP/SP come from thread_start. + struct task* new_task; + errno_t status = task_create(task_name, current->space, ns, sched_to_user, 0, &new_task); + if (status != 0) + return status; + + // Register the task as a handle in the caller's namespace. + struct namespace_desc desc = { + .type = NAMESPACE_DESC_TASK, + .rights = ZN_RIGHTS_ALL, + .value.task = new_task, + }; + + zn_handle_t handle; + status = namespace_add_desc(current->namespace, desc, &handle); + if (status != 0) + return status; + + if (!usercopy_write(out, &handle, sizeof(handle))) { + namespace_del_desc(current->namespace, handle); + return ZN_ERR_BAD_BUFFER; + } + + return 0; +} + +errno_t syscall_thread_start(struct arch_context* ctx) { + zn_handle_t thread_handle = ctx->ARCH_CTX_A0; + uintptr_t ip = ctx->ARCH_CTX_A1; + uintptr_t sp = ctx->ARCH_CTX_A2; + uintptr_t arg = ctx->ARCH_CTX_A3; + + struct task* current = percpu_get()->sched.current; + struct sched_percpu* sched = &percpu_get()->sched; + + // Look up the task. + struct namespace_desc desc; + errno_t status = namespace_get(current->namespace, thread_handle, &desc); + if (status != 0) + return status; + if (desc.type != NAMESPACE_DESC_TASK) + return ZN_ERR_BAD_HANDLE; + + struct task* task = desc.value.task; + + // Allocate a start info block for the thread entry. + struct task_start_info* info = mem_alloc(sizeof(struct task_start_info), 0); + if (!info) + return ENOMEM; + + info->ip = ip; + info->sp = sp; + info->arg = arg; + + // Re-initialize the task's arch context to enter user mode via sched_to_user_arg. + status = + arch_task_init(&task->context, sched_to_user, (uintptr_t)info, task->kernel_stack + KERNEL_STACK_SIZE, true); + if (status != 0) { + mem_free(info); + return status; + } + + // Add to run queue. + sched_add_task(sched, task); + return 0; +} + +errno_t syscall_thread_exit(struct arch_context* ctx) { + struct sched_percpu* sched = &percpu_get()->sched; + sched->current->state = TASK_STATE_DEAD; + sched_yield(sched); + unreachable(); +} diff --git a/syscalls/vas.c b/syscalls/vas.c new file mode 100644 index 0000000..2e94473 --- /dev/null +++ b/syscalls/vas.c @@ -0,0 +1,6 @@ +#include +#include + +errno_t syscall_vas_create(struct arch_context* ctx) { + return ZN_ERR_UNSUPPORTED; +}