marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: marv7000/zinnia:32f1ff25f7dfed98ad457c65bceabfac098810d7
Branches Tags
marv7000/zinnia:rewrite marv7000/zinnia:microkernel marv7000/zinnia:main
..
compare: marv7000/zinnia:main
Branches Tags
marv7000/zinnia:rewrite marv7000/zinnia:microkernel marv7000/zinnia:main

4 Commits
32f1ff25f7 .. main

Author SHA1 Message Date
marv7000 fce4d38045 meson: Fix noexecstack on clang 2026-04-25 15:00:22 +02:00
marv7000 7d1b867c36 fixes 2026-02-05 00:58:38 +01:00
marv7000 d22b96885a rename logo 2026-02-04 02:00:17 +01:00
marv7000 0e01345abf Initial commit 2026-02-04 01:58:49 +01:00
329 changed files with 4234 additions and 15937 deletions
Expand all files Collapse all files
.clang-format
+8 -12
View File
@@ -51,16 +51,12 @@ SpaceInEmptyBlock: false
SpacesBeforeTrailingComments: 1
IncludeBlocks: Merge
IncludeCategories:
- Regex: "^<zinnia/.*>"
Priority: 1
- Regex: "^<common/.*>"
Priority: 2
- Regex: "^<kernel/.*>"
Priority: 3
- Regex: "^<vdso/.*>"
Priority: 4
- Regex: '^<.*\.h>'
Priority: 5
- Regex: '^".*\.h"'
Priority: 6
- Regex: "^<zinnia/.*>"
Priority: 1
- Regex: "^<kernel/.*>"
Priority: 2
- Regex: '^<.*\.h>'
Priority: 3
- Regex: '^".*\.h"'
Priority: 4
SortIncludes: CaseInsensitive
.clangd
+1 -1
View File
@@ -1,2 +1,2 @@
Style:
AngledHeaders: [ "zinnia/.*", "uapi/.*" ]
AngledHeaders: ["zinnia/.*", " kernel/.*"]
.gitignore
+9 -4
View File
@@ -1,8 +1,13 @@
.config*
build/
# IDE Files
.vscode/
.idea/
.cache/
.wraplock
# Build artifacts
*.o
*.a
*.so
__pycache__/
compile_commands.json
# Default build dir
build/
.gitmodules
-3
View File
@@ -1,3 +0,0 @@
[submodule "subprojects/uacpi"]
path = subprojects/uacpi
url = https://github.com/uACPI/uacpi
Kconfig
-9
View File
@@ -1,9 +0,0 @@
mainmenu "Zinnia Kernel Configuration"
config KERNEL_FLAVOR
string "Kernel flavor name"
default "vanilla"
source "arch/$(ARCH)/Kconfig"
source "core/Kconfig"
source "dev/Kconfig"
LICENSE
-1
View File
@@ -1 +0,0 @@
LICENSES/ISC.txt
LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Marvin Friedrich
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
LICENSES/ISC.txt
-7
View File
@@ -1,7 +0,0 @@
ISC License
Copyright 2026 Marvin Friedrich and contributors
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
LICENSES/Unlicense.txt
-10
View File
@@ -1,10 +0,0 @@
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <http://unlicense.org/>
README.md
+23 -13
View File
@@ -1,26 +1,30 @@
<img src="logo.svg" width="10%"/>
<img src="zinnia.svg" width="10%"/>
# Zinnia
![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=green)
![GitHub top language](https://img.shields.io/github/languages/top/zinnia-os/zinnia)
![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=blue)
![GitHub Repo stars](https://img.shields.io/github/stars/zinnia-os/zinnia?style=flat)
![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/zinnia-os/zinnia/ci.yml)
![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/zinnia-os/zinnia?style=flat)
![Discord](https://img.shields.io/discord/1305893510704267327?logo=discord&logoColor=white&label=discord)
Zinnia is a lightweight Unix-like monolithic kernel targeting 64-bit devices.
# zinnia
## Building Zinnia
zinnia is a lightweight Unix-like microkernel targeting 64-bit devices.
To build the kernel you need:
- `meson`
- a `kconfig` frontend (like `kconfiglib` via `pip`)
- a GNU-like C compiler with support for C23 (e.g. `gcc` or `clang`)
> [!IMPORTANT]
> Please note that this project is a work in progress.
> Some parts may not work at all and/or will drastically change over time.
## Building
> [!NOTE]
> This repository contains only the kernel and drivers.
> If you want to get a bootable image, you might want to check out
> **https://github.com/zinnia-os/bootstrap** instead.
To build the kernel you need `meson` and a C23-compatible GNU-like compiler.
Currently supported are GCC and Clang toolchains.
To configure, run:
```sh
alldefconfig
meson setup $build_dir
```
@@ -28,3 +32,9 @@ And to build:
```sh
meson compile -C $build_dir
```
To cross-compile, you should follow the Meson cross-compilation guide.
## Debugging
Follow the debugging guide at **https://github.com/zinnia-os/bootstrap**
arch/x86_64/Kconfig
-16
View File
@@ -1,16 +0,0 @@
menu "x86_64 Architecture options"
config X86_64
bool
default y
select ACPI
select HAVE_MMU
select HAVE_MMU_GENERIC
help
64-bit x86 CPU support for AMD and Intel CPUs.
config TSC
bool "Invariant Timestamp Counter support"
default y
endmenu
arch/x86_64/apic.c
-157
View File
@@ -1,157 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/clock.h>
#include <zinnia/mmio.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/cpu.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/percpu.h>
static uint32_t lapic_msr(size_t offset) {
return 0x800 + (offset >> 4);
}
static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
if (lapic->xapic_regs)
return mmio_reg_read(lapic->xapic_regs, reg);
else
return asm_rdmsr(lapic_msr(reg.offset));
}
static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) {
if (lapic->xapic_regs)
mmio_reg_write(lapic->xapic_regs, reg, value);
else
asm_wrmsr(lapic_msr(reg.offset), value);
}
static uint64_t lapic_read_icr(struct local_apic* lapic) {
if (lapic->xapic_regs) {
uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO);
uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI);
return (hi << 32) | lo;
} else {
return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset));
}
}
static void lapic_write_icr(struct local_apic* lapic, uint64_t value) {
if (lapic->xapic_regs) {
mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32));
mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value);
} else {
asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value);
}
}
void lapic_init(struct local_apic* lapic) {
uint64_t apic_msr = asm_rdmsr(0x1B);
apic_msr |= 1 << 11; // Enable APIC.
struct cpuid cpuid = asm_cpuid(1, 0);
if (cpuid.ecx & CPUID_1C_X2APIC)
apic_msr |= 1 << 10;
else {
uintptr_t vaddr = vm_alloc(0x1000);
pmap_map(&kernel_vma.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
lapic->xapic_regs = (void*)vaddr;
assert(lapic->xapic_regs, "Failed to allocate virtual memory!");
}
asm_wrmsr(0x1B, apic_msr);
lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
if (lapic->xapic_regs)
lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id);
// Reset the TPR.
lapic_write_reg(lapic, APIC_REG_TPR, 0);
// Enable APIC bit in the SIVR.
uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR);
sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS);
sivr |= mmio_field(APIC_SIVR_ENABLE, 1);
lapic_write_reg(lapic, APIC_REG_SIVR, sivr);
if (lapic->xapic_regs) {
lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
// Logical destination = LAPIC ID.
lapic_write_reg(lapic, APIC_REG_LDR, lapic_read_reg(lapic, APIC_REG_ID));
}
// TODO: Parse MADT and setup NMI sources.
// Tell the APIC timer to divide by 16.
lapic_write_reg(lapic, APIC_REG_DCR, 3);
// Set the timer to the highest value possible.
lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0xFFFF'FFFF);
// Sleep for 10 milliseconds.
clock_spin_ns(10'000'000);
lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
}
void lapic_eoi(struct local_apic* lapic) {
lapic_write_reg(lapic, APIC_REG_EOI, 0);
}
constexpr uint64_t the_icr =
mmio_field(APIC_ICR_VECTOR, 128) | mmio_field(APIC_ICR_DELIVERY_MODE, APIC_DELIVERY_STARTUP);
void lapic_send_ipi(
struct local_apic* lapic,
uint32_t target,
uint8_t vector,
enum apic_delivery_mode delivery_mode,
enum apic_dest_mode dest_mode,
enum apic_level level,
enum apic_trigger_mode trigger_mode
) {
uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector);
icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode);
icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode);
icr |= mmio_field(APIC_ICR_LEVEL, level);
icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode);
if (lapic->xapic_regs)
icr |= mmio_field(APIC_ICR_DEST_XAPIC, target);
else
icr |= mmio_field(APIC_ICR_DEST_X2APIC, target);
lapic_write_icr(lapic, icr);
if (lapic->xapic_regs) {
while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING)
arch_cpu_pause();
}
}
constexpr uint16_t PIC1_COMMAND_PORT = 0x20;
constexpr uint16_t PIC1_DATA_PORT = 0x21;
constexpr uint16_t PIC2_COMMAND_PORT = 0xA0;
constexpr uint16_t PIC2_DATA_PORT = 0xA1;
void pic_disable() {
// We initialize the PIC properly, but completely disable it and use the APIC in favor of it.
// Remap IRQs so they start at 0x20 since interrupts 0x00..0x1F are used by CPU exceptions.
asm_outb(PIC1_COMMAND_PORT, 0x11);
asm_outb(PIC1_DATA_PORT, 0x20);
asm_outb(PIC1_DATA_PORT, 0x04);
asm_outb(PIC1_DATA_PORT, 0x01);
asm_outb(PIC1_DATA_PORT, 0xFF);
// Same for the slave PIC.
asm_outb(PIC2_COMMAND_PORT, 0x11);
asm_outb(PIC2_DATA_PORT, 0x28);
asm_outb(PIC2_DATA_PORT, 0x02);
asm_outb(PIC2_DATA_PORT, 0x01);
asm_outb(PIC2_DATA_PORT, 0xFF);
}
arch/x86_64/archctl.c
-32
View File
@@ -1,32 +0,0 @@
#include <zinnia/errno.h>
#include <zinnia/syscall.h>
#include <arch/archctl.h>
#include <arch/mem.h>
#include <uapi/archctl.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
struct syscall_ret arch_archctl(uintptr_t op, uintptr_t arg) {
struct syscall_ret ret = {};
switch (op) {
case __ARCHCTL_SET_FSBASE:
if (!arch_mem_addr_is_user(arg)) {
ret.error = EINVAL;
return ret;
}
asm_wrmsr(MSR_FS_BASE, arg);
return ret;
case __ARCHCTL_SET_GSBASE:
if (!arch_mem_addr_is_user(arg)) {
ret.error = EINVAL;
return ret;
}
// Update the inactive GSBASE MSR, not ours.
asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
return ret;
default:
ret.error = EINVAL;
return ret;
}
}
arch/x86_64/gdt.c
-77
View File
@@ -1,77 +0,0 @@
#include <zinnia/percpu.h>
#include <stddef.h>
#include <string.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/percpu.h>
#include <x86_64/tss.h>
const struct gdt gdt_base = {
.null = {0},
.kernel_code32 = {.val = 0x00cf9b000000ffff},
.kernel_data32 = {.val = 0x00cf93000000ffff},
.kernel_code64 = {.val = 0x00a09b0000000000},
.kernel_data64 = {.val = 0x0000930000000000},
.user_code32 = {.val = 0x00cffb000000ffff},
.user_data = {.val = 0x0000f30000000000},
.user_code64 = {.val = 0x00a0fb0000000000},
.tss = {
.limit0 = sizeof(struct tss) - 1,
.access = 0x89, // Present | DPL=0 | System | 64-bit TSS
.limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F,
.reserved = 0,
},
};
void gdt_load() {
struct gdt* gdt = &percpu_ref(arch_data)->gdt;
memcpy(gdt, &gdt_base, sizeof(gdt_base));
struct tss* tss = &percpu_ref(arch_data)->tss;
tss->iopb = sizeof(struct tss);
uintptr_t tss_addr = (uintptr_t)tss;
gdt->tss.base0 = tss_addr;
gdt->tss.base1 = tss_addr >> 16;
gdt->tss.base2 = tss_addr >> 24;
gdt->tss.base3 = tss_addr >> 32;
struct gdtr gdtr = {
.limit = sizeof(struct gdt) - 1,
.base = gdt,
};
asm volatile("lgdt %0" ::"m"(gdtr));
// Save the contents of MSR_GS_BASE, as they get cleared by a write to `gs`.
uintptr_t gs = asm_rdmsr(MSR_GS_BASE);
// Flush and reload the segment registers.
asm volatile(
"push %0\n"
"lea rax, [rip + 1f]\n"
"push rax\n"
"retfq\n"
"1:\n"
"mov ax, %1\n"
"mov ds, ax\n"
"mov es, ax\n"
"mov fs, ax\n"
"mov gs, ax\n"
"mov ss, ax\n"
:
: "i"(offsetof(struct gdt, kernel_code64)), "i"(offsetof(struct gdt, kernel_data64))
: "rax", "memory"
);
asm_wrmsr(MSR_GS_BASE, gs);
asm volatile(
"mov ax, %c0\n"
"ltr ax"
:
: "i"(offsetof(struct gdt, tss))
: "rax", "memory"
);
}
arch/x86_64/hpet.c
-85
View File
@@ -1,85 +0,0 @@
#include <zinnia/acpi.h>
#include <zinnia/assert.h>
#include <zinnia/clock.h>
#include <zinnia/cmdline.h>
#include <zinnia/initgraph.h>
#include <zinnia/mmio.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <uacpi/acpi.h>
#include <uacpi/status.h>
#include <uacpi/tables.h>
#include <uacpi/uacpi.h>
static bool use_hpet = true;
static void hpet_option(bool use) {
use_hpet = use;
}
CMDLINE_OPTION("x86_64.hpet", "Enable High Precision Event Timer clock", hpet_option);
struct hpet {
struct clock clock;
void* regs;
uint32_t period;
};
constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0};
constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32};
constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10};
constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1};
constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0};
static void hpet_reset(struct clock* c) {
struct hpet* hpet = containerof(c, struct hpet, clock);
mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0);
}
static uint64_t hpet_get_elapsed_ns(struct clock* c) {
struct hpet* hpet = containerof(c, struct hpet, clock);
uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER);
return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
}
static struct hpet hpet_clock = {
.clock = {
.name = "hpet",
.priority = 128,
.reset = hpet_reset,
.get_elapsed_ns = hpet_get_elapsed_ns,
},
};
static void hpet_init(struct boot_info*) {
if (!use_hpet)
return;
uacpi_table table = {0};
if (uacpi_table_find_by_signature("HPET", &table) != UACPI_STATUS_OK) {
log("x86_64/hpet: No HPET table found\n");
return;
}
struct acpi_hpet* hpet = table.ptr;
uintptr_t vaddr = vm_alloc(0x1000);
pmap_map(&kernel_vma.pmap, vaddr, hpet->address.address, PTE_READ | PTE_WRITE, CACHE_MMIO);
hpet_clock.regs = (void*)vaddr;
uacpi_table_unref(&table);
log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG);
cfg |= mmio_field(HPET_CFG_ENABLE, 1);
mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg);
uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP);
hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap);
assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!");
}
INIT_TASK(hpet_stage, "arch.x86_64.hpet", hpet_init);
INIT_DEPENDS(hpet_stage, acpi_stage);
INIT_ENTAILS(hpet_stage, percpu_stage);
arch/x86_64/idt.c
-258
View File
@@ -1,258 +0,0 @@
#include <zinnia/assert.h>
#include <zinnia/compiler.h>
#include <zinnia/irq.h>
#include <zinnia/panic.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/usercopy.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/percpu.h>
struct [[__packed]] idt_entry {
uint16_t base0;
uint16_t selector;
uint8_t ist;
uint8_t attrib;
uint16_t base1;
uint32_t base2;
uint32_t reserved;
};
static_assert(sizeof(struct idt_entry) == 16);
struct [[__packed]] idtr {
uint16_t limit;
struct idt_entry* base;
};
static_assert(sizeof(struct idtr) == 10);
void interrupt_handler(struct arch_context* ctx) {
struct percpu* cpu = percpu_get();
struct arch_percpu* arch_cpu = percpu_ref(arch_data);
uint64_t isr = ctx->isr;
switch (isr) {
case IDT_PF:
uint64_t cr2;
asm volatile("mov %0, cr2" : "=r"(cr2));
const struct vma_fault_info info = {
.ip = ctx->rip,
.addr = cr2,
.caused_by_user = (ctx->cs & CPL_USER) == CPL_USER,
.caused_by_write = ctx->error & (1 << 1),
.caused_by_fetch = ctx->error & (1 << 4),
.page_was_present = ctx->error & (1 << 0),
};
struct task* current = cpu->sched.current;
const bool irqs_were_on = ctx->rflags & RFLAGS_IF;
if (irqs_were_on)
irq_set_state(true);
const bool handled = current && current->space && vma_page_fault(current->space, &info);
if (irqs_were_on)
irq_set_state(false);
if (handled)
break;
if (current && current->usercopy_region) {
struct usercopy_region* region = current->usercopy_region;
if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) {
ctx->rip = (uintptr_t)region->fault_ip;
break;
}
}
panic(
"Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n",
info.caused_by_write ? "write to"
: info.caused_by_fetch ? "execute on"
: "read from",
info.page_was_present ? "present" : "non-present",
info.addr,
info.ip
);
case IDT_SPURIOUS:
break;
case IDT_IPI_RESCHED:
lapic_eoi(&arch_cpu->lapic);
arch_sched_preempt_disable();
if (arch_sched_preempt_enable())
sched_reschedule(&cpu->sched);
break;
case IDT_IPI_SHOOTDOWN:
lapic_eoi(&arch_cpu->lapic);
pmap_shootdown_handle();
break;
default:
if (isr < 0x20)
panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
assert(line, "Unhandled interrupt on ISR %lu", isr);
irq_line_raise(line);
}
}
#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
[[__naked]]
void interrupt_return() {
asm volatile(
"pop r15\n"
"pop r14\n"
"pop r13\n"
"pop r12\n"
"pop r11\n"
"pop r10\n"
"pop r9\n"
"pop r8\n"
"pop rsi\n"
"pop rdi\n"
"pop rbp\n"
"pop rdx\n"
"pop rcx\n"
"pop rbx\n"
"pop rax\n"
// Change GS back if we came from user mode.
"cmp word ptr [rsp + %c0], %c1\n"
"je 2f\n"
"swapgs\n"
"2:\n"
// Skip .error and .isr fields.
"add rsp, 0x10\n"
"iretq\n"
:
: "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64))
);
}
[[__naked]]
void interrupt_stub_internal() {
asm volatile(
// Load the kernel GS base if we're coming from user space.
"cmp word ptr [rsp + %c0], %c1\n"
"je 2f\n"
"swapgs\n"
"2:\n"
"push rax\n"
"push rbx\n"
"push rcx\n"
"push rdx\n"
"push rbp\n"
"push rdi\n"
"push rsi\n"
"push r8\n"
"push r9\n"
"push r10\n"
"push r11\n"
"push r12\n"
"push r13\n"
"push r14\n"
"push r15\n"
"cld\n"
// Zero out the base pointer since we can't trust it.
"xor rbp, rbp\n"
// Load the frame as first argument.
"mov rdi, rsp\n"
"call %c2\n"
"jmp %c3\n"
:
: "i"(CS_OFFSET), "i"(offsetof(struct gdt, kernel_code64)), "i"(interrupt_handler), "i"(interrupt_return)
);
}
// clang-format off
#define REPEAT_256(M) \
M(0) M(1) M(2) M(3) M(4) M(5) M(6) M(7) \
M(8) M(9) M(10) M(11) M(12) M(13) M(14) M(15) \
M(16) M(17) M(18) M(19) M(20) M(21) M(22) M(23) \
M(24) M(25) M(26) M(27) M(28) M(29) M(30) M(31) \
M(32) M(33) M(34) M(35) M(36) M(37) M(38) M(39) \
M(40) M(41) M(42) M(43) M(44) M(45) M(46) M(47) \
M(48) M(49) M(50) M(51) M(52) M(53) M(54) M(55) \
M(56) M(57) M(58) M(59) M(60) M(61) M(62) M(63) \
M(64) M(65) M(66) M(67) M(68) M(69) M(70) M(71) \
M(72) M(73) M(74) M(75) M(76) M(77) M(78) M(79) \
M(80) M(81) M(82) M(83) M(84) M(85) M(86) M(87) \
M(88) M(89) M(90) M(91) M(92) M(93) M(94) M(95) \
M(96) M(97) M(98) M(99) M(100) M(101) M(102) M(103) \
M(104) M(105) M(106) M(107) M(108) M(109) M(110) M(111) \
M(112) M(113) M(114) M(115) M(116) M(117) M(118) M(119) \
M(120) M(121) M(122) M(123) M(124) M(125) M(126) M(127) \
M(128) M(129) M(130) M(131) M(132) M(133) M(134) M(135) \
M(136) M(137) M(138) M(139) M(140) M(141) M(142) M(143) \
M(144) M(145) M(146) M(147) M(148) M(149) M(150) M(151) \
M(152) M(153) M(154) M(155) M(156) M(157) M(158) M(159) \
M(160) M(161) M(162) M(163) M(164) M(165) M(166) M(167) \
M(168) M(169) M(170) M(171) M(172) M(173) M(174) M(175) \
M(176) M(177) M(178) M(179) M(180) M(181) M(182) M(183) \
M(184) M(185) M(186) M(187) M(188) M(189) M(190) M(191) \
M(192) M(193) M(194) M(195) M(196) M(197) M(198) M(199) \
M(200) M(201) M(202) M(203) M(204) M(205) M(206) M(207) \
M(208) M(209) M(210) M(211) M(212) M(213) M(214) M(215) \
M(216) M(217) M(218) M(219) M(220) M(221) M(222) M(223) \
M(224) M(225) M(226) M(227) M(228) M(229) M(230) M(231) \
M(232) M(233) M(234) M(235) M(236) M(237) M(238) M(239) \
M(240) M(241) M(242) M(243) M(244) M(245) M(246) M(247) \
M(248) M(249) M(250) M(251) M(252) M(253) M(254) M(255)
// clang-format on
#define INTERRUPT_STUB(n) \
[[__naked]] \
static void interrupt_stub_##n() { \
asm volatile( \
".if (%c0 == 8 || (%c0 >= 10 && %c0 <= 14) || %c0 == 17 || %c0 == 21 || %c0 == 29 || %c0 == 30)\n" \
".else\n" \
"push 0\n" \
".endif\n" \
"push %c0\n" \
"jmp %c1\n" \
: \
: "i"(n), "i"(interrupt_stub_internal) \
); \
}
#define INTERRUPT_SETUP(n) \
do { \
uint64_t addr = (uint64_t)interrupt_stub_##n; \
idt[n].base0 = addr & 0xFFFF; \
idt[n].base1 = (addr >> 16) & 0xFFFF; \
idt[n].base2 = (addr >> 32) & 0xFFFF'FFFF; \
idt[n].ist = 0; \
idt[n].attrib = (1 << 7) | 0xE; \
idt[n].selector = offsetof(struct gdt, kernel_code64); \
idt[n].reserved = 0; \
} while (0);
REPEAT_256(INTERRUPT_STUB)
static struct idt_entry idt[256] = {};
void idt_init() {
REPEAT_256(INTERRUPT_SETUP)
idt[IDT_BP].attrib |= CPL_USER << 5;
}
void idt_load() {
struct idtr idtr = {
.limit = sizeof(idt) - 1,
.base = idt,
};
asm volatile("lidt %0" ::"m"(idtr));
}
arch/x86_64/include/bits/cpu.h
-5
View File
@@ -1,5 +0,0 @@
#pragma once
static inline void arch_cpu_pause() {
asm volatile("pause");
}
arch/x86_64/include/bits/elf.h
-34
View File
@@ -1,34 +0,0 @@
#pragma once
#include <zinnia/elf.h>
#define ELF_ARCH_CLASS ELFCLASS64
#define ELF_ARCH_DATA ELFDATA2LSB
#define ELF_ARCH_MACHINE EM_X86_64
#define ELF_R_SYM ELF64_R_SYM
#define ELF_R_TYPE ELF64_R_TYPE
#define ELF_ST_BIND ELF64_ST_BIND
#define ELF_ST_TYPE ELF64_ST_TYPE
typedef elf64_ehdr_t elf_ehdr_t;
typedef elf64_phdr_t elf_phdr_t;
typedef elf64_dyn_t elf_dyn_t;
typedef elf64_addr_t elf_addr_t;
typedef elf64_off_t elf_off_t;
typedef elf64_nhdr_t elf_nhdr_t;
typedef elf64_auxv_t elf_auxv_t;
typedef elf64_sym_t elf_sym_t;
typedef elf64_rela_t elf_rela_t;
typedef elf64_rel_t elf_rel_t;
typedef elf64_relr_t elf_relr_t;
enum {
R_NONE = 0,
R_64 = 1,
R_COPY = 5,
R_GLOB_DAT = 6,
R_JUMP_SLOT = 7,
R_RELATIVE = 8,
R_IRELATIVE = 37,
};
arch/x86_64/include/bits/pmap.h
-5
View File
@@ -1,5 +0,0 @@
#pragma once
#include <stdint.h>
typedef uint64_t pte_t;
arch/x86_64/include/x86_64/apic.h
-81
View File
@@ -1,81 +0,0 @@
#pragma once
#include <zinnia/mmio.h>
#include <stdint.h>
struct local_apic {
uint32_t id;
uint32_t ticks_per_10ms;
void* xapic_regs;
};
enum apic_delivery_mode {
APIC_DELIVERY_FIXED = 0b000,
APIC_DELIVERY_LOWESTPRIO = 0b001,
APIC_DELIVERY_SMI = 0b010,
APIC_DELIVERY_NMI = 0b100,
APIC_DELIVERY_INIT = 0b101,
APIC_DELIVERY_STARTUP = 0b110,
};
enum apic_dest_mode {
APIC_DEST_PHYSICAL = 0,
APIC_DEST_LOGICAL = 1,
};
enum apic_delivery_status {
APIC_DELIVERY_STATUS_IDLE = 0,
APIC_DELIVERY_STATUS_PENDING = 1,
};
enum apic_level {
APIC_LEVEL_DEASSERT = 0,
APIC_LEVEL_ASSERT = 1,
};
enum apic_trigger_mode {
APIC_TRIGGER_EDGE = 0,
APIC_TRIGGER_LEVEL = 1,
};
constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8};
constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8};
constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1};
constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300};
constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8};
constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3};
constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1};
constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1};
constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1};
constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1};
constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8};
constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32};
constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300};
constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8};
constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
void lapic_init(struct local_apic* lapic);
void lapic_eoi(struct local_apic* lapic);
void lapic_send_ipi(
struct local_apic* lapic,
uint32_t target,
uint8_t vector,
enum apic_delivery_mode delivery_mode,
enum apic_dest_mode dest_mode,
enum apic_level level,
enum apic_trigger_mode trigger_mode
);
void pic_disable();
arch/x86_64/include/x86_64/defs.h
-255
View File
@@ -1,255 +0,0 @@
#pragma once
#include <stdint.h>
enum : uint32_t {
CPUID_1C_SSE3 = 1u << 0,
CPUID_1C_PCLMUL = 1u << 1,
CPUID_1C_DTES64 = 1u << 2,
CPUID_1C_MONITOR = 1u << 3,
CPUID_1C_DS_CPL = 1u << 4,
CPUID_1C_VMX = 1u << 5,
CPUID_1C_SMX = 1u << 6,
CPUID_1C_EST = 1u << 7,
CPUID_1C_TM2 = 1u << 8,
CPUID_1C_SSSE3 = 1u << 9,
CPUID_1C_CID = 1u << 10,
CPUID_1C_SDBG = 1u << 11,
CPUID_1C_FMA = 1u << 12,
CPUID_1C_CX16 = 1u << 13,
CPUID_1C_XTPR = 1u << 14,
CPUID_1C_PDCM = 1u << 15,
CPUID_1C_PCID = 1u << 17,
CPUID_1C_DCA = 1u << 18,
CPUID_1C_SSE4_1 = 1u << 19,
CPUID_1C_SSE4_2 = 1u << 20,
CPUID_1C_X2APIC = 1u << 21,
CPUID_1C_MOVBE = 1u << 22,
CPUID_1C_POPCNT = 1u << 23,
CPUID_1C_TSC = 1u << 24,
CPUID_1C_AES = 1u << 25,
CPUID_1C_XSAVE = 1u << 26,
CPUID_1C_OSXSAVE = 1u << 27,
CPUID_1C_AVX = 1u << 28,
CPUID_1C_F16C = 1u << 29,
CPUID_1C_RDRAND = 1u << 30,
CPUID_1C_HYPERVISOR = 1u << 31,
};
enum : uint32_t {
CPUID_1D_FPU = 1u << 0,
CPUID_1D_VME = 1u << 1,
CPUID_1D_DE = 1u << 2,
CPUID_1D_PSE = 1u << 3,
CPUID_1D_TSC = 1u << 4,
CPUID_1D_MSR = 1u << 5,
CPUID_1D_PAE = 1u << 6,
CPUID_1D_MCE = 1u << 7,
CPUID_1D_CX8 = 1u << 8,
CPUID_1D_APIC = 1u << 9,
CPUID_1D_SEP = 1u << 11,
CPUID_1D_MTRR = 1u << 12,
CPUID_1D_PGE = 1u << 13,
CPUID_1D_MCA = 1u << 14,
CPUID_1D_CMOV = 1u << 15,
CPUID_1D_PAT = 1u << 16,
CPUID_1D_PSE36 = 1u << 17,
CPUID_1D_PSN = 1u << 18,
CPUID_1D_CLFLUSH = 1u << 19,
CPUID_1D_DS = 1u << 21,
CPUID_1D_ACPI = 1u << 22,
CPUID_1D_MMX = 1u << 23,
CPUID_1D_FXSR = 1u << 24,
CPUID_1D_SSE = 1u << 25,
CPUID_1D_SSE2 = 1u << 26,
CPUID_1D_SS = 1u << 27,
CPUID_1D_HTT = 1u << 28,
CPUID_1D_TM = 1u << 29,
CPUID_1D_IA64 = 1u << 30,
CPUID_1D_PBE = 1u << 31,
};
enum : uint32_t {
CPUID_7B_FSGSBASE = 1u << 0,
CPUID_7B_SGX = 1u << 2,
CPUID_7B_BMI = 1u << 3,
CPUID_7B_HLE = 1u << 4,
CPUID_7B_AVX2 = 1u << 5,
CPUID_7B_SMEP = 1u << 7,
CPUID_7B_BMI2 = 1u << 8,
CPUID_7B_RTM = 1u << 11,
CPUID_7B_AVX512F = 1u << 16,
CPUID_7B_AVX512DQ = 1u << 17,
CPUID_7B_RDSEED = 1u << 18,
CPUID_7B_ADX = 1u << 19,
CPUID_7B_SMAP = 1u << 20,
CPUID_7B_AVX512IFMA = 1u << 21,
CPUID_7B_CLFLUSHOPT = 1u << 23,
CPUID_7B_CLWB = 1u << 24,
CPUID_7B_AVX512CD = 1u << 28,
CPUID_7B_SHA = 1u << 29,
CPUID_7B_AVX512BW = 1u << 30,
CPUID_7B_AVX512VL = 1u << 31,
};
enum : uint32_t {
CPUID_7C_AVX512VBMI = 1u << 1,
CPUID_7C_UMIP = 1u << 2,
CPUID_7C_PKU = 1u << 3,
CPUID_7C_OSPKE = 1u << 4,
CPUID_7C_WAITPKG = 1u << 5,
CPUID_7C_AVX512VBMI2 = 1u << 6,
CPUID_7C_SHSTK = 1u << 7,
CPUID_7C_GFNI = 1u << 8,
CPUID_7C_VAES = 1u << 9,
CPUID_7C_VPCLMULQDQ = 1u << 10,
CPUID_7C_AVX512VNNI = 1u << 11,
CPUID_7C_AVX512BITALG = 1u << 12,
CPUID_7C_TME_EN = 1u << 13,
CPUID_7C_AVX512VPOPCNTDQ = 1u << 14,
CPUID_7C_RDPID = 1u << 22,
CPUID_7C_KL = 1u << 23,
CPUID_7C_CLDEMOTE = 1u << 25,
CPUID_7C_MOVDIRI = 1u << 27,
CPUID_7C_MOVDIR64B = 1u << 28,
CPUID_7C_ENQCMD = 1u << 29,
};
enum : uint32_t {
CPUID_7D_UINTR = 1u << 5,
CPUID_7D_AVX512VP2INTERSECT = 1u << 8,
CPUID_7D_SERIALIZE = 1u << 14,
CPUID_7D_TSXLDTRK = 1u << 16,
CPUID_7D_PCONFIG = 1u << 18,
CPUID_7D_IBT = 1u << 20,
CPUID_7D_AMX_BF16 = 1u << 22,
CPUID_7D_AVX512FP16 = 1u << 23,
CPUID_7D_AMX_TILE = 1u << 24,
CPUID_7D_AMX_INT8 = 1u << 25,
};
enum : uint64_t {
RFLAGS_CF = 1ull << 0,
RFLAGS_BRKI = 1ull << 1,
RFLAGS_PF = 1ull << 2,
RFLAGS_AF = 1ull << 4,
RFLAGS_ZF = 1ull << 6,
RFLAGS_SF = 1ull << 7,
RFLAGS_TF = 1ull << 8,
RFLAGS_IF = 1ull << 9,
RFLAGS_DF = 1ull << 10,
RFLAGS_OF = 1ull << 11,
RFLAGS_IOPL = 3ull << 12,
RFLAGS_NT = 1ull << 14,
RFLAGS_RF = 1ull << 16,
RFLAGS_VM = 1ull << 17,
RFLAGS_AC = 1ull << 18,
RFLAGS_VIF = 1ull << 19,
RFLAGS_VIP = 1ull << 20,
RFLAGS_ID = 1ull << 21,
};
enum : uint64_t {
XCR0_X87 = 1ull << 0,
XCR0_SSE = 1ull << 1,
XCR0_AVX = 1ull << 2,
XCR0_AXV512_OPMASK = 1ull << 5,
XCR0_AVX512_ZMM_HI256 = 1ull << 6,
XCR0_AVX512_HI16_ZMM = 1ull << 7,
};
enum : uint64_t {
CR0_PE = 1ull << 0,
CR0_MP = 1ull << 1,
CR0_EM = 1ull << 2,
CR0_TS = 1ull << 3,
CR0_ET = 1ull << 4,
CR0_NE = 1ull << 5,
CR0_WP = 1ull << 16,
CR0_AM = 1ull << 18,
CR0_NW = 1ull << 29,
CR0_CD = 1ull << 30,
CR0_PG = 1ull << 31,
};
enum : uint64_t {
CR4_VME = 1ull << 0,
CR4_PVI = 1ull << 1,
CR4_TSD = 1ull << 2,
CR4_DE = 1ull << 3,
CR4_PSE = 1ull << 4,
CR4_PAE = 1ull << 5,
CR4_MCE = 1ull << 6,
CR4_PGE = 1ull << 7,
CR4_PCE = 1ull << 8,
CR4_OSFXSR = 1ull << 9,
CR4_OSXMMEXCPT = 1ull << 10,
CR4_UMIP = 1ull << 11,
CR4_VMXE = 1ull << 13,
CR4_SMXE = 1ull << 14,
CR4_FSGSBASE = 1ull << 16,
CR4_PCIDE = 1ull << 17,
CR4_OSXSAVE = 1ull << 18,
CR4_SMEP = 1ull << 20,
CR4_SMAP = 1ull << 21,
CR4_PKE = 1ull << 22,
CR4_CET = 1ull << 23,
CR4_PKS = 1ull << 24,
};
enum : uint64_t {
MSR_EFER_SCE = 1ull << 0,
MSR_EFER_LME = 1ull << 8,
MSR_EFER_LMA = 1ull << 10,
MSR_EFER_NXE = 1ull << 11,
MSR_EFER_SVME = 1ull << 12,
MSR_EFER_LMSLE = 1ull << 13,
MSR_EFER_FFXSR = 1ull << 14,
MSR_EFER_TCE = 1ull << 15,
};
enum : uint32_t {
MSR_PAT = 0x277,
MSR_EFER = 0xC0000080,
MSR_STAR = 0xC0000081,
MSR_LSTAR = 0xC0000082,
MSR_CSTAR = 0xC0000083,
MSR_SFMASK = 0xC0000084,
MSR_FS_BASE = 0xC0000100,
MSR_GS_BASE = 0xC0000101,
MSR_KERNEL_GS_BASE = 0xC0000102,
};
enum : uint8_t {
PAT_UC = 0x0,
PAT_WC = 0x1,
PAT_WT = 0x4,
PAT_WP = 0x5,
PAT_WB = 0x6,
PAT_UCM = 0x7,
};
enum : uint8_t {
CPL_USER = 0x3,
CPL_KERNEL = 0x0,
};
enum : uint8_t {
IDT_DE = 0x0,
IDT_DB = 0x1,
IDT_BP = 0x3,
IDT_OF = 0x4,
IDT_BR = 0x5,
IDT_UD = 0x6,
IDT_NM = 0x7,
IDT_SS = 0xC,
IDT_GP = 0xD,
IDT_PF = 0xE,
IDT_MF = 0x10,
IDT_AC = 0x11,
IDT_XM = 0x13,
IDT_IPI_SHOOTDOWN = 0xFC,
IDT_TIMER = 0xFD,
IDT_SPURIOUS = 0xFE,
IDT_IPI_RESCHED = 0xFF,
};
arch/x86_64/include/x86_64/gdt.h
-56
View File
@@ -1,56 +0,0 @@
#pragma once
#include <zinnia/compiler.h>
#include <stdint.h>
union gdt_desc {
uint64_t val;
struct {
uint16_t limit0;
uint16_t base0;
uint8_t base1;
uint8_t access;
uint8_t limit1_flags;
uint8_t base2;
};
};
static_assert(sizeof(union gdt_desc) == 0x8);
union gdt_long_desc {
uint64_t val[2];
struct {
uint16_t limit0;
uint16_t base0;
uint8_t base1;
uint8_t access;
uint8_t limit1_flags;
uint8_t base2;
uint32_t base3;
uint32_t reserved;
};
};
static_assert(sizeof(union gdt_long_desc) == 0x10);
struct [[__packed]] gdt {
union gdt_desc null;
union gdt_desc kernel_code32;
union gdt_desc kernel_data32;
union gdt_desc kernel_code64;
union gdt_desc kernel_data64;
union gdt_desc user_code32;
union gdt_desc user_data;
union gdt_desc user_code64;
union gdt_long_desc tss;
};
struct [[__packed]] gdtr {
uint16_t limit;
struct gdt* base;
};
extern const struct gdt gdt_base;
// Initializes a GDT on the local CPU.
void gdt_load();
arch/x86_64/include/x86_64/idt.h
-11
View File
@@ -1,11 +0,0 @@
#pragma once
#include <zinnia/init.h>
[[__init]]
void idt_init();
// Loads the IDT on this CPU.
void idt_load();
void interrupt_return();
arch/x86_64/include/x86_64/percpu.h
-24
View File
@@ -1,24 +0,0 @@
#pragma once
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/gdt.h>
#include <x86_64/tss.h>
struct arch_percpu {
struct gdt gdt;
struct tss tss;
struct local_apic lapic;
struct irq_line* irq_lines[128];
size_t fpu_size;
void (*fpu_save)(void*);
void (*fpu_restore)(const void*);
bool can_smap;
};
extern struct arch_percpu arch_data;
// Initializes the calling CPU and makes `cpu` its per-CPU block.
void x86_64_cpu_setup(struct percpu* cpu);
arch/x86_64/include/x86_64/pmap.h
-19
View File
@@ -1,19 +0,0 @@
#pragma once
#include <zinnia/pmap.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
arch/x86_64/include/x86_64/smp.h
-24
View File
@@ -1,24 +0,0 @@
#pragma once
#include <zinnia/atomic.h>
#include <zinnia/compiler.h>
#include <zinnia/utils.h>
#include <stdint.h>
#include <x86_64/gdt.h>
struct [[__packed]] smp_info {
struct gdt gdt;
struct gdtr gdtr;
uint32_t farjmp_offset;
uint32_t farjmp_segment;
uint32_t temp_stack;
uint32_t temp_cr3;
uint64_t entry;
uint64_t hhdm_offset;
uint64_t percpu;
uint32_t lapic_id;
atomic_bool booted;
};
// Discovers all APs and starts them.
void x86_64_smp_init();
arch/x86_64/include/x86_64/syscall.h
-3
View File
@@ -1,3 +0,0 @@
#pragma once
void arch_syscall_stub();
arch/x86_64/include/x86_64/tsc.h
-3
View File
@@ -1,3 +0,0 @@
#pragma once
void tsc_init();
arch/x86_64/irq.c
-93
View File
@@ -1,93 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/percpu.h>
#include <zinnia/utils.h>
#include <arch/irq.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/percpu.h>
bool arch_irq_get_state() {
uint64_t flags;
asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
return flags & RFLAGS_IF;
}
bool arch_irq_set_state(bool state) {
bool old = arch_irq_get_state();
if (state)
asm volatile("sti" ::: "memory");
else
asm volatile("cli" ::: "memory");
return old;
}
void arch_irq_wait() {
asm volatile("hlt");
}
struct apic_msi_line {
struct msi_line msi;
uint8_t vector;
uint32_t lapic;
};
static phys_t apic_msi_msg_addr(struct msi_line* self) {
struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi);
return 0xFEE0'0000 | ((phys_t)line->lapic << 12);
}
static uint32_t apic_msi_msg_data(struct msi_line* self) {
return containerof(self, struct apic_msi_line, msi)->vector;
}
static enum irq_mode apic_msi_set_config(
struct irq_line* self,
enum irq_trigger_mode trigger,
enum irq_polarity polarity
) {
assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered");
return IRQ_MODE_EOI;
}
static void apic_msi_mask(struct irq_line* self) {}
static void apic_msi_unmask(struct irq_line* self) {}
static void apic_msi_eoi(struct irq_line* self) {
lapic_eoi(&percpu_ref(arch_data)->lapic);
}
errno_t arch_irq_alloc_msi(struct msi_line** out) {
struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
if (!line)
return ENOMEM;
irq_lock();
struct arch_percpu* arch_cpu = percpu_ref(arch_data);
size_t slot;
for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) {
if (!arch_cpu->irq_lines[slot])
break;
}
if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
irq_unlock();
mem_free(line);
return ENOSPC;
}
line->msi.line.set_config = apic_msi_set_config;
line->msi.line.mask = apic_msi_mask;
line->msi.line.unmask = apic_msi_unmask;
line->msi.line.eoi = apic_msi_eoi;
line->msi.msg_addr = apic_msi_msg_addr;
line->msi.msg_data = apic_msi_msg_data;
line->vector = slot + 0x20;
line->lapic = arch_cpu->lapic.id;
arch_cpu->irq_lines[slot] = &line->msi.line;
irq_unlock();
*out = &line->msi;
return 0;
}
arch/x86_64/mem.c
-36
View File
@@ -1,36 +0,0 @@
#include <zinnia/types.h>
#include <arch/mem.h>
#include <stddef.h>
#include <stdint.h>
size_t arch_mem_page_size() {
return 1 << 12;
}
size_t arch_mem_page_bits() {
return 12;
}
size_t arch_mem_level_bits() {
return 9;
}
size_t arch_mem_num_levels() {
return 4;
}
uintptr_t arch_mem_hhdm_addr() {
return 0xFFFF'8000'0000'0000;
}
uintptr_t arch_mem_pfndb_addr() {
return 0xFFFF'A000'0000'0000;
}
uintptr_t arch_mem_mapping_addr() {
return 0xFFFF'C000'0000'0000;
}
bool arch_mem_addr_is_user(uintptr_t addr) {
return addr < 0x0000'7FFF'FFFF'F000;
}
arch/x86_64/meson.build
-30
View File
@@ -1,30 +0,0 @@
kernel_c_args += [
'-mgeneral-regs-only',
'-mno-red-zone',
'-masm=intel',
]
kernel_sources += files(
'apic.c',
'archctl.c',
'gdt.c',
'hpet.c',
'idt.c',
'irq.c',
'mem.c',
'percpu.c',
'pmap.c',
'pmap.c',
'random.c',
'sched.c',
'serial.c',
'smp.c',
'start.c',
'syscall.c',
'tsc.c',
'usercopy.c',
)
if kconfig.get('CONFIG_PCI', 'n') == 'y'
kernel_sources += files('pci.c')
endif
arch/x86_64/pci.c
-50
View File
@@ -1,50 +0,0 @@
#include <zinnia/init.h>
#include <zinnia/pci.h>
#include <zinnia/spin.h>
#include <x86_64/asm.h>
constexpr uint16_t PCI_CONFIG_ADDRESS = 0xcf8;
constexpr uint16_t PCI_CONFIG_DATA = 0xcfc;
static struct spinlock port_lock = {};
static void port_select(const struct pci_address* addr, uint16_t offset) {
asm_outl(
PCI_CONFIG_ADDRESS,
1u << 31 | (uint32_t)addr->bus << 16 | (uint32_t)(addr->device & 0x1f) << 11 |
(uint32_t)(addr->function & 0x7) << 8 | (offset & 0xfc)
);
}
static uint32_t port_read32(struct pci_access*, const struct pci_address* addr, uint16_t offset) {
spin_lock(&port_lock);
port_select(addr, offset);
uint32_t value = asm_inl(PCI_CONFIG_DATA);
spin_unlock(&port_lock);
return value;
}
static void port_write32(struct pci_access*, const struct pci_address* addr, uint16_t offset, uint32_t value) {
spin_lock(&port_lock);
port_select(addr, offset);
asm_outl(PCI_CONFIG_DATA, value);
spin_unlock(&port_lock);
}
static struct pci_access port_access = {
.name = "port",
.domain = 0,
.start_bus = 0,
.end_bus = 255,
.priority = PCI_ACCESS_PRIO_LEGACY,
.read32 = port_read32,
.write32 = port_write32,
};
[[__init]]
static void pci_x86_stage_init(struct boot_info*) {
pci_access_register(&port_access);
}
INIT_TASK(pci_x86_stage, "arch.x86_64.pci", pci_x86_stage_init);
INIT_DEPENDS(pci_x86_stage, mem_stage);
INIT_ENTAILS(pci_x86_stage, pci_stage);
arch/x86_64/percpu.c
-145
View File
@@ -1,145 +0,0 @@
#include <zinnia/initgraph.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <config.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/idt.h>
#include <x86_64/percpu.h>
#include <x86_64/smp.h>
#include <x86_64/syscall.h>
#include <x86_64/tsc.h>
struct percpu* arch_percpu_get() {
struct percpu* result;
asm volatile("mov %0, gs:0" : "=r"(result)::"memory");
return result;
}
void arch_percpu_bsp_init() {
asm_wrmsr(MSR_GS_BASE, (uint64_t)&percpu_bsp);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
pic_disable();
gdt_load();
idt_init();
idt_load();
}
union ia32_star {
uint64_t value;
struct {
uint32_t rsvd;
uint16_t kernel_cs_ss;
uint16_t user_cs_ss;
};
};
static_assert(sizeof(union ia32_star) == 8);
[[__percpu]]
struct arch_percpu arch_data = {0};
void x86_64_cpu_setup(struct percpu* cpu) {
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
asm_wrmsr(MSR_FS_BASE, 0);
struct arch_percpu* arch_cpu = percpu_ref(arch_data);
gdt_load();
// TODO: not if FRED
idt_load();
asm_wrmsr(
MSR_PAT,
PAT_WB | PAT_WT << 8 | PAT_UCM << 16 | PAT_UC << 24 | (uint64_t)PAT_WP << 32 | (uint64_t)PAT_WC << 40 |
(uint64_t)PAT_UCM << 48 | (uint64_t)PAT_UC << 56
);
// Syscall extension.
asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
union ia32_star star = {0};
star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
star.user_cs_ss = offsetof(struct gdt, user_code32) | CPL_USER;
asm_wrmsr(MSR_STAR, star.value);
asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
// Mask all RFLAGS except for BRKI which is reserved.
asm_wrmsr(MSR_SFMASK, ~RFLAGS_BRKI);
// TODO: FRED
uint64_t cr0, cr4;
asm volatile("mov %0, cr0" : "=r"(cr0));
asm volatile("mov %0, cr4" : "=r"(cr4));
struct cpuid cpuid1 = asm_cpuid(1, 0);
struct cpuid cpuid7 = asm_cpuid(7, 0);
struct cpuid cpuid13 = asm_cpuid(13, 0);
// Enable SSE.
cr0 &= ~CR0_EM;
cr0 |= CR0_MP | CR0_WP;
cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
if (cpuid1.ecx & CPUID_1C_XSAVE) {
cr4 |= CR4_OSXSAVE;
asm volatile("mov cr4, %0" ::"r"(cr4));
uint64_t xcr0 = XCR0_X87 | XCR0_SSE;
// AVX
if (cpuid1.ecx & CPUID_1C_AVX)
xcr0 |= XCR0_AVX;
// AVX-512
if (cpuid7.ebx & CPUID_7B_AVX512F)
xcr0 |= XCR0_AXV512_OPMASK | XCR0_AVX512_ZMM_HI256 | XCR0_AVX512_HI16_ZMM;
asm_wrxcr(0, xcr0);
arch_cpu->fpu_size = cpuid13.ecx;
arch_cpu->fpu_save = asm_xsave;
arch_cpu->fpu_restore = asm_xrstor;
} else {
arch_cpu->fpu_size = 512;
arch_cpu->fpu_save = asm_fxsave;
arch_cpu->fpu_restore = asm_fxrstor;
}
if (cpuid7.ecx & CPUID_7C_UMIP)
cr4 |= CR4_UMIP;
if (cpuid7.ebx & CPUID_7B_SMEP)
cr4 |= CR4_SMEP;
if (cpuid7.ebx & CPUID_7B_SMAP) {
cr4 |= CR4_SMAP;
arch_cpu->can_smap = true;
}
if (cpuid7.ebx & CPUID_7B_FSGSBASE)
cr4 |= CR4_FSGSBASE;
asm volatile("mov cr0, %0" ::"r"(cr0));
asm volatile("mov cr4, %0" ::"r"(cr4));
lapic_init(&arch_cpu->lapic);
cpu->online = true;
}
void arch_percpu_init() {
#if CONFIG_TSC
tsc_init();
#endif
// Init the BSP.
x86_64_cpu_setup(&percpu_bsp);
// Init any APs.
x86_64_smp_init();
}
arch/x86_64/pmap.c
-84
View File
@@ -1,84 +0,0 @@
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/types.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/percpu.h>
#include <x86_64/pmap.h>
void pte_clear(pte_t* pte) {
*pte = 0;
}
void pte_flush(uintptr_t vaddr) {
asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
}
void pte_flush_all() {
uint64_t cr3;
asm volatile("mov %0, cr3" : "=r"(cr3));
asm volatile("mov cr3, %0" ::"r"(cr3) : "memory");
}
void pte_shootdown_kick(struct percpu* cpu) {
irq_lock();
lapic_send_ipi(
&percpu_ref(arch_data)->lapic,
percpu_ref_on(cpu, arch_data)->lapic.id,
IDT_IPI_SHOOTDOWN,
APIC_DELIVERY_FIXED,
APIC_DEST_PHYSICAL,
APIC_LEVEL_ASSERT,
APIC_TRIGGER_EDGE
);
irq_unlock();
}
pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
if (flags & PTE_USER)
result |= ARCH_FLAG_USER_MODE;
if (flags & PTE_DIR)
result |= ARCH_FLAG_READ_WRITE;
else {
if (flags & PTE_WRITE)
result |= ARCH_FLAG_READ_WRITE;
if (!(flags & PTE_EXEC))
result |= ARCH_FLAG_EXECUTE_DISABLE;
switch (cache) {
case CACHE_ANY:
case CACHE_WRITE_BACK:
break;
case CACHE_WRITE_THROUGH:
result |= ARCH_FLAG_WRITE_THROUGH;
break;
case CACHE_MMIO:
result |= ARCH_FLAG_CACHE_DISABLE | ARCH_FLAG_WRITE_THROUGH;
break;
case CACHE_WRITE_COMBINE:
result |= ARCH_FLAG_ATTRIBUTE_TABLE | ARCH_FLAG_WRITE_THROUGH;
break;
}
}
return result;
}
bool pte_is_present(pte_t* pte) {
return *pte & ARCH_FLAG_PRESENT;
}
bool pte_is_dir(pte_t* pte) {
return *pte & ARCH_FLAG_SIZE;
}
phys_t pte_address(pte_t* pte) {
return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
}
void pmap_set(struct pmap* pt) {
asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
}
arch/x86_64/random.c
-36
View File
@@ -1,36 +0,0 @@
#include <zinnia/initgraph.h>
#include <zinnia/print.h>
#include <zinnia/random.h>
#include <x86_64/asm.h>
#define CPUID_1C_RDRAND (1u << 30)
#define RDRAND_RETRIES 10
static bool rdrand_get_u64(struct random_source*, uint64_t* out) {
for (int i = 0; i < RDRAND_RETRIES; i++) {
uint64_t value;
uint8_t ok;
asm volatile("rdrand %0\n setc %1" : "=r"(value), "=qm"(ok) : : "cc");
if (ok) {
*out = value;
return true;
}
}
return false;
}
static struct random_source rdrand_source = {
.name = "x86_64.rdrand",
.quality = 100,
.get_u64 = rdrand_get_u64,
};
static void rdrand_init(struct boot_info*) {
if (asm_cpuid(1, 0).ecx & CPUID_1C_RDRAND) {
random_register(&rdrand_source);
} else {
log_warn("random: This CPU doesn't support RDRAND, ignoring\n");
}
}
INIT_TASK(rdrand_stage, "arch.x86_64.rdrand", rdrand_init);
INIT_ENTAILS(rdrand_stage, random_stage);
arch/x86_64/sched.c
-261
View File
@@ -1,261 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/sched.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <bits/sched.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/idt.h>
#include <x86_64/percpu.h>
// The task frame consists of registers that the C ABI marks as callee-saved.
// If we don't save them, these registers are lost during a context switch.
struct task_frame {
uint64_t rbx, rbp, r12, r13, r14, r15, rip;
};
[[__naked]]
static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) {
asm volatile(
"sub rsp, 0x30\n" // Make room for all regs (except RIP).
"mov [rsp + %c[rbx]], rbx\n"
"mov [rsp + %c[rbp]], rbp\n"
"mov [rsp + %c[r12]], r12\n"
"mov [rsp + %c[r13]], r13\n"
"mov [rsp + %c[r14]], r14\n"
"mov [rsp + %c[r15]], r15\n"
"mov [rdi], rsp\n" // rdi = old_rsp
"mov rsp, rsi\n" // rsi = new_rsp
"mov rbx, [rsp + %c[rbx]]\n"
"mov rbp, [rsp + %c[rbp]]\n"
"mov r12, [rsp + %c[r12]]\n"
"mov r13, [rsp + %c[r13]]\n"
"mov r14, [rsp + %c[r14]]\n"
"mov r15, [rsp + %c[r15]]\n"
"add rsp, 0x30\n"
"mov rax, rdx\n"
"ret" // This will conveniently move us to the RIP we put at this stack entry.
:
: [rbx] "i"(offsetof(struct task_frame, rbx)),
[rbp] "i"(offsetof(struct task_frame, rbp)),
[r12] "i"(offsetof(struct task_frame, r12)),
[r13] "i"(offsetof(struct task_frame, r13)),
[r14] "i"(offsetof(struct task_frame, r14)),
[r15] "i"(offsetof(struct task_frame, r15))
: "memory"
);
}
struct task* arch_sched_switch(struct task* from, struct task* to) {
struct arch_percpu* cpu = percpu_ref(arch_data);
struct arch_task_context* from_ctx = &from->context;
struct arch_task_context* to_ctx = &to->context;
cpu->tss.rsp0 = kernel_stack_top(&to->kernel_stack);
if (from_ctx->fpu_region != nullptr) {
cpu->fpu_save(from_ctx->fpu_region);
from_ctx->ds = asm_read_ds();
from_ctx->es = asm_read_es();
from_ctx->fs = asm_read_fs();
from_ctx->gs = asm_read_gs();
from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE);
from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
if (to_ctx->fpu_region != nullptr) {
cpu->fpu_restore(to_ctx->fpu_region);
asm_write_ds(to_ctx->ds);
asm_write_es(to_ctx->es);
asm_write_fs(to_ctx->fs);
// If we have to change the GS segment we need to reload the MSR, otherwise we lose its value.
if (to_ctx->gs != asm_read_gs()) {
uint64_t percpu = asm_rdmsr(MSR_GS_BASE);
asm_write_gs(to_ctx->gs);
asm_wrmsr(MSR_GS_BASE, percpu);
}
asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base);
asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base);
}
return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
}
// Calls task_entry with the correct parameters.
[[__naked]]
void task_entry_thunk() {
asm volatile(
"mov rdi, rbx\n"
"mov rsi, r12\n"
"mov rdx, rax\n"
"jmp task_entry"
:
:
: "memory", "rdi", "rsi", "rdx"
);
}
errno_t arch_task_init(
struct arch_task_context* context,
void* entry,
uintptr_t arg0,
uintptr_t stack_start,
bool is_user
) {
struct arch_percpu* cpu = percpu_ref(arch_data);
assert((stack_start & 0xF) == 0, "Task stack is not 16-byte aligned!");
// Prepare a dummy stack with an entry point function to return to.
struct task_frame* frame = (struct task_frame*)(stack_start - sizeof(uint64_t)) - 1;
frame->rbx = (uint64_t)entry;
frame->r12 = arg0;
frame->rip = (uint64_t)task_entry_thunk;
context->rsp = (uint64_t)frame;
if (is_user) {
// Allocate FPU space.
const size_t fpu_size = cpu->fpu_size;
uintptr_t fpu_base = vm_alloc(fpu_size);
if (!fpu_base)
return ENOMEM;
phys_t pages;
errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size())
pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
context->fpu_region = (void*)fpu_base;
context->fpu_phys = pages;
context->ds = asm_read_ds();
context->es = asm_read_es();
context->fs = asm_read_fs();
context->gs = asm_read_gs();
context->fs_base = asm_rdmsr(MSR_FS_BASE);
context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
return 0;
}
void arch_task_fini(struct arch_task_context* context) {
if (context->fpu_region == nullptr)
return;
const size_t page_size = arch_mem_page_size();
const size_t fpu_size = percpu_ref(arch_data)->fpu_size;
const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
for (size_t i = 0; i < fpu_size; i += page_size)
pmap_unmap(&kernel_vma.pmap, fpu_base + i);
pmap_shootdown(&kernel_vma.pmap);
mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
vm_free(fpu_base, fpu_size);
context->fpu_region = nullptr;
}
void arch_task_fork(struct arch_task_context* dst) {
if (dst->fpu_region)
percpu_ref(arch_data)->fpu_save(dst->fpu_region);
}
void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) {
*ctx = (struct arch_context){
.rip = ip,
.rsp = sp,
.rflags = 0x202,
.cs = offsetof(struct gdt, user_code64) | CPL_USER,
.ss = offsetof(struct gdt, user_data) | CPL_USER,
};
}
[[__naked]]
void arch_sched_preempt_disable() {
asm volatile(
"inc qword ptr gs:%c0\n"
"ret"
:
: "i"(offsetof(struct percpu, sched.preempt_level))
);
}
[[__naked]]
bool arch_sched_preempt_enable() {
asm volatile(
"dec qword ptr gs:%c0\n"
"setz al\n"
"movzx eax, al\n"
"ret\n"
:
: "i"(offsetof(struct percpu, sched.preempt_level))
: "cc"
);
}
void arch_sched_kick(struct percpu* cpu) {
if (!cpu->online)
return;
// The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler.
irq_lock();
lapic_send_ipi(
&percpu_ref(arch_data)->lapic,
percpu_ref_on(cpu, arch_data)->lapic.id,
IDT_IPI_RESCHED,
APIC_DELIVERY_FIXED,
APIC_DEST_PHYSICAL,
APIC_LEVEL_ASSERT,
APIC_TRIGGER_EDGE
);
irq_unlock();
}
[[__naked]]
void arch_sched_jump_to_context(struct arch_context* context) {
asm volatile(
"mov rsp, rdi\n"
"jmp %0\n" ::"r"(interrupt_return)
);
}
void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
struct arch_context context = {
.rip = ip,
.rsp = sp,
.rdi = arg,
.rflags = 0x202,
.cs = offsetof(struct gdt, user_code64) | CPL_USER,
.ss = offsetof(struct gdt, user_data) | CPL_USER,
};
uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
asm volatile(
"mov ds, %0\n"
"mov es, %0\n"
"mov fs, %0\n"
"mov gs, %0\n"
:
: "a"(0)
: "memory"
);
asm_wrmsr(MSR_FS_BASE, 0);
asm_wrmsr(MSR_GS_BASE, percpu);
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
arch_sched_jump_to_context(&context);
}
arch/x86_64/smp.c
-359
View File
@@ -1,359 +0,0 @@
#define pr_fmt(fmt) "smp: " fmt
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/atomic.h>
#include <zinnia/clock.h>
#include <zinnia/compiler.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/types.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <uacpi/acpi.h>
#include <uacpi/tables.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/percpu.h>
#include <x86_64/pmap.h>
#include <x86_64/smp.h>
extern uint8_t smp_trampoline_start[];
extern uint8_t smp_trampoline_data[];
extern uint8_t smp_trampoline_end[];
[[__naked, __used]]
static void smp_trampoline() {
asm volatile(
".pushsection .rodata\n"
".code16\n"
".global smp_trampoline_start\n"
"smp_trampoline_start:\n"
" cli\n"
" cld\n"
" jmp .Lmode16\n"
".org smp_trampoline_start + %c[data_off]\n"
".global smp_trampoline_data\n"
"smp_trampoline_data:\n"
" .skip %c[info_size]\n"
".set .Ldata_offset, %c[data_off]\n"
".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
".set .Linvalid_idtr_offset, (.Linvalid_idtr - smp_trampoline_start)\n"
".set .Lmode32_offset, (.Lmode32 - smp_trampoline_start)\n"
".set .Lmode64_offset, (.Lmode64 - smp_trampoline_start)\n"
".Lmode16:\n"
" mov bx, cs\n"
" shl ebx, 4\n"
" lidtd cs:.Linvalid_idtr_offset\n"
" lgdtd cs:.Lgdtr_offset\n"
" lea eax, [ebx + .Lmode32_offset]\n"
" mov dword ptr cs:.Lfarjmp_offset, eax\n"
" mov eax, %c[cr0]\n"
" mov cr0, eax\n"
" jmp fword ptr cs:.Lfarjmp_offset\n"
".code32\n"
".Lmode32:\n"
" mov ax, %c[kernel32_ds]\n"
" mov ds, ax\n"
" mov es, ax\n"
" mov fs, ax\n"
" mov gs, ax\n"
" mov ss, ax\n"
" xor eax, eax\n"
" lldt ax\n"
" mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n"
" xor eax, eax\n"
" mov cr4, eax\n"
" or eax, %c[cr4_pae]\n"
" mov cr4, eax\n"
" mov ecx, %c[msr_efer]\n"
" mov eax, %c[efer_lme]\n"
" xor edx, edx\n"
" wrmsr\n"
" mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n"
" mov cr3, eax\n"
" mov eax, cr0\n"
" or eax, %c[cr0_pg]\n"
" mov cr0, eax\n"
" lea eax, [ebx + .Lmode64_offset]\n"
" push %c[kernel64_cs]\n"
" push eax\n"
" retf\n"
".code64\n"
".Lmode64:\n"
" mov ax, %c[kernel64_ds]\n"
" mov ds, ax\n"
" mov es, ax\n"
" mov fs, ax\n"
" mov gs, ax\n"
" mov ss, ax\n"
" mov ecx, %c[msr_efer]\n"
" rdmsr\n"
" or eax, %c[efer_nxe]\n"
" wrmsr\n"
" xor ebp, ebp\n"
" lea rdi, [rbx + .Ldata_offset]\n"
" mov rax, [rbx + .Lentry_offset]\n"
" add rsp, [rbx + .Lhhdm_offset]\n"
" jmp rax\n"
".Linvalid_idtr:\n"
" .word 0\n"
" .quad 0\n"
".global smp_trampoline_end\n"
"smp_trampoline_end:\n"
".popsection\n"
" ud2\n"
:
: [data_off] "i"(8),
[info_size] "i"(sizeof(struct smp_info)),
[gdtr] "i"(offsetof(struct smp_info, gdtr)),
[farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
[temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
[temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)),
[entry] "i"(offsetof(struct smp_info, entry)),
[hhdm] "i"(offsetof(struct smp_info, hhdm_offset)),
[kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)),
[kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
[kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
[cr0] "i"(CR0_PE | CR0_ET),
[cr0_pg] "i"(CR0_PG),
[cr4_pae] "i"(CR4_PAE),
[msr_efer] "i"(MSR_EFER),
[efer_lme] "i"(MSR_EFER_LME),
[efer_nxe] "i"(MSR_EFER_NXE)
);
}
constexpr size_t AP_BOOT_STACK_SIZE = 0x8000;
constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000;
[[noreturn]]
static void ap_entry(phys_t info_addr) {
// We're still running on the temporary page table.
pmap_set(&kernel_vma.pmap);
struct smp_info* info = HHDM_PTR(info_addr);
struct percpu* cpu = (struct percpu*)info->percpu;
x86_64_cpu_setup(cpu);
irq_lock();
// This task keeps running on the boot stack, but is never scheduled again after the first yield.
sched_init(&cpu->sched);
// Let the BSP know that we're alive.
// `info` must not be touched after this point.
atomic_store(&info->booted, 1, ATOMIC_RELEASE);
log("CPU %zu online\n", cpu->id);
sched_yield(&cpu->sched);
unreachable();
}
static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
phys_t stack;
if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
log_err("Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
return false;
}
struct percpu* cpu = percpu_new();
if (!cpu) {
log_err("Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
return false;
}
log("Starting CPU %zu\n", cpu->id);
memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start);
struct smp_info* info = HHDM_PTR(info_addr);
memcpy(&info->gdt, &gdt_base, sizeof(info->gdt));
info->gdtr.limit = sizeof(struct gdt) - 1;
info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt));
info->farjmp_offset = 0; // Filled in by the trampoline.
info->farjmp_segment = offsetof(struct gdt, kernel_code32);
info->temp_stack = stack + AP_BOOT_STACK_SIZE;
info->temp_cr3 = temp_cr3;
info->entry = (uint64_t)ap_entry;
info->hhdm_offset = mem_hhdm_base;
info->percpu = (uint64_t)cpu;
info->lapic_id = lapic_id;
atomic_init(&info->booted, 0);
struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE);
clock_spin_ns(10'000'000);
lapic_send_ipi(
lapic,
lapic_id,
trampoline >> 12,
APIC_DELIVERY_STARTUP,
APIC_DEST_PHYSICAL,
APIC_LEVEL_ASSERT,
APIC_TRIGGER_EDGE
);
clock_spin_ns(10'000'000);
for (uint64_t waited = 0; !atomic_load(&info->booted, ATOMIC_ACQUIRE); waited += 1'000'000) {
if (waited >= AP_BOOT_TIMEOUT_NS) {
// The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
log_err("CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
return false;
}
clock_spin_ns(1'000'000);
}
return true;
}
static errno_t build_temp_table(phys_t* out) {
const size_t pgsz = arch_mem_page_size();
phys_t mem;
errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem);
if (status)
return status;
uint64_t* l4 = HHDM_PTR(mem);
uint64_t* l3 = HHDM_PTR(mem + pgsz);
const phys_t l2_base = mem + 2 * pgsz;
// Use 2 MiB pages.
l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
for (size_t i = 0; i < 4; i++) {
uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz);
l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
for (size_t j = 0; j < 512; j++)
l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE;
}
// Copy over the higher half from the kernel page table.
const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root);
for (size_t i = 256; i < 512; i++)
l4[i] = kernel_l4[i];
*out = mem;
return 0;
}
void x86_64_smp_init() {
const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page");
const size_t max_cpus = percpu_max_cpus();
if (max_cpus <= 1)
return;
uacpi_table table;
if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
log_warn("No MADT found, not starting any APs\n");
return;
}
phys_t trampoline;
if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
log_err("Failed to allocate 20-bit memory for the AP trampoline\n");
uacpi_table_unref(&table);
return;
}
phys_t temp_table;
if (build_temp_table(&temp_table)) {
log_err("Failed to allocate a temporary page table\n");
mem_phys_free(trampoline, 1);
uacpi_table_unref(&table);
return;
}
const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id;
bool all_booted = true;
size_t num_cpus = 1;
const struct acpi_madt* madt = table.ptr;
const uint8_t* entry = (const uint8_t*)madt->entries;
const uint8_t* end = (const uint8_t*)madt + madt->hdr.length;
while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) {
const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry;
// Don't trust malformed firmware tables.
if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
log_warn("MADT contains a malformed entry\n");
break;
}
uint32_t lapic_id = 0;
bool enabled = false;
switch (hdr->type) {
case ACPI_MADT_ENTRY_TYPE_LAPIC:
const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr;
lapic_id = lapic->id;
enabled = lapic->flags & ACPI_PIC_ENABLED;
break;
case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC:
const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr;
lapic_id = x2apic->id;
enabled = x2apic->flags & ACPI_PIC_ENABLED;
break;
}
if (enabled && lapic_id != bsp_id) {
if (start_ap(trampoline, temp_table, lapic_id))
num_cpus++;
else
all_booted = false;
}
entry += hdr->length;
}
uacpi_table_unref(&table);
// An AP which didn't respond might still be using these.
if (all_booted) {
mem_phys_free(temp_table, 6);
mem_phys_free(trampoline, 1);
}
log("%zu CPU(s) online\n", num_cpus);
}
arch/x86_64/start.c
-19
View File
@@ -1,19 +0,0 @@
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/init.h>
#include <zinnia/percpu.h>
#include <uapi/archctl.h>
#include <x86_64/apic.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
[[__init, __naked, __used]]
void _start() {
asm volatile(
"mov rsp, %0\n"
"jmp %1"
:
: "r"(__ld_stack_top), "r"(kernel_entry)
);
}
arch/x86_64/syscall.c
-73
View File
@@ -1,73 +0,0 @@
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <zinnia/syscall.h>
#include <bits/sched.h>
#include <stddef.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
// TODO: Refactor to only push callee-saved registers.
[[__naked]]
void arch_syscall_stub() {
asm volatile(
" swapgs\n"
" mov gs:%c[user_stack], rsp\n"
" mov rsp, gs:%c[kernel_stack]\n"
" cld\n"
"1:\n"
" push %c[user_data]\n" // ss
" push gs:%c[user_stack]\n"
" push r11\n" // rflags
" push %c[user_code64]\n" // cs
" push rcx\n" // rip
" push 0x00\n" // arch_context::error
" push 0x00\n" // arch_context::isr
" push rax\n"
" push rbx\n"
" push rcx\n"
" push rdx\n"
" push rbp\n"
" push rdi\n"
" push rsi\n"
" push r8\n"
" push r9\n"
" push r10\n"
" push r11\n"
" push r12\n"
" push r13\n"
" push r14\n"
" push r15\n"
" xor rbp, rbp\n"
" mov rdi, rsp\n" // SysV ABI first argument.
" call syscall_dispatch\n" // Call syscall handler.
" cli\n"
" pop r15\n"
" pop r14\n"
" pop r13\n"
" pop r12\n"
" pop r11\n"
" pop r10\n"
" pop r9\n"
" pop r8\n"
" pop rsi\n"
" pop rdi\n"
" pop rbp\n"
" pop rdx\n"
" pop rcx\n"
" pop rbx\n"
" pop rax\n"
" add rsp, 0x10\n" // Skip .error and .isr fields
" pop rcx\n" // rip
" add rsp, 0x08\n" // Skip cs
" pop r11\n" // rflags
" pop rsp\n" // rsp
" swapgs\n"
" sysretq\n"
:
: [user_stack] "i"(offsetof(struct percpu, user_stack)),
[kernel_stack] "i"(offsetof(struct percpu, kernel_stack)),
[user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER),
[user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER)
);
}
arch/x86_64/tsc.c
-88
View File
@@ -1,88 +0,0 @@
#define pr_fmt(fmt) "x86_64/tsc: " fmt
#include <zinnia/clock.h>
#include <zinnia/cmdline.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <stdint.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
static bool use_tsc = true;
static void tsc_option(bool opt) {
use_tsc = opt;
}
CMDLINE_OPTION("x86_64.tsc", "Enable timestamp counter", tsc_option);
constexpr uint64_t nanos_per_sec = 1'000'000'000;
static uint64_t tsc_freq = 0;
static uint64_t tsc_base_cycles = 0;
static uint64_t cycles_to_ns(uint64_t cycles, uint64_t freq) {
uint64_t seconds = cycles / freq;
uint64_t remaining_cycles = cycles % freq;
uint64_t fractional_ns = (uint64_t)((__uint128_t)remaining_cycles * (__uint128_t)nanos_per_sec / (__uint128_t)freq);
uint64_t ns = seconds * nanos_per_sec + fractional_ns;
return MIN(ns, UINT64_MAX);
}
static void tsc_reset(struct clock* clock) {
tsc_base_cycles = asm_rdtsc();
}
static uint64_t tsc_get_elapsed_ns(struct clock* clock) {
uint64_t cycles = asm_rdtsc() - tsc_base_cycles;
return cycles_to_ns(cycles, tsc_freq);
}
static struct clock tsc_clock = {
.name = "tsc",
.priority = 255,
.reset = tsc_reset,
.get_elapsed_ns = tsc_get_elapsed_ns,
};
void tsc_init() {
if (!use_tsc) {
log_warn("TSC disabled on the command line");
return;
}
if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
log_warn("No invariant TSC detected!\n");
return;
}
// Check for the TSC info leaf.
bool has_tsc_leaf = false;
struct cpuid tsc_leaf = {0};
if (asm_cpuid(0, 0).eax >= 0x15) {
tsc_leaf = asm_cpuid(0x15, 0);
has_tsc_leaf = true;
}
if (clock_available()) {
// Calibrate using existing clock.
uint64_t t1 = asm_rdtsc();
clock_spin_ns(10'000'000);
uint64_t t2 = asm_rdtsc();
tsc_freq = (t2 - t1) * 100;
} else if (has_tsc_leaf) {
// If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency.
// On a normal system, this should usually never be called and is a last resort
// since at this point we have at least the HPET timer.
if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
log("Calibrating TSC using CPUID 0x15");
tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
} else {
log_err("Unable to calibrate TSC using CPUID frequency information");
return;
}
} else {
log_err("Unable to calibrate the TSC without a clock or static frequency info!\n");
return;
}
log("Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
clock_switch(&tsc_clock);
}
arch/x86_64/usercopy.c
-142
View File
@@ -1,142 +0,0 @@
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <arch/usercopy.h>
#include <x86_64/percpu.h>
void arch_usercopy_allow(bool state) {
if (!percpu_ref(arch_data)->can_smap)
return;
if (state) {
asm volatile("stac" ::: "memory");
} else {
asm volatile("clac" ::: "memory");
}
}
extern void x86_64_read_start();
extern void x86_64_read_end();
extern void x86_64_read_fault();
static const struct usercopy_region read_region = {
.start_ip = x86_64_read_start,
.end_ip = x86_64_read_end,
.fault_ip = x86_64_read_fault,
};
[[__naked]]
bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) {
asm volatile(
// Setup regs for `rep movsb`.
" xchg rcx, rdx\n"
" lea rax, [rip + %0]\n"
" xchg [rdx], rax\n"
".global x86_64_read_start\n"
" x86_64_read_start:\n"
" rep movsb\n"
".global x86_64_read_end\n"
"x86_64_read_end:\n"
" xor rax, rax\n"
" xchg [rdx], rax\n"
" mov rax, 1\n"
" ret\n"
".global x86_64_read_fault\n"
"x86_64_read_fault:\n"
" xor rax, rax\n"
" mov [rdx], rax\n"
" ret\n"
:
: "i"(&read_region)
: "memory"
);
}
extern void x86_64_write_start();
extern void x86_64_write_end();
extern void x86_64_write_fault();
static const struct usercopy_region write_region = {
.start_ip = x86_64_write_start,
.end_ip = x86_64_write_end,
.fault_ip = x86_64_write_fault,
};
[[__naked]]
bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) {
asm volatile(
// Setup regs for `rep movsb`.
" xchg rcx, rdx\n"
" lea rax, [rip + %0]\n"
" mov [rdx], rax\n"
".global x86_64_write_start\n"
" x86_64_write_start:\n"
" rep movsb\n"
".global x86_64_write_end\n"
" x86_64_write_end:\n"
" xor rax, rax\n"
" xchg [rdx], rax\n"
" mov rax, 1\n"
" ret\n"
".global x86_64_write_fault\n"
"x86_64_write_fault:\n"
" xor rax, rax\n"
" mov [rdx], rax\n"
" ret\n"
:
: "i"(&write_region)
: "memory"
);
}
extern void x86_64_strlen_start();
extern void x86_64_strlen_end();
extern void x86_64_strlen_fault();
static const struct usercopy_region strlen_region = {
.start_ip = x86_64_strlen_start,
.end_ip = x86_64_strlen_end,
.fault_ip = x86_64_strlen_fault,
};
[[__naked]]
bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
asm volatile(
" lea rax, [rip + %0]\n"
" xchg [rcx], rax\n"
".global x86_64_strlen_start\n"
" x86_64_strlen_start:\n"
" xor r8, r8\n"
".Lloop:\n"
" cmp r8, rsi\n"
" je .Lleave\n"
" cmp byte ptr [rdi + r8], 0\n"
" je .Lleave\n"
" inc r8\n"
" jmp .Lloop\n"
".Lleave:\n"
" mov [rdx], r8\n"
".global x86_64_strlen_end\n"
"x86_64_strlen_end:\n"
" xor rax, rax\n"
" xchg [rcx], rax\n"
" mov rax, 1\n"
" ret\n"
".global x86_64_strlen_fault\n"
"x86_64_strlen_fault:\n"
" xor rax, rax\n"
" mov [rcx], rax\n"
" ret\n"
:
: "i"(&strlen_region)
: "memory"
);
}
boot/cmdline.c
-122
View File
@@ -1,122 +0,0 @@
#include <zinnia/cmdline.h>
#include <zinnia/init.h>
#include <zinnia/stdlib.h>
#include <stddef.h>
#include <string.h>
// Defined in linker script.
extern const struct cmdline_option* __ld_cmdline_start[];
extern const struct cmdline_option* __ld_cmdline_end[];
static char cmdline_buf[0x1000];
static char cmdline_raw[0x1000];
const char* cmdline_get() {
return cmdline_raw;
}
void cmdline_parse_buf(
char* cmdline_buf,
size_t len,
const struct cmdline_option* const* start,
const struct cmdline_option* const* end
) {
size_t idx = 0;
while (1) {
const struct cmdline_option* const* opt = start;
char* name = nullptr;
char* value = nullptr;
// Skip all leading spaces.
while (idx < len && cmdline_buf[idx] == ' ')
idx++;
if (idx >= len)
break;
size_t name_idx = idx;
name = cmdline_buf + name_idx;
// Find the next equal sign or space.
while (idx < len && cmdline_buf[idx] != '=' && cmdline_buf[idx] != ' ')
idx++;
if (idx > len)
break;
// Check if the option has a value (=foo).
char seperator = cmdline_buf[idx];
cmdline_buf[idx++] = 0;
if (seperator == '=') {
// Check if we need to escape the value.
char check;
if (cmdline_buf[idx] == '"') {
check = '"';
cmdline_buf[idx++] = 0;
} else {
check = ' ';
}
value = cmdline_buf + idx;
// Skip the value.
while (idx < len && cmdline_buf[idx] != check)
idx++;
if (idx > len)
break;
cmdline_buf[idx++] = 0;
}
// Find the corresponding option.
while (opt < end) {
const struct cmdline_option* current = *opt;
if (!strcmp(current->name, name)) {
if (value == nullptr)
break;
switch (current->option_type) {
case CMDLINE_STRING:
current->func_str(value);
break;
case CMDLINE_BOOL:
if (!strcmp(value, "on") || !strcmp(value, "yes") || !strcmp(value, "true") || !strcmp(value, "1"))
current->func_bool(true);
else
current->func_bool(false);
break;
case CMDLINE_INTPTR: {
int base = 10;
if (!strncmp(value, "0x", 2)) {
base = 0x10;
value += 2;
}
current->func_intptr(atol(value, base));
break;
}
case CMDLINE_UINTPTR: {
int base = 10;
if (!strncmp(value, "0x", 2)) {
base = 0x10;
value += 2;
}
current->func_uintptr(atolu(value, base));
break;
}
}
break;
}
opt++;
}
if (idx >= len)
break;
}
}
[[__init]]
void cmdline_parse(const char* cmdline) {
if (cmdline == nullptr)
cmdline = "";
const size_t len = strnlen(cmdline, sizeof(cmdline_buf) - 1);
memcpy(cmdline_buf, cmdline, len);
memcpy(cmdline_raw, cmdline, len);
cmdline_parse_buf(cmdline_buf, len, __ld_cmdline_start, __ld_cmdline_end);
}
boot/limine/entry.c
-128
View File
@@ -1,128 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/cmdline.h>
#include <zinnia/init.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <string.h>
#include "limine.h"
[[__initdata_sorted("limine.0")]]
static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
[[__initdata_sorted("limine.1")]]
static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
[[__initdata_sorted("limine.2")]]
static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
#define LIMINE_REQUEST(request, tag, rev) \
[[__initdata_sorted("limine.1")]] \
static volatile struct limine_##request request = { \
.id = tag, \
.revision = rev, \
.response = nullptr, \
}
LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
[[__initdata]]
static struct phys_mem mem[128];
[[__initdata]]
static struct boot_file files[8];
[[__init]]
void kernel_entry() {
kernel_early_init();
struct boot_info info = {};
const struct limine_memmap_response* mm_res = memmap_request.response;
const struct limine_hhdm_response* hhdm_res = hhdm_request.response;
const struct limine_module_response* module_res = module_request.response;
const struct limine_executable_address_response* exec_res = executable_address_request.response;
const struct limine_executable_file_response* exec_file_res = executable_file_request.response;
const struct limine_rsdp_response* rsdp_res = rsdp_request.response;
assert(mm_res, "memmap_request returned no response!");
assert(hhdm_res, "hhdm_request returned no response!");
info.cmdline = exec_file_res->executable_file->string;
info.kernel_file.data = (phys_t)((uintptr_t)exec_file_res->executable_file->address - hhdm_res->offset);
info.kernel_file.length = exec_file_res->executable_file->size;
info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
for (size_t i = 0; i < info.num_mem_maps; i++) {
mem[i].address = mm_res->entries[i]->base;
mem[i].length = mm_res->entries[i]->length;
switch (mm_res->entries[i]->type) {
case LIMINE_MEMMAP_USABLE:
mem[i].usage = PHYS_USABLE;
break;
case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
mem[i].usage = PHYS_RECLAIMABLE;
break;
case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
mem[i].usage = PHYS_STATIC;
break;
default:
mem[i].usage = PHYS_RESERVED;
break;
}
}
if (module_res) {
info.files = files;
info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
for (size_t i = 0; i < info.num_files; i++) {
files[i].data = (phys_t)module_res->modules[i]->address - hhdm_res->offset;
files[i].length = module_res->modules[i]->size;
strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1);
files[i].path[sizeof(files[i].path) - 1] = 0;
}
}
info.mem_map = mem;
info.phys_base = (phys_t)exec_res->physical_base;
info.virt_base = (uintptr_t)exec_res->virtual_base;
info.hhdm_base = (uintptr_t)hhdm_res->offset;
const struct limine_framebuffer_response* fb_res = framebuffer_request.response;
if (fb_res) {
// Only use the first framebuffer.
if (fb_res->framebuffer_count >= 1) {
const struct limine_framebuffer* fb = framebuffer_request.response->framebuffers[0];
info.fb.base = (phys_t)((uintptr_t)fb->address - hhdm_res->offset);
info.fb.width = fb->width;
info.fb.height = fb->height;
info.fb.pitch = fb->pitch;
info.fb.bytes_per_pixel = fb->bpp / 8;
info.fb.red.shift = fb->red_mask_shift;
info.fb.red.size = fb->red_mask_size;
info.fb.green.shift = fb->green_mask_shift;
info.fb.green.size = fb->green_mask_size;
info.fb.blue.shift = fb->blue_mask_shift;
info.fb.blue.size = fb->blue_mask_size;
}
}
#if CONFIG_ACPI
if (rsdp_request.response) {
info.rsdp = (phys_t)rsdp_request.response->address;
if (info.rsdp >= info.hhdm_base)
info.rsdp -= info.hhdm_base;
}
#endif
kernel_main(&info);
}
boot/limine/limine.h
-669
View File
@@ -1,669 +0,0 @@
/* SPDX-License-Identifier: 0BSD */
/* Copyright (C) 2022-2026 Mintsuki and contributors.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef LIMINE_H
#define LIMINE_H 1
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/* Misc */
#ifdef LIMINE_NO_POINTERS
# define LIMINE_PTR(TYPE) uint64_t
#else
# define LIMINE_PTR(TYPE) TYPE
#endif
#define LIMINE_REQUESTS_START_MARKER { 0xf6b8f4b39de7d1ae, 0xfab91a6940fcb9cf, \
0x785c6ed015d3e316, 0x181e920a7852b9d9 }
#define LIMINE_REQUESTS_END_MARKER { 0xadc0e0531bb10d03, 0x9572709f31764c62 }
#define LIMINE_BASE_REVISION(N) { 0xf9562b2d5c95a6c8, 0x6a7b384944536bdc, (N) }
#define LIMINE_BASE_REVISION_SUPPORTED(VAR) ((VAR)[2] == 0)
#define LIMINE_LOADED_BASE_REVISION_VALID(VAR) ((VAR)[1] != 0x6a7b384944536bdc)
#define LIMINE_LOADED_BASE_REVISION(VAR) ((VAR)[1])
#define LIMINE_COMMON_MAGIC 0xc7b1dd30df4c8b88, 0x0a82e883a194f07b
struct limine_uuid {
uint32_t a;
uint16_t b;
uint16_t c;
uint8_t d[8];
};
#define LIMINE_MEDIA_TYPE_GENERIC 0
#define LIMINE_MEDIA_TYPE_OPTICAL 1
#define LIMINE_MEDIA_TYPE_TFTP 2
struct limine_file {
uint64_t revision;
LIMINE_PTR(void *) address;
uint64_t size;
LIMINE_PTR(char *) path;
LIMINE_PTR(char *) string;
uint32_t media_type;
uint32_t unused;
uint8_t tftp_ipv4[4];
uint32_t tftp_port;
uint32_t partition_index;
uint32_t mbr_disk_id;
struct limine_uuid gpt_disk_uuid;
struct limine_uuid gpt_part_uuid;
struct limine_uuid part_uuid;
};
/* Boot info */
#define LIMINE_BOOTLOADER_INFO_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xf55038d8e2a1202f, 0x279426fcf5f59740 }
struct limine_bootloader_info_response {
uint64_t revision;
LIMINE_PTR(char *) name;
LIMINE_PTR(char *) version;
};
struct limine_bootloader_info_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_bootloader_info_response *) response;
};
/* Executable command line */
#define LIMINE_EXECUTABLE_CMDLINE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x4b161536e598651e, 0xb390ad4a2f1f303a }
struct limine_executable_cmdline_response {
uint64_t revision;
LIMINE_PTR(char *) cmdline;
};
struct limine_executable_cmdline_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_executable_cmdline_response *) response;
};
/* Firmware type */
#define LIMINE_FIRMWARE_TYPE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8c2f75d90bef28a8, 0x7045a4688eac00c3 }
#define LIMINE_FIRMWARE_TYPE_X86BIOS 0
#define LIMINE_FIRMWARE_TYPE_EFI32 1
#define LIMINE_FIRMWARE_TYPE_EFI64 2
#define LIMINE_FIRMWARE_TYPE_SBI 3
struct limine_firmware_type_response {
uint64_t revision;
uint64_t firmware_type;
};
struct limine_firmware_type_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_firmware_type_response *) response;
};
/* Stack size */
#define LIMINE_STACK_SIZE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x224ef0460a8e8926, 0xe1cb0fc25f46ea3d }
struct limine_stack_size_response {
uint64_t revision;
};
struct limine_stack_size_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_stack_size_response *) response;
uint64_t stack_size;
};
/* HHDM */
#define LIMINE_HHDM_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x48dcf1cb8ad2b852, 0x63984e959a98244b }
struct limine_hhdm_response {
uint64_t revision;
uint64_t offset;
};
struct limine_hhdm_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_hhdm_response *) response;
};
/* Framebuffer */
#define LIMINE_FRAMEBUFFER_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9d5827dcd881dd75, 0xa3148604f6fab11b }
#define LIMINE_FRAMEBUFFER_RGB 1
struct limine_video_mode {
uint64_t pitch;
uint64_t width;
uint64_t height;
uint16_t bpp;
uint8_t memory_model;
uint8_t red_mask_size;
uint8_t red_mask_shift;
uint8_t green_mask_size;
uint8_t green_mask_shift;
uint8_t blue_mask_size;
uint8_t blue_mask_shift;
};
struct limine_framebuffer {
LIMINE_PTR(void *) address;
uint64_t width;
uint64_t height;
uint64_t pitch;
uint16_t bpp;
uint8_t memory_model;
uint8_t red_mask_size;
uint8_t red_mask_shift;
uint8_t green_mask_size;
uint8_t green_mask_shift;
uint8_t blue_mask_size;
uint8_t blue_mask_shift;
uint8_t unused[7];
uint64_t edid_size;
LIMINE_PTR(void *) edid;
/* Response revision 1 */
uint64_t mode_count;
LIMINE_PTR(struct limine_video_mode **) modes;
};
struct limine_framebuffer_response {
uint64_t revision;
uint64_t framebuffer_count;
LIMINE_PTR(struct limine_framebuffer **) framebuffers;
};
struct limine_framebuffer_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_framebuffer_response *) response;
};
/* Flanterm FB init params */
#define LIMINE_FLANTERM_FB_INIT_PARAMS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3259399fe7c5f126, 0xe01c1c8c5db9d1a9 }
#define LIMINE_FLANTERM_FB_ROTATE_0 0
#define LIMINE_FLANTERM_FB_ROTATE_90 1
#define LIMINE_FLANTERM_FB_ROTATE_180 2
#define LIMINE_FLANTERM_FB_ROTATE_270 3
struct limine_flanterm_fb_init_params {
LIMINE_PTR(uint32_t *) canvas;
uint64_t canvas_size;
uint32_t ansi_colours[8];
uint32_t ansi_bright_colours[8];
uint32_t default_bg;
uint32_t default_fg;
uint32_t default_bg_bright;
uint32_t default_fg_bright;
LIMINE_PTR(void *) font;
uint64_t font_width;
uint64_t font_height;
uint64_t font_spacing;
uint64_t font_scale_x;
uint64_t font_scale_y;
uint64_t margin;
uint64_t rotation;
};
struct limine_flanterm_fb_init_params_response {
uint64_t revision;
uint64_t entry_count;
LIMINE_PTR(struct limine_flanterm_fb_init_params **) entries;
};
struct limine_flanterm_fb_init_params_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_flanterm_fb_init_params_response *) response;
};
/* Paging mode */
#define LIMINE_PAGING_MODE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95c1a0edab0944cb, 0xa4e5cb3842f7488a }
#define LIMINE_PAGING_MODE_X86_64_4LVL 0
#define LIMINE_PAGING_MODE_X86_64_5LVL 1
#define LIMINE_PAGING_MODE_X86_64_MIN LIMINE_PAGING_MODE_X86_64_4LVL
#define LIMINE_PAGING_MODE_X86_64_DEFAULT LIMINE_PAGING_MODE_X86_64_4LVL
#define LIMINE_PAGING_MODE_AARCH64_4LVL 0
#define LIMINE_PAGING_MODE_AARCH64_5LVL 1
#define LIMINE_PAGING_MODE_AARCH64_MIN LIMINE_PAGING_MODE_AARCH64_4LVL
#define LIMINE_PAGING_MODE_AARCH64_DEFAULT LIMINE_PAGING_MODE_AARCH64_4LVL
#define LIMINE_PAGING_MODE_RISCV_SV39 0
#define LIMINE_PAGING_MODE_RISCV_SV48 1
#define LIMINE_PAGING_MODE_RISCV_SV57 2
#define LIMINE_PAGING_MODE_RISCV_MIN LIMINE_PAGING_MODE_RISCV_SV39
#define LIMINE_PAGING_MODE_RISCV_DEFAULT LIMINE_PAGING_MODE_RISCV_SV48
#define LIMINE_PAGING_MODE_LOONGARCH_4LVL 0
#define LIMINE_PAGING_MODE_LOONGARCH_MIN LIMINE_PAGING_MODE_LOONGARCH_4LVL
#define LIMINE_PAGING_MODE_LOONGARCH_DEFAULT LIMINE_PAGING_MODE_LOONGARCH_4LVL
struct limine_paging_mode_response {
uint64_t revision;
uint64_t mode;
};
struct limine_paging_mode_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_paging_mode_response *) response;
uint64_t mode;
uint64_t max_mode;
uint64_t min_mode;
};
/* MP */
#define LIMINE_MP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95a67b819a1b857e, 0xa0b61b723b6a73e0 }
struct limine_mp_info;
typedef void (*limine_goto_address)(struct limine_mp_info *);
#if defined (__x86_64__) || defined (__i386__)
#define LIMINE_MP_RESPONSE_X86_64_X2APIC (1 << 0)
struct limine_mp_info {
uint32_t processor_id;
uint32_t lapic_id;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint32_t flags;
uint32_t bsp_lapic_id;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info **) cpus;
};
#elif defined (__aarch64__)
struct limine_mp_info {
uint32_t processor_id;
uint32_t reserved1;
uint64_t mpidr;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint64_t flags;
uint64_t bsp_mpidr;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info **) cpus;
};
#elif defined (__riscv) && (__riscv_xlen == 64)
struct limine_mp_info {
uint64_t processor_id;
uint64_t hartid;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint64_t flags;
uint64_t bsp_hartid;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info **) cpus;
};
#elif defined (__loongarch__) && (__loongarch_grlen == 64)
struct limine_mp_info {
uint64_t processor_id;
uint64_t phys_id;
uint64_t reserved;
LIMINE_PTR(limine_goto_address) goto_address;
uint64_t extra_argument;
};
struct limine_mp_response {
uint64_t revision;
uint64_t flags;
uint64_t bsp_phys_id;
uint64_t cpu_count;
LIMINE_PTR(struct limine_mp_info **) cpus;
};
#else
#error Unknown architecture
#endif
#define LIMINE_MP_REQUEST_X86_64_X2APIC (1 << 0)
struct limine_mp_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_mp_response *) response;
uint64_t flags;
};
/* Memory map */
#define LIMINE_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x67cf3d9d378a806f, 0xe304acdfc50c3c62 }
#define LIMINE_MEMMAP_USABLE 0
#define LIMINE_MEMMAP_RESERVED 1
#define LIMINE_MEMMAP_ACPI_RECLAIMABLE 2
#define LIMINE_MEMMAP_ACPI_NVS 3
#define LIMINE_MEMMAP_BAD_MEMORY 4
#define LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE 5
#define LIMINE_MEMMAP_EXECUTABLE_AND_MODULES 6
#define LIMINE_MEMMAP_FRAMEBUFFER 7
#define LIMINE_MEMMAP_RESERVED_MAPPED 8
struct limine_memmap_entry {
uint64_t base;
uint64_t length;
uint64_t type;
};
struct limine_memmap_response {
uint64_t revision;
uint64_t entry_count;
LIMINE_PTR(struct limine_memmap_entry **) entries;
};
struct limine_memmap_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_memmap_response *) response;
};
/* Entry point */
#define LIMINE_ENTRY_POINT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x13d86c035a1cd3e1, 0x2b0caa89d8f3026a }
typedef void (*limine_entry_point)(void);
struct limine_entry_point_response {
uint64_t revision;
};
struct limine_entry_point_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_entry_point_response *) response;
LIMINE_PTR(limine_entry_point) entry;
};
/* Executable File */
#define LIMINE_EXECUTABLE_FILE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xad97e90e83f1ed67, 0x31eb5d1c5ff23b69 }
struct limine_executable_file_response {
uint64_t revision;
LIMINE_PTR(struct limine_file *) executable_file;
};
struct limine_executable_file_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_executable_file_response *) response;
};
/* Module */
#define LIMINE_MODULE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3e7e279702be32af, 0xca1c4f3bd1280cee }
#define LIMINE_INTERNAL_MODULE_REQUIRED (1 << 0)
#define LIMINE_INTERNAL_MODULE_COMPRESSED (1 << 1)
struct limine_internal_module {
LIMINE_PTR(const char *) path;
LIMINE_PTR(const char *) string;
uint64_t flags;
};
struct limine_module_response {
uint64_t revision;
uint64_t module_count;
LIMINE_PTR(struct limine_file **) modules;
};
struct limine_module_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_module_response *) response;
/* Request revision 1 */
uint64_t internal_module_count;
LIMINE_PTR(struct limine_internal_module **) internal_modules;
};
/* RSDP */
#define LIMINE_RSDP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xc5e77b6b397e7b43, 0x27637845accdcf3c }
struct limine_rsdp_response {
uint64_t revision;
LIMINE_PTR(void *) address;
};
struct limine_rsdp_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_rsdp_response *) response;
};
/* SMBIOS */
#define LIMINE_SMBIOS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9e9046f11e095391, 0xaa4a520fefbde5ee }
struct limine_smbios_response {
uint64_t revision;
LIMINE_PTR(void *) entry_32;
LIMINE_PTR(void *) entry_64;
};
struct limine_smbios_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_smbios_response *) response;
};
/* EFI system table */
#define LIMINE_EFI_SYSTEM_TABLE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x5ceba5163eaaf6d6, 0x0a6981610cf65fcc }
struct limine_efi_system_table_response {
uint64_t revision;
LIMINE_PTR(void *) address;
};
struct limine_efi_system_table_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_efi_system_table_response *) response;
};
/* TPM event log */
#define LIMINE_TPM_EVENT_LOG_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x98e094fc7e76e979, 0xee8d8775c54e1d1f }
#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_1_2 1
#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_2 2
struct limine_tpm_event_log_response {
uint64_t revision;
uint64_t format;
uint64_t size;
LIMINE_PTR(void *) address;
};
struct limine_tpm_event_log_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_tpm_event_log_response *) response;
};
/* EFI memory map */
#define LIMINE_EFI_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x7df62a431d6872d5, 0xa4fcdfb3e57306c8 }
struct limine_efi_memmap_response {
uint64_t revision;
LIMINE_PTR(void *) memmap;
uint64_t memmap_size;
uint64_t desc_size;
uint64_t desc_version;
};
struct limine_efi_memmap_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_efi_memmap_response *) response;
};
/* Date at boot */
#define LIMINE_DATE_AT_BOOT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x502746e184c088aa, 0xfbc5ec83e6327893 }
struct limine_date_at_boot_response {
uint64_t revision;
int64_t timestamp;
};
struct limine_date_at_boot_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_date_at_boot_response *) response;
};
/* Executable address */
#define LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x71ba76863cc55f63, 0xb2644a48c516a487 }
struct limine_executable_address_response {
uint64_t revision;
uint64_t physical_base;
uint64_t virtual_base;
};
struct limine_executable_address_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_executable_address_response *) response;
};
/* Device Tree Blob */
#define LIMINE_DTB_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xb40ddb48fb54bac7, 0x545081493f81ffb7 }
struct limine_dtb_response {
uint64_t revision;
LIMINE_PTR(void *) dtb_ptr;
};
struct limine_dtb_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_dtb_response *) response;
};
/* RISC-V Boot Hart ID */
#define LIMINE_RISCV_BSP_HARTID_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x1369359f025525f9, 0x2ff2a56178391bb6 }
struct limine_riscv_bsp_hartid_response {
uint64_t revision;
uint64_t bsp_hartid;
};
struct limine_riscv_bsp_hartid_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_riscv_bsp_hartid_response *) response;
};
/* Bootloader Performance */
#define LIMINE_BOOTLOADER_PERFORMANCE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x6b50ad9bf36d13ad, 0xdc4c7e88fc759e17 }
struct limine_bootloader_performance_response {
uint64_t revision;
uint64_t reset_usec;
uint64_t init_usec;
uint64_t exec_usec;
};
struct limine_bootloader_performance_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_bootloader_performance_response *) response;
};
#define LIMINE_X86_64_KEEP_IOMMU_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8ebaabe51f490179, 0x2aa86a59ffb4ab0f }
struct limine_x86_64_keep_iommu_response {
uint64_t revision;
};
struct limine_x86_64_keep_iommu_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_x86_64_keep_iommu_response *) response;
};
/* TSC (Timestamp Counter) Frequency */
#define LIMINE_TSC_FREQUENCY_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x10f2ee1d87d195e4, 0xf747a2b78f6ddb31 }
struct limine_tsc_frequency_response {
uint64_t revision;
uint64_t frequency;
};
struct limine_tsc_frequency_request {
uint64_t id[4];
uint64_t revision;
LIMINE_PTR(struct limine_tsc_frequency_response *) response;
};
#ifdef __cplusplus
}
#endif
#endif
boot/main.c
-94
View File
@@ -1,94 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/clock.h>
#include <zinnia/cmdline.h>
#include <zinnia/compiler.h>
#include <zinnia/console.h>
#include <zinnia/elf.h>
#include <zinnia/init.h>
#include <zinnia/initgraph.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <zinnia/vmo.h>
#include <config.h>
#include <stdint.h>
static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
static void kernel_main_task(uintptr_t arg0) {
errno_t status = 0;
struct boot_info* info = (struct boot_info*)arg0;
log("main: Hello from main task!\n");
// Finish the initgraph.
initgraph_run(nullptr, info);
// TODO: Try to run init.
assert(info->num_files >= 1, "No init executable provided");
struct vma* init_space;
assert(vma_new(&init_space) == 0, "");
struct boot_file* file_data = &info->files[0];
struct task* init_task = nullptr;
struct paged_vmo* init_file;
status = vmo_new_phys(&init_file);
assert(status == 0, "Failed to allocate a memory object");
vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
// Load init.
const char* init_argv[] = {"init", nullptr};
const char* init_envp[] = {nullptr};
struct exec_info exec = {
.space = init_space,
.file_obj = &init_file->object,
.argv = init_argv,
.envp = init_envp,
};
status = elf_load(&exec, &init_task);
vmo_unref(&init_file->object);
assert(status == 0, "Failed to load init process (%i)", status);
sched_add_task(&percpu_get()->sched, init_task);
while (true) {
irq_wait();
}
}
[[__init]]
void kernel_early_init() {
percpu_bsp_init();
percpu_get()->online = true;
irq_lock();
}
[[noreturn]]
void kernel_main(struct boot_info* info) {
cmdline_parse(info->cmdline);
log("%s\n", zinnia_banner); // Say hello!
log("Command line: %s\n", info->cmdline);
initgraph_run(sched_stage, info);
struct task* main_task;
task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
sched_add_task(&percpu_get()->sched, main_task);
irq_unlock();
sched_yield(&percpu_get()->sched);
unreachable();
}
core/Kconfig
-18
View File
@@ -1,18 +0,0 @@
menu "Kernel Core options"
source "core/mem/Kconfig"
choice SCHED_ALG
prompt "Scheduler algorithm"
default SCHED_DUMB
help
The algorithm to use for scheduling.
config SCHED_DUMB
bool "Dumb Scheduler"
help
Very basic scheduler, without any priorities.
endchoice
endmenu
core/clock.c
-64
View File
@@ -1,64 +0,0 @@
#define pr_fmt(fmt) "clock: " fmt
#include <zinnia/atomic.h>
#include <zinnia/clock.h>
#include <zinnia/compiler.h>
#include <zinnia/print.h>
#include <arch/cpu.h>
#include <stdint.h>
static _Atomic(struct clock*) active_clock;
static _Atomic(int64_t) realtime_offset_ns;
static uint64_t clock_read_elapsed(struct clock* clock) {
if (__unlikely(!clock))
return 0;
return clock->base + clock->get_elapsed_ns(clock);
}
bool clock_available() {
return atomic_load(&active_clock, ATOMIC_ACQUIRE) != nullptr;
}
uint64_t clock_get_elapsed_ns() {
return clock_read_elapsed(atomic_load(&active_clock, ATOMIC_ACQUIRE));
}
uint64_t clock_get_realtime_ns() {
int64_t ns = (int64_t)clock_get_elapsed_ns() + atomic_load(&realtime_offset_ns, ATOMIC_RELAXED);
return ns < 0 ? 0 : ns;
}
void clock_set_realtime_ns(uint64_t ns) {
atomic_store(&realtime_offset_ns, (int64_t)ns - (int64_t)clock_get_elapsed_ns(), ATOMIC_RELAXED);
}
bool clock_switch(struct clock* clock) {
struct clock* old = atomic_load(&active_clock, ATOMIC_ACQUIRE);
do {
if (old && clock->priority <= old->priority)
return false;
uint64_t elapsed = clock_read_elapsed(old);
clock->reset(clock);
clock->base = elapsed;
} while (!atomic_compare_exchange(&active_clock, &old, clock, ATOMIC_RELEASE, ATOMIC_ACQUIRE));
log("Switching to clock source \"%s\"\n", clock->name);
return true;
}
void clock_spin_ns(uint64_t ns) {
if (!clock_available()) {
log_err("Unable to sleep for %lu ns, no clock available!\n", ns);
return;
}
uint64_t target = clock_get_elapsed_ns() + ns;
while (clock_get_elapsed_ns() < target) {
arch_cpu_pause();
}
}
core/console.c
-47
View File
@@ -1,47 +0,0 @@
#include <zinnia/console.h>
#include <zinnia/init.h>
#include <zinnia/print.h>
#include <zinnia/spin.h>
static struct console* registered = nullptr;
static struct spinlock console_lock;
void console_write(const char* buf, size_t len) {
spin_lock(&console_lock);
for (struct console* con = registered; con; con = con->next)
con->write(con, buf, len);
spin_unlock(&console_lock);
}
static void console_replay_write(void* ctx, const char* buf, size_t len) {
struct console* con = ctx;
con->write(con, buf, len);
}
bool console_register(struct console* con) {
if (!con->init(con))
return false;
con->write(con, "\ec", 2);
log_replay(console_replay_write, con);
spin_lock(&console_lock);
con->next = registered;
registered = con;
spin_unlock(&console_lock);
log("console: Registered console \"%s\"\n", con->name);
return true;
}
void console_unregister(struct console* con) {
spin_lock(&console_lock);
for (struct console** link = &registered; *link; link = &(*link)->next) {
if (*link == con) {
*link = con->next;
break;
}
}
spin_unlock(&console_lock);
}
INIT_TASK(console_stage, "core.console");
core/dpc.c
-3
View File
@@ -1,3 +0,0 @@
#include <zinnia/dpc.h>
// TODO
core/initgraph.c
-194
View File
@@ -1,194 +0,0 @@
#define pr_fmt(fmt) "init: " fmt
#include <zinnia/assert.h>
#include <zinnia/cmdline.h>
#include <zinnia/initgraph.h>
#include <zinnia/print.h>
extern struct init_node* const __ld_initgraph_nodes_start[];
extern struct init_node* const __ld_initgraph_nodes_end[];
extern const struct init_edges* const __ld_initgraph_edges_start[];
extern const struct init_edges* const __ld_initgraph_edges_end[];
static bool dump_graph = false;
static void initgraph_option(bool value) {
dump_graph = value;
}
CMDLINE_OPTION("initgraph", "Print the init graph in DOT format after running", initgraph_option);
static bool edges_linked = false;
static struct init_node* failed_node = nullptr;
struct init_queue {
struct init_node* head;
struct init_node* tail;
};
static void queue_push(struct init_queue* queue, struct init_node* node) {
node->queue_next = nullptr;
if (queue->tail)
queue->tail->queue_next = node;
else
queue->head = node;
queue->tail = node;
}
static struct init_node* queue_pop(struct init_queue* queue) {
struct init_node* node = queue->head;
if (node) {
queue->head = node->queue_next;
if (!queue->head)
queue->tail = nullptr;
node->queue_next = nullptr;
}
return node;
}
static void link_edges() {
for (auto cur = __ld_initgraph_edges_start; cur < __ld_initgraph_edges_end; cur++) {
const struct init_edges* group = *cur;
for (size_t i = 0; i < group->count; i++) {
struct init_edge* edge = &group->edges[i];
edge->source = group->depends ? group->nodes[i] : group->node;
edge->target = group->depends ? group->node : group->nodes[i];
edge->propagates_disable = group->depends;
edge->next_out = edge->source->out_edges;
edge->source->out_edges = edge;
edge->next_in = edge->target->in_edges;
edge->target->in_edges = edge;
edge->target->unsatisfied++;
}
}
edges_linked = true;
}
static void mark_wanted(struct init_node* goal) {
if (!goal) {
for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++)
(*cur)->wanted = true;
return;
}
struct init_queue queue = {};
if (!goal->wanted) {
goal->wanted = true;
queue_push(&queue, goal);
}
struct init_node* node;
while ((node = queue_pop(&queue))) {
for (struct init_edge* edge = node->in_edges; edge; edge = edge->next_in) {
if (!edge->source->wanted) {
edge->source->wanted = true;
queue_push(&queue, edge->source);
}
}
}
}
static bool is_ready(struct init_node* node) {
return node->wanted && !node->done && node->unsatisfied == 0;
}
static void reach_node(struct init_node* node, struct boot_info* info) {
if (node->disabled) {
log_status("Skipping stage \"%s\"\n", node->name);
} else {
log_status("Running stage \"%s\"\n", node->name);
failed_node = node;
switch (node->action) {
case INIT_EMPTY:
break;
case INIT_CALLBACK:
node->callback(info);
break;
case INIT_GATE:
if (!node->gate(info)) {
node->disabled = true;
node->failed = true;
}
break;
}
failed_node = nullptr;
node->executed = true;
}
node->done = true;
}
static void dump() {
kprintf("digraph initgraph {\n");
for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
struct init_node* node = *cur;
const char* style = "";
if (node->failed || node == failed_node)
style = ", color=red, fontcolor=red";
else if (!node->executed)
style = ", color=gray, fontcolor=gray";
kprintf("\tn%p [label=\"%s\"%s];\n", node, node->name, style);
for (struct init_edge* edge = node->in_edges; edge; edge = edge->next_in)
kprintf(
"\tn%p -> n%p%s;\n",
edge->source,
edge->target,
edge->source->executed && edge->target->executed ? "" : " [color=gray]"
);
}
kprintf("}\n");
}
void initgraph_panic() {
static bool dumping = false;
if (!dump_graph || !failed_node || dumping)
return;
dumping = true;
dump();
}
void initgraph_run(struct init_node* goal, struct boot_info* info) {
if (!edges_linked)
link_edges();
mark_wanted(goal);
struct init_queue pending = {};
for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
if (is_ready(*cur))
queue_push(&pending, *cur);
}
struct init_node* node;
while ((node = queue_pop(&pending))) {
reach_node(node, info);
for (struct init_edge* edge = node->out_edges; edge; edge = edge->next_out) {
struct init_node* successor = edge->target;
assert(successor->unsatisfied != 0, "Stage \"%s\" has no unsatisfied dependencies left", successor->name);
successor->unsatisfied--;
if (node->disabled && edge->propagates_disable)
successor->disabled = true;
if (is_ready(successor))
queue_push(&pending, successor);
}
}
for (auto cur = __ld_initgraph_nodes_start; cur < __ld_initgraph_nodes_end; cur++) {
if ((*cur)->wanted && !(*cur)->done)
failed_node = *cur;
assert(
!(*cur)->wanted || (*cur)->done,
"The dependencies for stage \"%s\" could not be resolved",
(*cur)->name
);
}
if (!goal) {
log_status("All stages are complete!\n");
if (dump_graph)
dump();
}
}
core/irq.c
-69
View File
@@ -1,69 +0,0 @@
#include <zinnia/assert.h>
#include <zinnia/irq.h>
#include <zinnia/list.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <arch/irq.h>
[[__percpu]]
struct irq_percpu irq_data;
bool irq_set_state(bool state) {
return arch_irq_set_state(state);
}
bool irq_get_state() {
return arch_irq_get_state();
}
void irq_wait() {
arch_irq_wait();
}
void irq_lock() {
bool prev = arch_irq_set_state(false);
struct irq_percpu* state = percpu_ref(irq_data);
if (state->depth++ == 0)
state->saved_state = prev;
}
void irq_unlock() {
struct irq_percpu* state = percpu_ref(irq_data);
if (--state->depth == 0 && state->saved_state)
arch_irq_set_state(true);
}
void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
irq_lock();
line->mode = line->set_config(line, trigger, polarity);
irq_unlock();
}
void irq_line_raise(struct irq_line* line) {
assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
irq_lock();
line->is_busy = true;
bool claimed = false;
struct irq_handler* handler;
list_foreach(handler, &line->handlers, next) {
if (handler->raise(handler) == IRQ_STATUS_HANDLED)
claimed = true;
}
if (!claimed)
log("irq: Spurious interrupt\n");
if ((line->mode & IRQ_MODE_EOI) && line->eoi)
line->eoi(line);
line->is_busy = false;
irq_unlock();
}
void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
irq_lock();
list_insert_head(&line->handlers, handler, next);
irq_unlock();
}
core/ksyms.c
-135
View File
@@ -1,135 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/compiler.h>
#include <zinnia/elf.h>
#include <zinnia/hashmap.h>
#include <zinnia/init.h>
#include <zinnia/ksyms.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <bits/elf.h>
#include <string.h>
[[__visibility("hidden")]]
extern const elf_sym_t __ld_dynsym_start[];
[[__visibility("hidden")]]
extern const elf_sym_t __ld_dynsym_end[];
[[__visibility("hidden")]]
extern const char __ld_dynstr_start[];
[[__visibility("hidden")]]
extern const char __ld_dynstr_end[];
[[__visibility("hidden")]]
extern const elf_dyn_t _DYNAMIC[];
static uintptr_t slide;
static bool slide_known;
static const char* kernel_name;
static hashmap(const char*, uintptr_t) symbols;
static uint64_t sym_hash(const void* key, size_t) {
const char* name = *(const char* const*)key;
return hashmap_hash_default(name, strlen(name));
}
static bool sym_eq(const void* a, const void* b, size_t) {
return !strcmp(*(const char* const*)a, *(const char* const*)b);
}
static size_t num_syms() {
return __ld_dynsym_end - __ld_dynsym_start;
}
static const char* dynstr(uint64_t offset) {
if (offset >= (size_t)(__ld_dynstr_end - __ld_dynstr_start))
return nullptr;
return &__ld_dynstr_start[offset];
}
static uintptr_t kernel_slide() {
if (slide_known)
return slide;
for (size_t i = 0; i < num_syms(); i++) {
const char* name = dynstr(__ld_dynsym_start[i].st_name);
if (name != nullptr && !strcmp(name, "ksym_lookup")) {
slide = (uintptr_t)ksym_lookup - __ld_dynsym_start[i].st_value;
break;
}
}
slide_known = true;
return slide;
}
void ksym_init() {
const uintptr_t base = kernel_slide();
for (const elf_dyn_t* dyn = _DYNAMIC; dyn->d_tag != DT_NULL; dyn++) {
if (dyn->d_tag == DT_SONAME)
kernel_name = dynstr(dyn->d_un.d_val);
}
hashmap_init(&symbols);
size_t count = 0;
for (size_t i = 0; i < num_syms(); i++) {
const elf_sym_t* sym = &__ld_dynsym_start[i];
if (sym->st_shndx == SHN_UNDEF)
continue;
const char* name = dynstr(sym->st_name);
if (name == nullptr || *name == 0)
continue;
const errno_t status = hashmap_insert(&symbols, name, sym->st_value + base, sym_hash, sym_eq);
assert(status == 0, "Failed to register kernel symbol \"%s\" (%i)", name, status);
count++;
}
log("ksyms: Registered %zu kernel symbols\n", count);
}
bool ksym_find(const char* name, uintptr_t* addr) {
uintptr_t* value = hashmap_get(&symbols, name, sym_hash, sym_eq);
if (value == nullptr)
return false;
*addr = *value;
return true;
}
const char* ksym_kernel_name() {
return kernel_name;
}
bool ksym_lookup(uintptr_t addr, const char** name, size_t* offset) {
addr -= kernel_slide();
for (size_t i = 0; i < num_syms(); i++) {
const elf_sym_t* sym = &__ld_dynsym_start[i];
if (ELF_ST_TYPE(sym->st_info) != STT_FUNC || sym->st_shndx == SHN_UNDEF)
continue;
if (addr >= sym->st_value && addr - sym->st_value < sym->st_size) {
const char* sym_name = dynstr(sym->st_name);
if (sym_name == nullptr)
return false;
*name = sym_name;
*offset = addr - sym->st_value;
return true;
}
}
return false;
}
void ksym_format(uintptr_t addr, char* buf, size_t size) {
const char* name;
size_t offset;
if (ksym_lookup(addr, &name, &offset))
ksnprintf(buf, size, "%s+0x%zx", name, offset);
else
ksnprintf(buf, size, "<\?\?\?>");
}
[[__init]]
static void ksyms_stage_init(struct boot_info*) {
ksym_init();
}
INIT_TASK(ksyms_stage, "core.ksyms", ksyms_stage_init);
INIT_DEPENDS(ksyms_stage, mem_stage);
core/lib/guid.c
-31
View File
@@ -1,31 +0,0 @@
#include <zinnia/guid.h>
#include <zinnia/print.h>
#include <string.h>
bool guid_is_zero(const struct guid* guid) {
static const struct guid zero = {};
return guid_equal(guid, &zero);
}
bool guid_equal(const struct guid* a, const struct guid* b) {
return memcmp(a, b, sizeof(struct guid)) == 0;
}
void guid_format(const struct guid* guid, char out[static GUID_STR_LEN]) {
ksnprintf(
out,
GUID_STR_LEN,
"%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x",
guid->time_low,
guid->time_mid,
guid->time_hi_version,
guid->clock_seq[0],
guid->clock_seq[1],
guid->node[0],
guid->node[1],
guid->node[2],
guid->node[3],
guid->node[4],
guid->node[5]
);
}
core/lib/iovec.c
-69
View File
@@ -1,69 +0,0 @@
#include <zinnia/iovec.h>
#include <zinnia/usercopy.h>
#include <zinnia/utils.h>
#include <string.h>
// Copies between `buf` and the iterator. If `to_iter` is set, data flows from `buf` into the iovecs.
static errno_t iovec_iter_copy(iovec_iter_t* iter, void* buf, size_t len, bool to_iter, size_t* out_copied) {
size_t progress = 0;
errno_t status = 0;
while (progress < len && iter->current_idx < iter->iovecs_len) {
iovec_t* vec = &iter->iovecs[iter->current_idx];
size_t remaining = vec->len - iter->current_offset;
if (remaining == 0) {
iter->current_idx++;
iter->current_offset = 0;
continue;
}
size_t chunk = MIN(remaining, len - progress);
void* ptr = (void*)(vec->base + iter->current_offset);
uint8_t* kbuf = (uint8_t*)buf + progress;
if (iter->user) {
bool ok = to_iter ? usercopy_write((__user void*)ptr, kbuf, chunk)
: usercopy_read(kbuf, (const __user void*)ptr, chunk);
if (!ok) {
status = EFAULT;
break;
}
} else if (to_iter) {
memcpy(ptr, kbuf, chunk);
} else {
memcpy(kbuf, ptr, chunk);
}
progress += chunk;
iter->current_offset += chunk;
iter->total_offset += chunk;
}
if (out_copied)
*out_copied = progress;
return status;
}
iovec_iter_t iovec_iter_kernel(iovec_t* iovecs, size_t count) {
size_t total = 0;
for (size_t i = 0; i < count; i++)
total += iovecs[i].len;
return (iovec_iter_t){
.iovecs = iovecs,
.iovecs_len = count,
.total_len = total,
.total_offset = 0,
.current_idx = 0,
.current_offset = 0,
.user = false,
};
}
errno_t iovec_iter_copy_to(iovec_iter_t* iter, const void* src, size_t len, size_t* out_copied) {
return iovec_iter_copy(iter, (void*)src, len, true, out_copied);
}
errno_t iovec_iter_copy_from(iovec_iter_t* iter, void* dst, size_t len, size_t* out_copied) {
return iovec_iter_copy(iter, dst, len, false, out_copied);
}
core/lib/meson.build
-11
View File
@@ -1,11 +0,0 @@
# Freestanding libc functions that are needed in the kernel
kernel_sources += files(
'guid.c',
'iovec.c',
'radix.c',
'ringbuf.c',
'runtime.c',
'stdlib.c',
'string.c',
)
core/lib/radix.c
-255
View File
@@ -1,255 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/radix.h>
static bool covers(const struct radix_node* node, uint64_t index) {
return node->shift + RADIX_BITS >= 64 || (index >> (node->shift + RADIX_BITS)) == 0;
}
static size_t slot_of(const struct radix_node* node, uint64_t index) {
return (index >> node->shift) & (RADIX_SLOTS - 1);
}
static struct radix_node* node_alloc(uint8_t shift, struct radix_node* parent, uint8_t offset) {
struct radix_node* node = mem_alloc(sizeof(struct radix_node), 0);
if (!node)
return nullptr;
node->shift = shift;
node->parent = parent;
node->offset = offset;
return node;
}
static void free_subtree(struct radix_node* node) {
if (node->shift > 0) {
for (size_t i = 0; i < RADIX_SLOTS; i++) {
if (node->slots[i])
free_subtree(node->slots[i]);
}
}
mem_free(node);
}
void radix_init(struct radix_tree* tree) {
tree->root = nullptr;
}
void radix_destroy(struct radix_tree* tree) {
if (tree->root)
free_subtree(tree->root);
tree->root = nullptr;
}
bool radix_empty(struct radix_tree* tree) {
return tree->root == nullptr;
}
static struct radix_node* leaf_of(struct radix_tree* tree, uint64_t index) {
struct radix_node* node = tree->root;
if (!node || !covers(node, index))
return nullptr;
while (node->shift > 0) {
node = node->slots[slot_of(node, index)];
if (!node)
return nullptr;
}
return node;
}
void* radix_lookup(struct radix_tree* tree, uint64_t index) {
struct radix_node* leaf = leaf_of(tree, index);
return leaf ? leaf->slots[slot_of(leaf, index)] : nullptr;
}
void** radix_slot(struct radix_tree* tree, uint64_t index) {
struct radix_node* leaf = leaf_of(tree, index);
return leaf ? &leaf->slots[slot_of(leaf, index)] : nullptr;
}
static errno_t grow(struct radix_tree* tree, uint64_t index) {
if (!tree->root) {
uint8_t shift = 0;
while (shift + RADIX_BITS < 64 && (index >> (shift + RADIX_BITS)) != 0)
shift += RADIX_BITS;
tree->root = node_alloc(shift, nullptr, 0);
return tree->root ? 0 : ENOMEM;
}
while (!covers(tree->root, index)) {
struct radix_node* old = tree->root;
struct radix_node* top = node_alloc(old->shift + RADIX_BITS, nullptr, 0);
if (!top)
return ENOMEM;
top->slots[0] = old;
top->count = 1;
for (size_t t = 0; t < RADIX_NUM_TAGS; t++) {
if (old->tags[t])
top->tags[t] = 1;
}
old->parent = top;
old->offset = 0;
tree->root = top;
}
return 0;
}
errno_t radix_insert(struct radix_tree* tree, uint64_t index, void* value) {
errno_t status = grow(tree, index);
if (status)
return status;
struct radix_node* node = tree->root;
while (node->shift > 0) {
const size_t slot = slot_of(node, index);
struct radix_node* child = node->slots[slot];
if (!child) {
child = node_alloc(node->shift - RADIX_BITS, node, slot);
if (!child)
return ENOMEM;
node->slots[slot] = child;
node->count++;
}
node = child;
}
const size_t slot = slot_of(node, index);
if (node->slots[slot])
return EEXIST;
node->slots[slot] = value;
node->count++;
return 0;
}
static void clear_tag_upwards(struct radix_node* node, size_t slot, enum radix_tag tag) {
while (node) {
node->tags[tag] &= ~((uint64_t)1 << slot);
if (node->tags[tag])
return;
slot = node->offset;
node = node->parent;
}
}
void* radix_remove(struct radix_tree* tree, uint64_t index) {
struct radix_node* node = leaf_of(tree, index);
if (!node)
return nullptr;
size_t slot = slot_of(node, index);
void* value = node->slots[slot];
if (!value)
return nullptr;
for (size_t t = 0; t < RADIX_NUM_TAGS; t++) {
if (node->tags[t] & ((uint64_t)1 << slot))
clear_tag_upwards(node, slot, t);
}
node->slots[slot] = nullptr;
node->count--;
while (node->count == 0) {
struct radix_node* parent = node->parent;
if (!parent) {
tree->root = nullptr;
mem_free(node);
break;
}
slot = node->offset;
parent->slots[slot] = nullptr;
parent->count--;
mem_free(node);
node = parent;
}
return value;
}
void radix_tag_set(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
struct radix_node* node = leaf_of(tree, index);
if (!node || !node->slots[slot_of(node, index)])
return;
size_t slot = slot_of(node, index);
while (node) {
const uint64_t bit = (uint64_t)1 << slot;
if (node->tags[tag] & bit)
return;
node->tags[tag] |= bit;
slot = node->offset;
node = node->parent;
}
}
void radix_tag_clear(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
struct radix_node* node = leaf_of(tree, index);
if (!node)
return;
const size_t slot = slot_of(node, index);
if (node->tags[tag] & ((uint64_t)1 << slot))
clear_tag_upwards(node, slot, tag);
}
bool radix_tag_get(struct radix_tree* tree, uint64_t index, enum radix_tag tag) {
struct radix_node* node = leaf_of(tree, index);
return node && (node->tags[tag] & ((uint64_t)1 << slot_of(node, index)));
}
bool radix_tagged(struct radix_tree* tree, enum radix_tag tag) {
return tree->root && tree->root->tags[tag];
}
struct gang {
size_t max;
size_t found;
void** out;
uint64_t* out_index;
int tag;
};
static void gang_walk(struct radix_node* node, uint64_t base, uint64_t first, struct gang* g) {
size_t start = 0;
if (first > base) {
const uint64_t delta = (first - base) >> node->shift;
if (delta >= RADIX_SLOTS)
return;
start = delta;
}
for (size_t i = start; i < RADIX_SLOTS && g->found < g->max; i++) {
if (!node->slots[i])
continue;
if (g->tag >= 0 && !(node->tags[g->tag] & ((uint64_t)1 << i)))
continue;
const uint64_t index = base + ((uint64_t)i << node->shift);
if (node->shift == 0) {
if (g->out)
g->out[g->found] = node->slots[i];
if (g->out_index)
g->out_index[g->found] = index;
g->found++;
} else {
gang_walk(node->slots[i], index, first > index ? first : index, g);
}
}
}
static size_t gang_lookup(struct radix_tree* tree, uint64_t first, int tag, size_t max, void** out, uint64_t* idx) {
struct radix_node* root = tree->root;
if (!root || max == 0 || !covers(root, first))
return 0;
struct gang g = {.max = max, .out = out, .out_index = idx, .tag = tag};
gang_walk(root, 0, first, &g);
return g.found;
}
size_t radix_gang_lookup(struct radix_tree* tree, uint64_t first, size_t max, void** out, uint64_t* out_index) {
return gang_lookup(tree, first, -1, max, out, out_index);
}
size_t radix_gang_lookup_tag(
struct radix_tree* tree,
uint64_t first,
enum radix_tag tag,
size_t max,
void** out,
uint64_t* out_index
) {
return gang_lookup(tree, first, tag, max, out, out_index);
}
core/lib/ringbuf.c
-94
View File
@@ -1,94 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/ringbuf.h>
#include <zinnia/utils.h>
#include <string.h>
errno_t ringbuf_init(struct ringbuf* rb, size_t capacity) {
rb->data = mem_alloc(capacity, ALLOC_NOZERO);
if (!rb->data)
return ENOMEM;
rb->capacity = capacity;
rb->head = 0;
rb->len = 0;
return 0;
}
void ringbuf_fini(struct ringbuf* rb) {
mem_free(rb->data);
rb->data = nullptr;
rb->capacity = 0;
rb->len = 0;
}
static size_t tail(const struct ringbuf* rb) {
return (rb->head + rb->len) % rb->capacity;
}
static size_t write_chunk(const struct ringbuf* rb, size_t len) {
return MIN(len, MIN(ringbuf_space(rb), rb->capacity - tail(rb)));
}
static size_t read_chunk(const struct ringbuf* rb, size_t len) {
return MIN(len, MIN(rb->len, rb->capacity - rb->head));
}
static void advance_read(struct ringbuf* rb, size_t len) {
rb->head = (rb->head + len) % rb->capacity;
rb->len -= len;
if (rb->len == 0)
rb->head = 0;
}
size_t ringbuf_write(struct ringbuf* rb, const void* src, size_t len) {
size_t done = 0;
size_t chunk;
while ((chunk = write_chunk(rb, len - done))) {
memcpy(rb->data + tail(rb), (const uint8_t*)src + done, chunk);
rb->len += chunk;
done += chunk;
}
return done;
}
size_t ringbuf_read(struct ringbuf* rb, void* dst, size_t len) {
size_t done = 0;
size_t chunk;
while ((chunk = read_chunk(rb, len - done))) {
memcpy((uint8_t*)dst + done, rb->data + rb->head, chunk);
advance_read(rb, chunk);
done += chunk;
}
return done;
}
errno_t ringbuf_write_iter(struct ringbuf* rb, iovec_iter_t* iter, size_t len, size_t* out_written) {
size_t done = 0;
errno_t status = 0;
size_t chunk;
while (!status && (chunk = write_chunk(rb, len - done))) {
size_t copied;
status = iovec_iter_copy_from(iter, rb->data + tail(rb), chunk, &copied);
rb->len += copied;
done += copied;
if (copied < chunk)
break;
}
*out_written = done;
return done ? 0 : status;
}
errno_t ringbuf_read_iter(struct ringbuf* rb, iovec_iter_t* iter, size_t len, size_t* out_read) {
size_t done = 0;
errno_t status = 0;
size_t chunk;
while (!status && (chunk = read_chunk(rb, len - done))) {
size_t copied;
status = iovec_iter_copy_to(iter, rb->data + rb->head, chunk, &copied);
advance_read(rb, copied);
done += copied;
if (copied < chunk)
break;
}
*out_read = done;
return done ? 0 : status;
}
core/lib/runtime.c
-93
View File
@@ -1,93 +0,0 @@
// Compiler runtime support.
#include <stdint.h>
uint64_t __udivdi3(uint64_t dividend, uint64_t divisor) {
if (divisor == 0)
return 0;
if (dividend < divisor)
return 0;
unsigned shift = __builtin_clzll(divisor) - __builtin_clzll(dividend);
divisor <<= shift;
uint64_t quotient = 0;
for (;;) {
if (dividend >= divisor) {
dividend -= divisor;
quotient |= (uint64_t)1 << shift;
}
if (shift == 0)
break;
shift--;
divisor >>= 1;
}
return quotient;
}
static unsigned clz128(__uint128_t value) {
uint64_t high = (uint64_t)(value >> 64);
if (high != 0)
return (unsigned)__builtin_clzll(high);
return 64 + (unsigned)__builtin_clzll((uint64_t)value);
}
__uint128_t __udivmodti4(__uint128_t dividend, __uint128_t divisor, __uint128_t* remainder) {
if (divisor == 0) {
if (remainder)
*remainder = dividend;
return 0;
}
if (dividend < divisor) {
if (remainder)
*remainder = dividend;
return 0;
}
unsigned shift = clz128(divisor) - clz128(dividend);
divisor <<= shift;
__uint128_t quotient = 0;
for (;;) {
if (dividend >= divisor) {
dividend -= divisor;
quotient |= (__uint128_t)1 << shift;
}
if (shift == 0)
break;
shift--;
divisor >>= 1;
}
if (remainder)
*remainder = dividend;
return quotient;
}
__uint128_t __udivti3(__uint128_t dividend, __uint128_t divisor) {
return __udivmodti4(dividend, divisor, nullptr);
}
__uint128_t __umodti3(__uint128_t dividend, __uint128_t divisor) {
__uint128_t remainder;
__udivmodti4(dividend, divisor, &remainder);
return remainder;
}
core/lib/stdlib.c
-102
View File
@@ -1,102 +0,0 @@
#include <zinnia/stdlib.h>
#include <stdint.h>
#include <string.h>
static void reverse(char* s) {
int i, j;
char c;
for (i = 0, j = strlen(s) - 1; i < j; i++, j--) {
c = s[i];
s[i] = s[j];
s[j] = c;
}
}
static int char_to_digit(char c) {
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'a' && c <= 'z')
return c - 'a' + 10;
if (c >= 'A' && c <= 'Z')
return c - 'A' + 10;
return -1;
}
unsigned long atolu(const char* str, int base) {
unsigned long result = 0;
if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
str += 2;
}
while (*str) {
int digit = char_to_digit(*str++);
if (digit < 0 || digit >= base)
break;
result = result * base + digit;
}
return result;
}
long atol(const char* str, int base) {
int neg = 0;
if (*str == '-') {
neg = 1;
str++;
}
unsigned long val = atolu(str, base);
return neg ? -(long)val : (long)val;
}
char* lutoa(uint64_t value, char* str, int base) {
if (base < 2 || base > 36)
return nullptr;
size_t i = 0;
do {
uint64_t digit = value % base;
str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
value /= base;
} while (value);
str[i] = '\0';
reverse(str);
return str;
}
char* ltoa(int64_t value, char* str, int base) {
if (base < 2 || base > 36)
return nullptr;
uint64_t magnitude;
int negative = 0;
if (value < 0) {
negative = 1;
magnitude = (uint64_t)(-(value + 1)) + 1;
// avoids INT64_MIN overflow
} else {
magnitude = (uint64_t)value;
}
size_t i = 0;
do {
uint64_t digit = magnitude % base;
str[i++] = (digit < 10) ? '0' + digit : 'a' + (digit - 10);
magnitude /= base;
} while (magnitude);
if (negative)
str[i++] = '-';
str[i] = '\0';
reverse(str);
return str;
}
core/mem/Kconfig
-8
View File
@@ -1,8 +0,0 @@
menu "Memory Management options"
config HAVE_MMU
bool
config HAVE_MMU_GENERIC
bool
endmenu
core/mem/init.c
-178
View File
@@ -1,178 +0,0 @@
#define pr_fmt(fmt) "mem: " fmt
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/console.h>
#include <zinnia/init.h>
#include <zinnia/panic.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
extern uint8_t __ld_text_start[];
extern uint8_t __ld_text_end[];
extern uint8_t __ld_rodata_start[];
extern uint8_t __ld_rodata_end[];
extern uint8_t __ld_data_start[];
extern uint8_t __ld_data_end[];
extern uint8_t __ld_kernel_start[];
struct vma kernel_vma = {0};
[[__init]]
void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) {
// This function creates a kernel page table and initializes all memory managers.
const size_t pgsz = arch_mem_page_size();
log("Memory map:\n");
for (size_t i = 0; i < map_len; i++) {
if (map[i].length == 0)
continue;
const char* label = nullptr;
switch (map[i].usage) {
case PHYS_RESERVED:
label = "Reserved";
break;
case PHYS_USABLE:
label = "Usable";
break;
case PHYS_STATIC:
label = "Static";
break;
case PHYS_RECLAIMABLE:
label = "Reclaimable";
break;
}
log("[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
}
// Set up the bootstrap allocator.
struct phys_mem* largest = nullptr;
for (size_t i = 0; i < map_len; i++) {
if (map[i].usage == PHYS_USABLE && (!largest || map[i].length > largest->length))
largest = &map[i];
}
assert(largest && largest->length >= pgsz, "No usable memory for bootstrap allocator");
mem_phys_bootstrap(largest);
log("Using region [%p - %p] for bootstrap allocator\n",
(void*)largest->address,
(void*)(largest->address + largest->length - 1));
// Set the HHDM base address to the address given by the loader.
// We must not keep any virtual addresses to this region,
// since we're likely going to map it at a different base address.
mem_hhdm_base = tmp_hhdm;
assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table");
tailq_init(&kernel_vma.mappings);
// text
log("Mapping text segment at %p\n", __ld_text_start);
for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vma.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ | PTE_EXEC,
CACHE_ANY
);
assert(!status, "Failed to map %p with error %i", p, status);
}
// rodata
log("Mapping rodata segment at %p\n", __ld_rodata_start);
for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vma.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ,
CACHE_ANY
);
assert(!status, "Failed to map %p with error %i", p, status);
}
// data
log("Mapping data segment at %p\n", __ld_data_start);
for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vma.pmap,
(uintptr_t)p,
(phys_t)(p - __ld_kernel_start + kernel_phys),
PTE_READ | PTE_WRITE,
CACHE_ANY
);
assert(!status, "Failed to map %p with error %i", p, status);
}
// The kernel address space is divided into 4 segments.
// For example, they could live at:
// HHDM FFFF'8000'0000'0000
// PFNDB FFFF'A000'0000'0000
// Mappings FFFF'C000'0000'0000
// Kernel FFFF'FFFF'8000'0000
// Map all physical memory to the HHDM address.
tmp_hhdm = arch_mem_hhdm_addr();
for (size_t i = 0; i < map_len; i++) {
if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC && map[i].usage != PHYS_RECLAIMABLE)
continue;
for (size_t p = 0; p < map[i].length; p += pgsz) {
uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
phys_t paddr = (phys_t)(map[i].address + p);
errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
assert(!status, "Failed to map HHDM page %p to %p with error %i", (void*)vaddr, (void*)paddr, status);
}
}
mem_hhdm_base = tmp_hhdm;
// Switch to our own page table.
pmap_set(&kernel_vma.pmap);
// We record metadata for every single page of available memory in a large array.
// This array is contiguous in virtual memory, but is sparsely populated.
// Only those array entries which represent usable memory are mapped.
for (size_t i = 0; i < map_len; i++) {
if (map[i].length == 0 || map[i].usage != PHYS_USABLE)
continue;
const size_t num_pages = (map[i].length + pgsz - 1) / pgsz;
const size_t metadata_size = num_pages * sizeof(struct page);
const uintptr_t metadata_start_vaddr = arch_mem_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size;
// Find the page-aligned boundaries that contain this metadata.
const uintptr_t aligned_start = ALIGN_DOWN(metadata_start_vaddr, pgsz);
const uintptr_t aligned_end = ALIGN_UP(metadata_end_vaddr, pgsz);
const size_t length = aligned_end - aligned_start;
const uintptr_t vaddr = aligned_start;
for (size_t page = 0; page < length; page += pgsz) {
phys_t paddr;
assert(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!");
assert(
!pmap_map(&kernel_vma.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_ANY),
"Failed to map PFNDB memory"
);
}
}
vm_pfndb = (struct page*)arch_mem_pfndb_addr();
// We don't need the bootstrap allocator from this point on.
// Initialize the real page allocator.
mem_phys_init(map, map_len);
slab_init();
log("Memory initialization complete\n");
}
[[__init]]
static void mem_stage_init(struct boot_info* info) {
mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
}
INIT_TASK(mem_stage, "core.mem", mem_stage_init);
INIT_DEPENDS(mem_stage, console_stage);
core/mem/meson.build
-17
View File
@@ -1,17 +0,0 @@
kernel_sources += files(
'fault.c',
'init.c',
'pmm.c',
'slab.c',
'stack.c',
'usercopy.c',
'virt.c',
'vma.c',
'vmo.c',
)
if kconfig.get('CONFIG_HAVE_MMU', 'n') == 'y'
kernel_sources += files(
'pmap.c',
)
endif
core/mem/pmap.c
-239
View File
@@ -1,239 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/spin.h>
#include <zinnia/vma.h>
#include <arch/cpu.h>
#include <config.h>
// TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
static_assert(CONFIG_HAVE_MMU_GENERIC);
errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
flags &= ~ALLOC_NOZERO;
phys_t addr;
errno_t status = mem_phys_alloc(1, flags, &addr);
if (status)
return status;
pte_t* root = HHDM_PTR(addr);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = num_entries / 2; i < num_entries; i++) {
phys_t dir;
status = mem_phys_alloc(1, flags, &dir);
if (status)
return status;
root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
}
struct pmap result = {
.root = addr,
.lock = (struct spinlock){0},
.is_user = false,
};
*pt = result;
return 0;
}
errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
phys_t user_l1;
errno_t status = mem_phys_alloc(1, 0, &user_l1);
if (status)
return status;
void* user_l1_ptr = HHDM_PTR(user_l1);
void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
struct pmap result = {
.root = user_l1,
.lock = (struct spinlock){0},
.is_user = true,
};
*pt = result;
return 0;
}
// Gets a reference to the PTE at the given virtual address.
// If `check_only` is set, only checks if the PTE exists,
// and doesn't allocate new levels if they don't already exist.
// If it can't allocate a page if it has to, returns `nullptr`.
static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) {
pte_t* current_head = HHDM_PTR(pt->root);
size_t index = 0;
for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) {
const size_t addr_mask = (1 << arch_mem_level_bits()) - 1;
const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level);
const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0);
index = (vaddr >> addr_shift) & addr_mask;
// The last level is used to access the actual PTE, so break the loop then.
// We still need to update the index beforehand, that's why we can't just end early.
if (level == 0)
break;
pte_t* pte = &current_head[index];
if (pte_is_present(pte)) {
// Get the next level.
*pte |= pte_build(0, level_flags, CACHE_ANY);
current_head = HHDM_PTR(pte_address(pte));
} else {
// If the current level isn't present, we can skip the rest.
if (check_only)
return nullptr;
phys_t addr;
if (mem_phys_alloc(1, 0, &addr) != 0)
return nullptr;
*pte = pte_build(addr, level_flags, CACHE_ANY);
current_head = HHDM_PTR(addr);
}
}
return &current_head[index];
}
errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
spin_lock(&pt->lock);
errno_t status = 0;
pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
if (pte == nullptr) {
status = ENOMEM;
goto fail;
}
*pte = pte_build(paddr, flags, cache);
fail:
spin_unlock(&pt->lock);
return status;
}
errno_t pmap_unmap(struct pmap* pt, uintptr_t vaddr) {
spin_lock(&pt->lock);
errno_t status = 0;
pte_t* pte = get_pte(pt, vaddr, false, true);
if (pte == nullptr || !pte_is_present(pte)) {
status = ENOENT;
goto fail;
}
pte_clear(pte);
pte_flush(vaddr);
fail:
spin_unlock(&pt->lock);
return status;
}
errno_t pmap_extract(struct pmap* pt, uintptr_t vaddr, phys_t* out) {
spin_lock(&pt->lock);
errno_t status = 0;
pte_t* pte = get_pte(pt, vaddr, false, true);
if (pte == nullptr || !pte_is_present(pte))
status = ENOENT;
else
*out = pte_address(pte);
spin_unlock(&pt->lock);
return status;
}
errno_t pmap_protect(struct pmap* pt, uintptr_t vaddr, enum pte_flags flags) {
spin_lock(&pt->lock);
errno_t status = 0;
pte_t* pte = get_pte(pt, vaddr, false, true);
if (pte == nullptr || !pte_is_present(pte)) {
status = ENOENT;
goto fail;
}
*pte = pte_build(pte_address(pte), flags, CACHE_ANY);
pte_flush(vaddr);
fail:
spin_unlock(&pt->lock);
return status;
}
static void free_table(phys_t table, int8_t level) {
if (level > 0) {
pte_t* entries = HHDM_PTR(table);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = 0; i < num_entries; i++) {
if (pte_is_present(&entries[i]))
free_table(pte_address(&entries[i]), level - 1);
}
}
mem_phys_free(table, 1);
}
void pmap_destroy(struct pmap* pt) {
if (!pt->is_user)
return;
pte_t* root = HHDM_PTR(pt->root);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = 0; i < num_entries / 2; i++) {
if (pte_is_present(&root[i]))
free_table(pte_address(&root[i]), arch_mem_num_levels() - 2);
}
mem_phys_free(pt->root, 1);
pt->root = 0;
}
[[__percpu]]
static atomic_size_t shootdown_req;
[[__percpu]]
static atomic_size_t shootdown_ack;
void pmap_shootdown_handle() {
const size_t req = atomic_load(percpu_ref(shootdown_req), ATOMIC_ACQUIRE);
if (atomic_load(percpu_ref(shootdown_ack), ATOMIC_RELAXED) == req)
return;
pte_flush_all();
atomic_store(percpu_ref(shootdown_ack), req, ATOMIC_RELEASE);
}
void pmap_shootdown(struct pmap* pt) {
const size_t count = percpu_count();
if (count < 2)
return;
irq_lock();
struct percpu* self = percpu_get();
for (size_t i = 0; i < count; i++) {
struct percpu* cpu = percpu_cpu(i);
if (cpu == self || !cpu->online)
continue;
atomic_fetch_add(percpu_ref_on(cpu, shootdown_req), 1, ATOMIC_RELEASE);
pte_shootdown_kick(cpu);
}
for (size_t i = 0; i < count; i++) {
struct percpu* cpu = percpu_cpu(i);
if (cpu == self || !cpu->online)
continue;
const size_t req = atomic_load(percpu_ref_on(cpu, shootdown_req), ATOMIC_RELAXED);
while ((ptrdiff_t)(req - atomic_load(percpu_ref_on(cpu, shootdown_ack), ATOMIC_ACQUIRE)) > 0) {
pmap_shootdown_handle();
arch_cpu_pause();
}
}
irq_unlock();
}
core/mem/pmm.c
-193
View File
@@ -1,193 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/compiler.h>
#include <zinnia/panic.h>
#include <zinnia/print.h>
#include <zinnia/spin.h>
#include <zinnia/virt.h>
#include <stdint.h>
uintptr_t mem_hhdm_base = 0;
static errno_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
static errno_t (*free_fn)(phys_t start, size_t num_pages);
errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
return alloc_fn(num_pages, flags, out);
}
errno_t mem_phys_free(phys_t start, size_t num_pages) {
return free_fn(start, num_pages);
}
static struct phys_mem bump_mem = {0};
static struct phys_mem* bump_region = nullptr;
static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
const size_t bytes = num_pages * arch_mem_page_size();
if (bytes > bump_mem.length)
return ENOMEM;
*out = bump_mem.address;
bump_mem.length -= bytes;
bump_mem.address += bytes;
if (!(flags & ALLOC_NOZERO)) {
memset(HHDM_PTR(*out), 0, bytes);
}
return 0;
}
static errno_t bump_free(phys_t addr, size_t num_pages) {
// We don't free bootstrap memory.
panic("Attempted to free bootstrap memory!");
}
void mem_phys_bootstrap(struct phys_mem* mem) {
alloc_fn = bump_alloc;
free_fn = bump_free;
// Save the region by reference and value.
// We can't directly modify the region by reference, because that
// interferes with e.g. HHDM mapping logic.
bump_mem = *mem;
bump_region = mem;
}
static struct page* pmm_head = nullptr;
static struct spinlock pmm_lock = {0};
static size_t pmm_total_free = 0;
static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
spin_lock(&pmm_lock);
const size_t bytes = num_pages * arch_mem_page_size();
phys_t limit = UINTPTR_MAX;
if (flags & ALLOC_MEM20) {
limit = (phys_t)1 << 20;
} else if (flags & ALLOC_MEM32) {
limit = (phys_t)1 << 32;
}
struct page* it = pmm_head;
struct page* prev_it = nullptr;
while (it) {
const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
// If the region is too high up or doesn't have enough pages left, continue searching.
if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) {
prev_it = it;
it = it->freelist.next;
continue;
}
// If the current block has exactly enough pages left, consume it entirely.
if (__unlikely(it->freelist.count == num_pages)) {
if (prev_it)
prev_it->freelist.next = it->freelist.next;
else
pmm_head = it->freelist.next;
it->freelist.next = nullptr;
it->freelist.count = 0;
if (__likely(out))
*out = page_addr;
}
// If not, we can just shrink the block.
else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) {
it->freelist.count -= num_pages;
if (__likely(out))
*out = page_addr + (it->freelist.count * arch_mem_page_size());
} else {
struct page* rest = it + num_pages;
rest->freelist.count = it->freelist.count - num_pages;
rest->freelist.next = it->freelist.next;
if (prev_it)
prev_it->freelist.next = rest;
else
pmm_head = rest;
it->freelist.next = nullptr;
it->freelist.count = 0;
if (__likely(out))
*out = page_addr;
}
goto success;
}
spin_unlock(&pmm_lock);
return ENOMEM;
success:
// If the NOZERO flag is *not* specified, zero memory.
if (__likely(out) && __likely(!(flags & ALLOC_NOZERO))) {
memset(HHDM_PTR(*out), 0, bytes);
}
pmm_total_free -= num_pages;
spin_unlock(&pmm_lock);
return 0;
}
static void freelist_insert(struct page* page, size_t num_pages) {
struct page* prev = nullptr;
struct page* next = pmm_head;
while (next && next < page) {
prev = next;
next = next->freelist.next;
}
page->freelist.count = num_pages;
page->freelist.next = next;
if (next && page + page->freelist.count == next) {
page->freelist.count += next->freelist.count;
page->freelist.next = next->freelist.next;
next->freelist.next = nullptr;
next->freelist.count = 0;
}
if (prev && prev + prev->freelist.count == page) {
prev->freelist.count += page->freelist.count;
prev->freelist.next = page->freelist.next;
page->freelist.next = nullptr;
page->freelist.count = 0;
} else if (prev) {
prev->freelist.next = page;
} else {
pmm_head = page;
}
}
static errno_t freelist_free(phys_t addr, size_t num_pages) {
spin_lock(&pmm_lock);
freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages);
pmm_total_free += num_pages;
spin_unlock(&pmm_lock);
return 0;
}
void mem_phys_init(struct phys_mem* map, size_t length) {
alloc_fn = freelist_alloc;
free_fn = freelist_free;
bump_region->address = bump_mem.address;
bump_region->length = bump_mem.length;
for (size_t i = 0; i < length; i++) {
// Regions smaller than a page are useless.
if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
continue;
const size_t num_pages = map[i].length / arch_mem_page_size();
freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages);
pmm_total_free += num_pages;
}
const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
log("pmm: Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
}
core/mem/slab.c
-143
View File
@@ -1,143 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/spin.h>
#include <string.h>
typedef struct {
size_t num_pages; // Amount of pages connected to this slab.
size_t size; // Size of this slab.
} slab_info;
struct slab {
struct spinlock lock; // Access lock.
size_t ent_size; // Size of one entry.
void** head;
};
typedef struct {
struct slab* slab;
} slab_header;
// Initializes the SLAB structures.
static struct slab slabs[8] = {0};
static errno_t slab_new(struct slab* slab, size_t size) {
// Allocate a new page for the head.
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
if (status)
return status;
slab->head = (void**)(addr + arch_mem_hhdm_addr());
slab->ent_size = size;
const size_t offset = ALIGN_UP(sizeof(slab_header), size);
const size_t available_size = arch_mem_page_size() - offset;
slab_header* ptr = (slab_header*)slab->head;
ptr->slab = slab;
slab->head = (void**)((void*)slab->head + offset);
void** arr = slab->head;
const size_t max = available_size / size - 1;
const size_t fact = size / sizeof(void*);
for (size_t i = 0; i < max; i++) {
arr[i * fact] = &arr[(i + 1) * fact];
}
arr[max * fact] = nullptr;
return 0;
}
void slab_init(void) {
slab_new(&slabs[0], 16);
slab_new(&slabs[1], 32);
slab_new(&slabs[2], 64);
slab_new(&slabs[3], 128);
slab_new(&slabs[4], 256);
slab_new(&slabs[5], 512);
slab_new(&slabs[6], 1024);
slab_new(&slabs[7], 2048);
}
static void* slab_do_alloc(struct slab* slab) {
spin_lock(&slab->lock);
if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) {
spin_unlock(&slab->lock);
return nullptr;
}
void** old_free = slab->head;
slab->head = *old_free;
memset(old_free, 0, slab->ent_size);
spin_unlock(&slab->lock);
return old_free;
}
static void slab_do_free(struct slab* slab, void* addr) {
spin_lock(&slab->lock);
if (__unlikely(addr == nullptr))
goto cleanup;
void** new_head = addr;
*new_head = slab->head;
slab->head = new_head;
cleanup:
spin_unlock(&slab->lock);
}
static inline struct slab* slab_find_size(size_t size) {
for (size_t i = 0; i < ARRAY_SIZE(slabs); i++) {
if (slabs[i].ent_size >= size)
return &slabs[i];
}
return nullptr;
}
void* mem_alloc(size_t size, enum alloc_flags flags) {
if (__unlikely(size == 0))
return nullptr;
struct slab* slab = slab_find_size(size);
if (slab != nullptr) {
return slab_do_alloc(slab);
}
size_t num_pages = ROUND_UP(size, arch_mem_page_size());
// Allocate the pages plus an additional page for metadata.
phys_t ret;
errno_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
if (__unlikely(status != 0))
return nullptr;
ret = ret + (phys_t)arch_mem_hhdm_addr();
// Write metadata into the first page.
slab_info* info = (slab_info*)ret;
info->num_pages = num_pages;
info->size = size;
// Skip the first page and return the next one.
return (void*)(ret + arch_mem_page_size());
}
void mem_free(void* addr) {
if (__unlikely(addr == nullptr))
return;
// If the address is page aligned.
if ((size_t)addr == ALIGN_DOWN((size_t)addr, arch_mem_page_size())) {
slab_info* info = (slab_info*)(addr - arch_mem_page_size());
mem_phys_free(((phys_t)info - (phys_t)arch_mem_hhdm_addr()), info->num_pages + 1);
return;
}
slab_header* header = (slab_header*)(ALIGN_DOWN((size_t)addr, arch_mem_page_size()));
slab_do_free(header->slab, addr);
}
core/mem/stack.c
-61
View File
@@ -1,61 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/errno.h>
#include <zinnia/pmap.h>
#include <zinnia/stack.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <zinnia/vma.h>
#include <arch/mem.h>
errno_t stack_init(struct stack* ks, size_t size) {
if (!ks || size == 0)
return EINVAL;
const size_t page_size = arch_mem_page_size();
size = ALIGN_UP(size, page_size);
uintptr_t va = vm_alloc(size + page_size);
if (!va)
return ENOMEM;
phys_t phys;
errno_t status = mem_phys_alloc(size / page_size, 0, &phys);
if (status)
goto err_vm;
const uintptr_t base = va + page_size;
size_t mapped = 0;
for (; mapped < size; mapped += page_size) {
status = pmap_map(&kernel_vma.pmap, base + mapped, phys + mapped, PTE_READ | PTE_WRITE, CACHE_WRITE_BACK);
if (status)
goto err_map;
}
ks->base = base;
ks->phys = phys;
ks->size = size;
return 0;
err_map:
for (size_t i = 0; i < mapped; i += page_size)
pmap_unmap(&kernel_vma.pmap, base + i);
mem_phys_free(phys, size / page_size);
err_vm:
vm_free(va, size + page_size);
return status;
}
void stack_destroy(struct stack* ks) {
if (!ks || !ks->base)
return;
const size_t page_size = arch_mem_page_size();
for (size_t i = 0; i < ks->size; i += page_size)
pmap_unmap(&kernel_vma.pmap, ks->base + i);
pmap_shootdown(&kernel_vma.pmap);
mem_phys_free(ks->phys, ks->size / page_size);
vm_free(ks->base - page_size, ks->size + page_size);
*ks = (struct stack){0};
}
core/mem/usercopy.c
-44
View File
@@ -1,44 +0,0 @@
#include <zinnia/percpu.h>
#include <zinnia/usercopy.h>
#include <arch/mem.h>
#include <arch/usercopy.h>
static bool is_user_range(uintptr_t addr, size_t len) {
if (len == 0)
return arch_mem_addr_is_user(addr);
return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1);
}
bool usercopy_read(void* dst, const __user void* src, size_t len) {
if (!is_user_range((uintptr_t)src, len))
return false;
arch_usercopy_allow(true);
bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
return success;
}
bool usercopy_write(__user void* dst, const void* src, size_t len) {
if (!is_user_range((uintptr_t)dst, len))
return false;
arch_usercopy_allow(true);
bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
return success;
}
bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
if (!arch_mem_addr_is_user((uintptr_t)str))
return false;
size_t unused;
if (len == nullptr)
len = &unused;
arch_usercopy_allow(true);
bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
return success;
}
core/mem/virt.c
-26
View File
@@ -1,26 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <stddef.h>
struct page* vm_pfndb = nullptr;
// TODO: Replace with actual allocator.
static _Atomic(size_t) mapping_offset = 0;
uintptr_t vm_alloc(size_t length) {
uintptr_t base = arch_mem_mapping_addr();
size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE);
return base + offset;
}
void vm_free(uintptr_t ptr, size_t length) {
// TODO
}
uintptr_t vm_page_addr(struct page* page) {
return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
}
core/mem/vma.c
-158
View File
@@ -1,158 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/errno.h>
#include <zinnia/mman.h>
#include <zinnia/pmap.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <stdckdint.h>
// Try to create a new, empty virtual address space.
errno_t vma_new(struct vma** out) {
debug_assert(out != nullptr, "Missing out");
errno_t res;
if (!out)
return EINVAL;
struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
if (!vmspace)
return ENOMEM;
res = pmap_new_user(&vmspace->pmap, 0);
if (res)
goto err1;
tailq_init(&vmspace->mappings);
*out = vmspace;
return 0;
err1:
mem_free(vmspace);
err0:
return res;
}
// Delete a virtual address space.
void vma_free(struct vma* v) {
struct vma_mapping* mapping;
while ((mapping = tailq_first(&v->mappings)) != nullptr) {
tailq_remove(&v->mappings, mapping, link);
vmo_unref(mapping->object);
mem_free(mapping);
}
}
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
panic("TODO");
}
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
panic("TODO");
}
static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (page >= m->start_page && page < m->end_page)
return m;
}
return nullptr;
}
static bool can_access(enum prot_flags prot, const struct vma_fault_info* info) {
if (info->caused_by_write)
return prot & PROT_WRITE;
if (info->caused_by_fetch)
return prot & PROT_EXEC;
return prot & (PROT_READ | PROT_WRITE);
}
static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
enum pte_flags pf = 0;
if (prot & PROT_READ)
pf |= PTE_READ;
if (prot & PROT_WRITE)
pf |= PTE_WRITE;
if (prot & PROT_EXEC)
pf |= PTE_EXEC;
if (v->pmap.is_user)
pf |= PTE_USER;
return pf;
}
bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
const size_t page_size = arch_mem_page_size();
const size_t faulty_page = info->addr / page_size;
spin_lock(&v->lock);
struct vma_mapping* mapping = find_mapping(v, faulty_page);
if (!mapping || !can_access(mapping->prot, info)) {
spin_unlock(&v->lock);
return false;
}
struct vmo* object = mapping->object;
const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
const enum prot_flags prot = mapping->prot;
vmo_ref(object);
spin_unlock(&v->lock);
struct page* page;
if (object->get_page(object, object_page, &page)) {
vmo_unref(object);
return false;
}
spin_lock(&v->lock);
mapping = find_mapping(v, faulty_page);
const bool still_valid = mapping && mapping->object == object &&
(faulty_page - mapping->start_page) + mapping->offset_page == object_page && can_access(mapping->prot, info);
if (still_valid) {
errno_t status =
pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
assert(status == 0, "Failed to map a demand-loaded page");
}
spin_unlock(&v->lock);
vmo_unref(object);
return true;
}
errno_t vma_map(
struct vma* vma,
struct vmo* vmo,
uintptr_t addr,
size_t len,
enum prot_flags flags,
uintptr_t vmo_offset
) {
if (!vma || !vmo)
return EINVAL;
uintptr_t end;
if (ckd_add(&end, addr, len))
return ENOMEM;
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
if (!mapping)
return ENOMEM;
vmo_ref(vmo);
mapping->start_page = addr / page_size;
mapping->end_page = ALIGN_UP(end, page_size) / page_size;
mapping->offset_page = vmo_offset / page_size;
mapping->object = vmo;
mapping->prot = flags;
spin_lock(&vma->lock);
tailq_insert_head(&vma->mappings, mapping, link);
spin_unlock(&vma->lock);
return 0;
}
core/mem/vmo.c
-263
View File
@@ -1,263 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/errno.h>
#include <zinnia/list.h>
#include <zinnia/utils.h>
#include <zinnia/vmo.h>
void vmo_ref(struct vmo* obj) {
refcount_inc(&obj->rc);
}
void vmo_unref(struct vmo* obj) {
if (refcount_dec(&obj->rc))
obj->destroy(obj);
}
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t misalign = (progress + offset) % page_size;
const size_t page_index = (progress + offset) / page_size;
const size_t copy_size = MIN(page_size - misalign, len - progress);
struct page* p;
status = obj->get_page(obj, page_index, &p);
if (status != 0)
break;
uintptr_t page_addr = vm_page_addr(p);
memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
progress += copy_size;
}
if (actual_read)
*actual_read = progress;
return status;
}
errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t misalign = (progress + offset) % page_size;
const size_t page_index = (progress + offset) / page_size;
const size_t copy_size = MIN(page_size - misalign, len - progress);
struct page* p;
status = obj->get_page(obj, page_index, &p);
if (status != 0)
break;
uintptr_t page_addr = vm_page_addr(p);
memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
progress += copy_size;
}
if (actual_written)
*actual_written = progress;
return status;
}
errno_t vmo_copy(
struct vmo* target,
uintptr_t target_offset,
struct vmo* src,
uintptr_t src_offset,
size_t len,
size_t* actual_copied
) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
errno_t status = 0;
while (progress < len) {
const size_t target_misalign = (progress + target_offset) % page_size;
const size_t src_misalign = (progress + src_offset) % page_size;
const size_t target_page_index = (progress + target_offset) / page_size;
const size_t src_page_index = (progress + src_offset) / page_size;
const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
struct page* tpage;
status = target->get_page(target, target_page_index, &tpage);
if (status != 0)
break;
struct page* spage;
status = src->get_page(src, src_page_index, &spage);
if (status != 0)
break;
uintptr_t taddr = vm_page_addr(tpage);
uintptr_t saddr = vm_page_addr(spage);
memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
progress += copy_size;
}
if (actual_copied)
*actual_copied = progress;
return status;
}
static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
// Try to find the requested page in the cache.
struct page_list* iter;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
// We found a cached page!
*out = iter->value;
spin_unlock(&paged->lock);
return 0;
}
spin_unlock(&paged->lock);
// Get page from out pager if we don't have one already.
struct page* new_page;
errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
if (status)
return status;
struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
if (!entry) {
paged->source.put_page(&paged->source, offset_idx, new_page);
return ENOMEM;
}
entry->offset = offset_idx;
entry->value = new_page;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
*out = iter->value;
spin_unlock(&paged->lock);
mem_free(entry);
paged->source.put_page(&paged->source, offset_idx, new_page);
return 0;
}
// Add page to the object cache.
list_insert_head(&paged->cache, entry, next);
spin_unlock(&paged->lock);
*out = new_page;
return 0;
}
static void paged_destroy(struct vmo* vmo) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
struct page_list* iter = list_first(&paged->cache);
while (iter) {
struct page_list* next = list_next(iter, next);
paged->source.put_page(&paged->source, iter->offset, iter->value);
mem_free(iter);
iter = next;
}
mem_free(paged);
}
errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
if (!result)
return ENOMEM;
refcount_init(&result->object.rc);
result->source = pager;
result->object.get_page = paged_get_page;
result->object.destroy = paged_destroy;
*out = result;
return 0;
}
static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
if (status != 0)
return status;
size_t idx = addr / arch_mem_page_size();
*out = &vm_pfndb[idx];
return 0;
}
static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
return mem_phys_free(vm_page_addr(page), 1);
}
static const struct pager_ops phys_pager = {
.get_page = phys_get_page,
.put_page = phys_put_page,
};
errno_t vmo_new_phys(struct paged_vmo** out) {
return vmo_new_paged(phys_pager, out);
}
void vmo_truncate_paged(struct paged_vmo* paged, size_t length) {
const size_t page_size = arch_mem_page_size();
const size_t keep_pages = ROUND_UP(length, page_size);
spin_lock(&paged->lock);
struct page_list** pp = &list_first(&paged->cache);
while (*pp) {
struct page_list* entry = *pp;
if (entry->offset >= keep_pages) {
*pp = list_next(entry, next);
paged->source.put_page(&paged->source, entry->offset, entry->value);
mem_free(entry);
} else {
pp = &list_next(entry, next);
}
}
spin_unlock(&paged->lock);
}
struct phys_range_vmo {
struct vmo object;
uintptr_t start;
size_t len;
};
static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
if (offset_idx >= p->len / arch_mem_page_size())
return EINVAL;
*out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
return 0;
}
static void phys_range_destroy(struct vmo* vmo) {
mem_free(containerof(vmo, struct phys_range_vmo, object));
}
errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
if (!vmo)
return ENOMEM;
refcount_init(&vmo->object.rc);
vmo->object.get_page = phys_range_get_page;
vmo->object.destroy = phys_range_destroy;
vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
vmo->len = ALIGN_UP(length, arch_mem_page_size());
*out = &vmo->object;
return 0;
}
core/module.c
-615
View File
@@ -1,615 +0,0 @@
#define pr_fmt(fmt) "module: " fmt
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/cmdline.h>
#include <zinnia/elf.h>
#include <zinnia/ksyms.h>
#include <zinnia/module.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/spin.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <zinnia/vma.h>
#include <bits/elf.h>
#include <stdckdint.h>
#include <string.h>
extern errno_t (*__ld_mod_entry_start[])(struct module*);
extern errno_t (*__ld_mod_entry_end[])(struct module*);
static struct spinlock module_lock;
static tailq_head(struct module) module_list;
static bool in_range(size_t total, uint64_t offset, uint64_t size) {
return offset <= total && size <= total - offset;
}
static void* image_ptr(struct module* mod, uint64_t vaddr, uint64_t size) {
uintptr_t addr, end;
if (ckd_add(&addr, mod->bias, vaddr) || ckd_add(&end, addr, size))
return nullptr;
for (size_t i = 0; i < mod->num_segments; i++) {
const struct module_segment* seg = &mod->segments[i];
if (addr >= seg->virt && end <= seg->virt + seg->size)
return (void*)addr;
}
return nullptr;
}
static const char* image_cstr(struct module* mod, uint64_t vaddr, uint64_t size) {
const char* str = image_ptr(mod, vaddr, size);
if (str == nullptr || size == 0 || strnlen(str, size) >= size)
return nullptr;
return str;
}
static const char* table_str(const char* table, size_t size, uint64_t offset) {
if (offset >= size)
return nullptr;
const char* str = table + offset;
if (strnlen(str, size - offset) >= size - offset)
return nullptr;
return str;
}
static const char* sym_name(struct module* mod, uint64_t offset) {
return table_str(mod->strtab, mod->strtab_size, offset);
}
static struct module* find_locked(const char* name) {
for (struct module* m = tailq_first(&module_list); m != nullptr; m = tailq_next(m, link)) {
if (!strcmp(m->name, name))
return m;
}
return nullptr;
}
void module_init() {
spin_init(&module_lock);
tailq_init(&module_list);
for (auto start = __ld_mod_entry_start; start < __ld_mod_entry_end; start += 1) {
errno_t mod_status = (*start)(nullptr);
assert(mod_status == 0, "Built-in entry point returned %i", mod_status);
}
}
static errno_t resolve_symbol(struct module* mod, uint32_t index, uintptr_t* out) {
if (index >= mod->num_syms)
return ENOEXEC;
const elf_sym_t* sym = &((const elf_sym_t*)mod->symtab)[index];
if (sym->st_shndx != SHN_UNDEF) {
*out = mod->bias + sym->st_value;
return 0;
}
const char* name = sym_name(mod, sym->st_name);
if (name == nullptr)
return ENOEXEC;
if (ksym_find(name, out))
return 0;
if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
*out = 0;
return 0;
}
log_err("%s: Unresolved symbol \"%s\"\n", mod->name, name);
return ENOENT;
}
static errno_t store_word(struct module* mod, uint64_t vaddr, uintptr_t value) {
uintptr_t* where = image_ptr(mod, vaddr, sizeof(uintptr_t));
if (where == nullptr)
return ENOEXEC;
*where = value;
return 0;
}
static errno_t relocate_relative(struct module* mod, uint64_t vaddr) {
uintptr_t* where = image_ptr(mod, vaddr, sizeof(uintptr_t));
if (where == nullptr)
return ENOEXEC;
*where += mod->bias;
return 0;
}
static errno_t apply_rela(struct module* mod, uint64_t addr, uint64_t size, uint64_t entsize) {
if (entsize == 0)
entsize = sizeof(elf_rela_t);
if (entsize < sizeof(elf_rela_t))
return ENOEXEC;
const uint8_t* table = image_ptr(mod, addr, size);
if (table == nullptr)
return ENOEXEC;
for (uint64_t off = 0; off + sizeof(elf_rela_t) <= size; off += entsize) {
elf_rela_t rela;
memcpy(&rela, table + off, sizeof(rela));
errno_t status = 0;
uintptr_t value;
switch (ELF_R_TYPE(rela.r_info)) {
case R_NONE:
break;
case R_64:
case R_GLOB_DAT:
case R_JUMP_SLOT:
status = resolve_symbol(mod, ELF_R_SYM(rela.r_info), &value);
if (status == 0)
status = store_word(mod, rela.r_offset, value + rela.r_addend);
break;
case R_RELATIVE:
status = store_word(mod, rela.r_offset, mod->bias + rela.r_addend);
break;
default:
log_err("%s: Unsupported relocation type %u\n", mod->name, ELF_R_TYPE(rela.r_info));
status = ENOEXEC;
break;
}
if (status)
return status;
}
return 0;
}
static errno_t apply_relr(struct module* mod, uint64_t addr, uint64_t size, uint64_t entsize) {
if (entsize != 0 && entsize != sizeof(elf_relr_t))
return ENOEXEC;
const elf_relr_t* table = image_ptr(mod, addr, size);
if (table == nullptr)
return ENOEXEC;
uint64_t where = 0;
for (size_t i = 0; i < size / sizeof(elf_relr_t); i++) {
elf_relr_t entry = table[i];
errno_t status;
if ((entry & 1) == 0) {
status = relocate_relative(mod, entry);
if (status)
return status;
where = entry + sizeof(elf_relr_t);
} else {
for (size_t bit = 0; (entry >>= 1) != 0; bit++) {
if ((entry & 1) == 0)
continue;
status = relocate_relative(mod, where + bit * sizeof(elf_relr_t));
if (status)
return status;
}
where += (sizeof(elf_relr_t) * 8 - 1) * sizeof(elf_relr_t);
}
}
return 0;
}
static void module_free(struct module* mod) {
const size_t page_size = arch_mem_page_size();
mem_free(mod->cmdline);
for (size_t i = 0; i < mod->num_segments; i++) {
const struct module_segment* seg = &mod->segments[i];
for (size_t off = 0; off < seg->size; off += page_size)
pmap_unmap(&kernel_vma.pmap, seg->virt + off);
}
pmap_shootdown(&kernel_vma.pmap);
for (size_t i = 0; i < mod->num_segments; i++) {
const struct module_segment* seg = &mod->segments[i];
mem_phys_free(seg->phys, seg->size / page_size);
}
mem_free(mod->segments);
if (mod->base != 0)
vm_free(mod->base, mod->size);
mem_free(mod);
}
static enum pte_flags prot_from_phdr(const elf_phdr_t* phdr) {
enum pte_flags prot = 0;
if (phdr->p_flags & PF_R)
prot |= PTE_READ;
if (phdr->p_flags & PF_W)
prot |= PTE_WRITE;
if (phdr->p_flags & PF_X)
prot |= PTE_EXEC;
return prot;
}
static errno_t read_headers(const void* data, size_t length, elf_ehdr_t* ehdr, elf_phdr_t** out) {
const uint8_t* file = data;
if (data == nullptr || length < sizeof(elf_ehdr_t))
return ENOEXEC;
memcpy(ehdr, file, sizeof(*ehdr));
if (memcmp(ehdr->e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0 || ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
ehdr->e_ident[EI_CLASS] != ELF_ARCH_CLASS || ehdr->e_ident[EI_DATA] != ELF_ARCH_DATA ||
ehdr->e_machine != ELF_ARCH_MACHINE || ehdr->e_type != ET_DYN)
return ENOEXEC;
if (ehdr->e_phentsize < sizeof(elf_phdr_t) || ehdr->e_phnum == 0 ||
!in_range(length, ehdr->e_phoff, (uint64_t)ehdr->e_phnum * ehdr->e_phentsize))
return ENOEXEC;
elf_phdr_t* phdrs = mem_alloc(ehdr->e_phnum * sizeof(elf_phdr_t), 0);
if (phdrs == nullptr)
return ENOMEM;
for (size_t i = 0; i < ehdr->e_phnum; i++)
memcpy(&phdrs[i], file + ehdr->e_phoff + i * ehdr->e_phentsize, sizeof(elf_phdr_t));
*out = phdrs;
return 0;
}
errno_t module_load(const void* data, size_t length, const char* cmdline, struct module** out) {
const uint8_t* file = data;
const size_t page_size = arch_mem_page_size();
elf_ehdr_t ehdr;
elf_phdr_t* phdrs;
errno_t header_status = read_headers(data, length, &ehdr, &phdrs);
if (header_status)
return header_status;
struct module* mod = mem_alloc(sizeof(struct module), 0);
if (mod == nullptr) {
mem_free(phdrs);
return ENOMEM;
}
mod->version = "";
mod->author = "";
mod->desc = "";
mod->license = "";
errno_t status = ENOEXEC;
bool listed = false;
uintptr_t load_min = UINTPTR_MAX;
uintptr_t load_end = 0;
size_t num_loads = 0;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
const elf_phdr_t* phdr = &phdrs[i];
if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
continue;
if (phdr->p_filesz > phdr->p_memsz || !in_range(length, phdr->p_offset, phdr->p_filesz))
goto fail;
uintptr_t end;
if (ckd_add(&end, phdr->p_vaddr, phdr->p_memsz) || end > UINTPTR_MAX - page_size)
goto fail;
const uintptr_t start = ALIGN_DOWN(phdr->p_vaddr, page_size);
if (start < load_end)
goto fail;
if (load_min == UINTPTR_MAX)
load_min = start;
load_end = ALIGN_UP(end, page_size);
num_loads++;
}
if (num_loads == 0)
goto fail;
status = ENOMEM;
mod->segments = mem_alloc(num_loads * sizeof(struct module_segment), 0);
if (mod->segments == nullptr)
goto fail;
mod->base = vm_alloc(load_end - load_min);
if (mod->base == 0)
goto fail;
mod->size = load_end - load_min;
mod->bias = mod->base - load_min;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
const elf_phdr_t* phdr = &phdrs[i];
if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
continue;
const uintptr_t start = ALIGN_DOWN(phdr->p_vaddr, page_size);
const size_t size = ALIGN_UP(phdr->p_vaddr + phdr->p_memsz, page_size) - start;
phys_t phys;
status = mem_phys_alloc(size / page_size, 0, &phys);
if (status)
goto fail;
struct module_segment* seg = &mod->segments[mod->num_segments++];
*seg = (struct module_segment){.phys = phys, .virt = mod->bias + start, .size = size};
for (size_t off = 0; off < size; off += page_size) {
status = pmap_map(&kernel_vma.pmap, seg->virt + off, phys + off, PTE_READ | PTE_WRITE, CACHE_WRITE_BACK);
if (status)
goto fail;
}
memcpy((void*)(mod->bias + phdr->p_vaddr), file + phdr->p_offset, phdr->p_filesz);
}
status = ENOEXEC;
const elf_dyn_t* dyn = nullptr;
size_t num_dyn = 0;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
if (phdrs[i].p_type != PT_DYNAMIC)
continue;
dyn = image_ptr(mod, phdrs[i].p_vaddr, phdrs[i].p_memsz);
num_dyn = phdrs[i].p_memsz / sizeof(elf_dyn_t);
break;
}
if (dyn == nullptr)
goto fail;
uint64_t dt_strtab = 0, dt_strsz = 0, dt_symtab = 0, dt_hash = 0, dt_soname = 0;
uint64_t dt_rela = 0, dt_relasz = 0, dt_relaent = 0;
uint64_t dt_jmprel = 0, dt_pltrelsz = 0, dt_pltrel = DT_RELA;
uint64_t dt_relr = 0, dt_relrsz = 0, dt_relrent = 0;
bool has_strtab = false, has_symtab = false, has_hash = false, has_soname = false;
bool has_rela = false, has_jmprel = false, has_relr = false, has_needed = false;
for (size_t i = 0; i < num_dyn && dyn[i].d_tag != DT_NULL; i++) {
const uint64_t val = dyn[i].d_un.d_val;
switch (dyn[i].d_tag) {
case DT_STRTAB:
dt_strtab = val;
has_strtab = true;
break;
case DT_STRSZ:
dt_strsz = val;
break;
case DT_SYMTAB:
dt_symtab = val;
has_symtab = true;
break;
case DT_HASH:
dt_hash = val;
has_hash = true;
break;
case DT_SONAME:
dt_soname = val;
has_soname = true;
break;
case DT_NEEDED:
has_needed = true;
break;
case DT_RELA:
dt_rela = val;
has_rela = true;
break;
case DT_RELASZ:
dt_relasz = val;
break;
case DT_RELAENT:
dt_relaent = val;
break;
case DT_JMPREL:
dt_jmprel = val;
has_jmprel = true;
break;
case DT_PLTRELSZ:
dt_pltrelsz = val;
break;
case DT_PLTREL:
dt_pltrel = val;
break;
case DT_RELR:
dt_relr = val;
has_relr = true;
break;
case DT_RELRSZ:
dt_relrsz = val;
break;
case DT_RELRENT:
dt_relrent = val;
break;
default:
break;
}
}
if (!has_strtab || !has_symtab || !has_hash || !has_soname || dt_strsz == 0 || dt_pltrel != DT_RELA)
goto fail;
const elf_hash_t* hash = image_ptr(mod, dt_hash, sizeof(elf_hash_t));
if (hash == nullptr)
goto fail;
mod->num_syms = hash->n_chain;
mod->symtab = image_ptr(mod, dt_symtab, (uint64_t)mod->num_syms * sizeof(elf_sym_t));
mod->strtab = image_ptr(mod, dt_strtab, dt_strsz);
mod->strtab_size = dt_strsz;
if (mod->symtab == nullptr || mod->strtab == nullptr) {
mod->num_syms = 0;
goto fail;
}
mod->name = sym_name(mod, dt_soname);
if (mod->name == nullptr || *mod->name == 0)
goto fail;
for (size_t i = 0; has_needed && i < num_dyn && dyn[i].d_tag != DT_NULL; i++) {
if (dyn[i].d_tag != DT_NEEDED)
continue;
// TODO: Modules can technically claim they have dependencies on other modules, but we don't support that yet.
const char* needed = sym_name(mod, dyn[i].d_un.d_val);
const char* kernel_name = ksym_kernel_name();
if (needed == nullptr || kernel_name == nullptr || strcmp(needed, kernel_name) != 0) {
log_err("%s: Modules can only depend on the kernel, not \"%s\"\n", mod->name, needed ? needed : "?");
goto fail;
}
}
if (has_relr) {
status = apply_relr(mod, dt_relr, dt_relrsz, dt_relrent);
if (status)
goto fail;
}
if (has_rela) {
status = apply_rela(mod, dt_rela, dt_relasz, dt_relaent);
if (status)
goto fail;
}
if (has_jmprel) {
status = apply_rela(mod, dt_jmprel, dt_pltrelsz, dt_relaent);
if (status)
goto fail;
}
spin_lock(&module_lock);
if (find_locked(mod->name) != nullptr) {
spin_unlock(&module_lock);
status = EEXIST;
goto fail;
}
tailq_insert_tail(&module_list, mod, link);
listed = true;
spin_unlock(&module_lock);
const struct cmdline_option* const* opts_start = nullptr;
const struct cmdline_option* const* opts_end = nullptr;
status = ENOEXEC;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
const elf_phdr_t* phdr = &phdrs[i];
const char* str;
switch (phdr->p_type) {
case PT_MOD_VERSION:
if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
mod->version = str;
break;
case PT_MOD_AUTHOR:
if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
mod->author = str;
break;
case PT_MOD_DESC:
if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
mod->desc = str;
break;
case PT_MOD_LICENSE:
if ((str = image_cstr(mod, phdr->p_vaddr, phdr->p_memsz)) != nullptr)
mod->license = str;
break;
case PT_MOD_CMDLINE:
if (phdr->p_memsz < sizeof(void*))
break;
opts_start = image_ptr(mod, phdr->p_vaddr, phdr->p_memsz);
if (opts_start == nullptr)
goto fail;
opts_end = opts_start + (phdr->p_memsz / sizeof(void*));
break;
case PT_MOD_EXIT: {
if (phdr->p_memsz < sizeof(void*))
break;
const void* slot = image_ptr(mod, phdr->p_vaddr, sizeof(void*));
if (slot == nullptr)
goto fail;
memcpy(&mod->exit, slot, sizeof(void*));
break;
}
default:
break;
}
}
if (image_ptr(mod, ehdr.e_entry, 1) == nullptr)
goto fail;
size_t seg_idx = 0;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
const elf_phdr_t* phdr = &phdrs[i];
if (phdr->p_type != PT_LOAD || phdr->p_memsz == 0)
continue;
const struct module_segment* seg = &mod->segments[seg_idx++];
const enum pte_flags prot = prot_from_phdr(phdr);
for (size_t off = 0; off < seg->size; off += page_size) {
pmap_unmap(&kernel_vma.pmap, seg->virt + off);
status = pmap_map(&kernel_vma.pmap, seg->virt + off, seg->phys + off, prot, CACHE_WRITE_BACK);
if (status)
goto fail;
}
}
if (cmdline == nullptr)
cmdline = "";
const size_t cmdline_len = strlen(cmdline);
mod->cmdline = mem_alloc(cmdline_len + 1, 0);
if (mod->cmdline == nullptr) {
status = ENOMEM;
goto fail;
}
memcpy(mod->cmdline, cmdline, cmdline_len);
if (opts_start != nullptr)
cmdline_parse_buf(mod->cmdline, cmdline_len, opts_start, opts_end);
log("Loaded module \"%s\" (%s, %s) at %p\n", mod->name, mod->desc, mod->version, (void*)mod->base);
errno_t (*entry)(struct module*) = (errno_t (*)(struct module*))(mod->bias + ehdr.e_entry);
status = entry(mod);
if (status) {
log_err("%s: Entry point returned %i\n", mod->name, status);
goto fail;
}
mem_free(phdrs);
if (out != nullptr)
*out = mod;
return 0;
fail:
if (listed) {
spin_lock(&module_lock);
tailq_remove(&module_list, mod, link);
spin_unlock(&module_lock);
}
module_free(mod);
mem_free(phdrs);
return status;
}
errno_t module_unload(struct module* mod) {
if (mod == nullptr)
return EINVAL;
if (mod->exit == nullptr)
return ENOTSUP;
spin_lock(&module_lock);
tailq_remove(&module_list, mod, link);
spin_unlock(&module_lock);
errno_t status = mod->exit(mod);
if (status) {
log_err("%s: Exit function returned %i\n", mod->name, status);
spin_lock(&module_lock);
tailq_insert_tail(&module_list, mod, link);
spin_unlock(&module_lock);
return status;
}
log("Unloaded module \"%s\"\n", mod->name);
module_free(mod);
return 0;
}
struct module* module_find(const char* name) {
spin_lock(&module_lock);
struct module* mod = find_locked(name);
spin_unlock(&module_lock);
return mod;
}
core/panic.c
-73
View File
@@ -1,73 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/initgraph.h>
#include <zinnia/irq.h>
#include <zinnia/ksyms.h>
#include <zinnia/panic.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <stdarg.h>
#include <stdint.h>
#define panic_log(msg, ...) kprintf("[ !!!! ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
static void print_backtrace() {
constexpr size_t max_depth = 32;
void** fp = __builtin_frame_address(0);
panic_log("Backtrace:\n");
for (size_t i = 0; i < max_depth; i++) {
if (!fp || (uintptr_t)fp < mem_hhdm_base || (uintptr_t)fp % alignof(void*))
break;
void* ret = fp[1];
if (!ret)
break;
const char* name;
size_t offset;
// The return address may already point past the end of a noreturn call.
if (ksym_lookup((uintptr_t)ret - 1, &name, &offset))
panic_log(" #%zu [%p] %s+0x%zx\n", i, ret, name, offset + 1);
else
panic_log(" #%zu [%p] <\?\?\?>\n", i, ret);
void** next = fp[0];
if (next <= fp)
break;
fp = next;
}
}
[[noreturn]]
void panic(const char* msg, ...) {
// TODO: Stop all other CPUs with IPI.
irq_set_state(false);
struct percpu* cpu = percpu_get();
struct task* current = cpu->sched.current;
kprintf(
"[ !!!! ] \e[31m"
"Kernel panic on CPU %zu (T%zu, \"%s\"):\e[0m\n",
cpu->id,
current ? current->id : 0,
current ? current->name : nullptr
);
kprintf("[ !!!! ] \e[31m");
va_list args;
va_start(args, msg);
kvprintf(msg, args);
va_end(args);
kprintf("\e[0m");
print_backtrace();
panic_log("End of panic message\n");
initgraph_panic();
while (true) {
irq_wait();
}
}
core/percpu.c
-77
View File
@@ -1,77 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/cmdline.h>
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/percpu.h>
static _Atomic(size_t) num_cpus = 1;
static size_t max_cpus = SIZE_MAX;
static void smp_option(uintptr_t value) {
max_cpus = value;
}
CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option);
[[__used, __section(".percpu.init")]]
struct percpu percpu_bsp = {
.self = &percpu_bsp,
.id = 0,
.online = false,
};
struct percpu* percpu_new() {
const size_t pgsz = arch_mem_page_size();
const size_t stride = percpu_stride();
const size_t id = atomic_load(&num_cpus, ATOMIC_RELAXED);
if (id >= (UINTPTR_MAX - (uintptr_t)__ld_percpu_start) / stride)
return nullptr;
phys_t mem;
if (mem_phys_alloc(stride / pgsz, 0, &mem))
return nullptr;
const uintptr_t base = (uintptr_t)percpu_cpu(id);
for (size_t offset = 0; offset < stride; offset += pgsz) {
if (pmap_map(&kernel_vma.pmap, base + offset, mem + offset, PTE_READ | PTE_WRITE, CACHE_ANY)) {
while (offset) {
offset -= pgsz;
pmap_unmap(&kernel_vma.pmap, base + offset);
}
mem_phys_free(mem, stride / pgsz);
return nullptr;
}
}
struct percpu* cpu = (struct percpu*)base;
cpu->self = cpu;
cpu->id = id;
cpu->online = false;
atomic_store(&num_cpus, id + 1, ATOMIC_RELEASE);
return cpu;
}
size_t percpu_max_cpus() {
return max_cpus;
}
size_t percpu_count() {
return atomic_load(&num_cpus, ATOMIC_RELAXED);
}
struct percpu* percpu_get() {
return arch_percpu_get();
}
void percpu_bsp_init() {
arch_percpu_bsp_init();
}
void percpu_init(struct boot_info*) {
arch_percpu_init();
}
INIT_TASK(percpu_stage, "core.percpu", percpu_init);
INIT_DEPENDS(percpu_stage, mem_stage);
core/print.c
-495
View File
@@ -1,495 +0,0 @@
#include <zinnia/clock.h>
#include <zinnia/console.h>
#include <zinnia/panic.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/spin.h>
#include <zinnia/stdlib.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) {
for (size_t i = 0; i < count; i++)
callback(ctx, &c, 1);
}
void print_fmt(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
// Amount of bytes written.
int32_t written = 0;
while (*fmt != '\0') {
size_t maxrem = INT32_MAX - written;
if (fmt[0] != '%' || fmt[1] == '%') {
if (fmt[0] == '%')
fmt++;
size_t amount = 1;
while (fmt[amount] && fmt[amount] != '%')
amount++;
if (maxrem < amount) {
return;
}
callback(ctx, fmt, amount);
fmt += amount;
written += amount;
continue;
}
const char* format_begun_at = fmt++;
bool force_sign = false;
bool blank_sign = false;
bool write_prefix = false;
bool pad_zero = false;
bool left_align = false;
size_t width = 0;
bool has_precision = false;
size_t precision = 0;
size_t size = sizeof(uint32_t);
char number[32];
check_fmt:
memset(number, 0, sizeof(number));
switch (*fmt) {
// Flags
case '+': {
force_sign = true;
fmt++;
goto check_fmt;
}
case ' ': {
blank_sign = true;
fmt++;
goto check_fmt;
}
case '#': {
write_prefix = true;
fmt++;
goto check_fmt;
}
case '0': {
pad_zero = true;
fmt++;
goto check_fmt;
}
case '-': {
left_align = true;
fmt++;
goto check_fmt;
}
// Width
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
width = 0;
while (*fmt >= '0' && *fmt <= '9') {
width = width * 10 + (*fmt - '0');
fmt++;
}
goto check_fmt;
}
// Precision
case '.': {
has_precision = true;
fmt++;
precision = 0;
if (*fmt == '*') {
const int32_t arg = va_arg(args, int32_t);
if (arg < 0)
has_precision = false;
else
precision = arg;
fmt++;
} else {
while (*fmt >= '0' && *fmt <= '9') {
precision = precision * 10 + (*fmt - '0');
fmt++;
}
}
goto check_fmt;
}
// Length
case 'h': {
if (fmt[1] == 'h') {
size = sizeof(uint8_t);
fmt++;
} else
size = sizeof(uint16_t);
fmt++;
goto check_fmt;
}
case 'l': {
// There is no difference between `long int` and `long long int`.
if (fmt[1] == 'l')
fmt++;
fmt++;
size = sizeof(uint64_t);
goto check_fmt;
}
case 'z': {
size = sizeof(size_t);
fmt++;
goto check_fmt;
}
// Character
case 'c': {
char c = (char)va_arg(args, int32_t);
const size_t pad = width > 1 ? width - 1 : 0;
if (!left_align)
print_pad(callback, ctx, ' ', pad);
callback(ctx, &c, sizeof(c));
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += 1 + pad;
fmt++;
break;
}
// String of characters
case 's': {
const char* str = va_arg(args, const char*);
if (!str)
str = "(null)";
size_t len = 0;
if (has_precision)
len = strnlen(str, precision);
else
len = strlen(str);
const size_t pad = width > len ? width - len : 0;
if (!left_align)
print_pad(callback, ctx, ' ', pad);
callback(ctx, str, len);
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += len + pad;
fmt++;
break;
}
// Signed decimal integer
case 'i':
case 'd': {
switch (size) {
case sizeof(int8_t):
ltoa((int8_t)va_arg(args, int32_t), number, 10);
break;
case sizeof(int16_t):
ltoa((int16_t)va_arg(args, int32_t), number, 10);
break;
case sizeof(int32_t):
ltoa((int32_t)va_arg(args, int32_t), number, 10);
break;
default:
ltoa(va_arg(args, int64_t), number, 10);
break;
}
goto print_num;
}
// Unsigned decimal integer
case 'u': {
switch (size) {
case sizeof(uint8_t):
lutoa((uint8_t)va_arg(args, uint32_t), number, 10);
break;
case sizeof(uint16_t):
lutoa((uint16_t)va_arg(args, uint32_t), number, 10);
break;
case sizeof(uint32_t):
lutoa(va_arg(args, uint32_t), number, 10);
break;
case sizeof(uint64_t):
lutoa(va_arg(args, uint64_t), number, 10);
break;
}
goto print_num;
}
// Unsigned hexadecimal integer
case 'X':
case 'x': {
switch (size) {
case sizeof(uint8_t):
lutoa((uint8_t)va_arg(args, uint32_t), number, 16);
break;
case sizeof(uint16_t):
lutoa((uint16_t)va_arg(args, uint32_t), number, 16);
break;
case sizeof(uint32_t):
lutoa(va_arg(args, uint32_t), number, 16);
break;
case sizeof(uint64_t):
lutoa(va_arg(args, uint64_t), number, 16);
break;
}
print_num:
const char* digits = number;
char sign = 0;
if (digits[0] == '-') {
sign = '-';
digits++;
} else if (force_sign) {
sign = '+';
} else if (blank_sign) {
sign = ' ';
}
const size_t len = strlen(digits);
size_t zeros = (has_precision && precision > len) ? precision - len : 0;
const size_t total = (sign ? 1 : 0) + (write_prefix ? 2 : 0) + zeros + len;
size_t pad = width > total ? width - total : 0;
if (pad_zero && !left_align && !has_precision) {
zeros += pad;
pad = 0;
}
if (!left_align)
print_pad(callback, ctx, ' ', pad);
if (sign)
callback(ctx, &sign, 1);
if (write_prefix)
callback(ctx, "0x", 2);
print_pad(callback, ctx, '0', zeros);
callback(ctx, digits, len);
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += width > total ? width : total;
fmt++;
break;
}
// Pointer address
case 'p': {
const size_t num = va_arg(args, size_t);
const size_t buf_size = sizeof(size_t) * 2 + 1;
char str[buf_size];
lutoa(num, number, 0x10);
const size_t len = strlen(number); // Get the length of the final number.
for (size_t i = 0; i < buf_size; i++)
str[i] = '0';
size_t offset = sizeof(str) - len - 1;
callback(ctx, str, offset);
callback(ctx, number, len);
fmt++;
written += len;
break;
}
// The number of characters written so far is stored in the pointed location.
case 'n': {
int32_t* const ptr = va_arg(args, int32_t*);
*ptr = written;
fmt++;
break;
}
// No format, just normal text.
default: {
size_t len = fmt - format_begun_at;
if (*fmt)
len++;
callback(ctx, format_begun_at, len);
written += len;
fmt = format_begun_at + len;
break;
}
}
}
}
static void printf_console(void*, const char* msg, size_t len) {
console_write(msg, len);
}
static struct spinlock print_lock = {};
void kprintf(const char* message, ...) {
spin_lock(&print_lock);
va_list args;
va_start(args, message);
print_fmt(printf_console, nullptr, message, args);
va_end(args);
spin_unlock(&print_lock);
}
struct snprintf_ctx {
char* buf;
size_t size;
size_t len;
};
static void snprintf_write(void* ctx, const char* str, size_t len) {
struct snprintf_ctx* c = ctx;
// Always keep room for the terminator.
if (c->len < c->size) {
size_t room = c->size - c->len - 1;
memcpy(c->buf + c->len, str, len < room ? len : room);
}
c->len += len;
}
size_t kvsnprintf(char* buf, size_t size, const char* format, va_list args) {
struct snprintf_ctx ctx = {.buf = buf, .size = size};
print_fmt(snprintf_write, &ctx, format, args);
if (size > 0)
buf[ctx.len < size ? ctx.len : size - 1] = '\0';
return ctx.len;
}
size_t ksnprintf(char* buf, size_t size, const char* format, ...) {
va_list args;
va_start(args, format);
size_t len = kvsnprintf(buf, size, format, args);
va_end(args);
return len;
}
void kvprintf(const char* message, va_list args) {
spin_lock(&print_lock);
print_fmt(printf_console, nullptr, message, args);
spin_unlock(&print_lock);
}
struct log_record {
uint64_t time_ns;
uint32_t tid;
uint16_t len;
uint8_t level;
};
#define LOG_BUFFER_SIZE (64 * 1024)
#define LOG_MESSAGE_MAX 255
static uint8_t log_buffer[LOG_BUFFER_SIZE];
// Protected by print_lock.
static size_t log_head = 0;
static size_t log_tail = 0;
static void log_ring_copy_in(const void* src, size_t len) {
for (size_t i = 0; i < len; i++)
log_buffer[(log_head + i) % LOG_BUFFER_SIZE] = ((const uint8_t*)src)[i];
log_head += len;
}
static void log_ring_peek(size_t offset, void* dst, size_t len) {
for (size_t i = 0; i < len; i++)
((uint8_t*)dst)[i] = log_buffer[(offset + i) % LOG_BUFFER_SIZE];
}
static void log_store(const struct log_record* rec, const char* msg) {
const size_t total = sizeof(*rec) + rec->len;
// Drop the oldest records until the new one fits.
while (LOG_BUFFER_SIZE - (log_head - log_tail) < total) {
struct log_record old;
log_ring_peek(log_tail, &old, sizeof(old));
log_tail += sizeof(old) + old.len;
}
log_ring_copy_in(rec, sizeof(*rec));
log_ring_copy_in(msg, rec->len);
}
struct log_line {
char buf[LOG_MESSAGE_MAX];
size_t len;
};
static void log_line_append(void* ctx, const char* str, size_t len) {
struct log_line* line = ctx;
if (len > sizeof(line->buf) - line->len)
len = sizeof(line->buf) - line->len;
memcpy(line->buf + line->len, str, len);
line->len += len;
}
// ANSI colors for each level.
static const char* const log_level_colors[] = {
[LOG_DEBUG] = "\e[90m",
[LOG_STATUS] = "\e[32m",
[LOG_INFO] = "",
[LOG_WARN] = "\e[33m",
[LOG_ERR] = "\e[31m",
[LOG_PANIC] = "\e[1;31m",
};
static void log_cb_printf(log_fn_t callback, void* ctx, const char* fmt, ...) {
va_list args;
va_start(args, fmt);
print_fmt(callback, ctx, fmt, args);
va_end(args);
}
// Formats a single record and passes it to `callback`.
static void log_emit(log_fn_t callback, void* ctx, const struct log_record* rec, const char* msg) {
log_cb_printf(
callback,
ctx,
"[%5lu.%06lu] %s",
rec->time_ns / 1'000'000'000,
rec->time_ns / 1'000 % 1'000'000,
log_level_colors[rec->level]
);
callback(ctx, msg, rec->len);
callback(ctx, "\e[0m\n", 5);
}
void log_replay(log_fn_t callback, void* ctx) {
char msg[LOG_MESSAGE_MAX];
for (size_t offset = log_tail; offset != log_head;) {
struct log_record rec;
log_ring_peek(offset, &rec, sizeof(rec));
log_ring_peek(offset + sizeof(rec), msg, rec.len);
log_emit(callback, ctx, &rec, msg);
offset += sizeof(rec) + rec.len;
}
}
void log_vwrite(enum log_level level, const char* message, va_list args) {
log_vwrite_prefixed(level, "", message, args);
}
void log_vwrite_prefixed(enum log_level level, const char* prefix, const char* message, va_list args) {
struct log_line line = {};
log_line_append(&line, prefix, strlen(prefix));
print_fmt(log_line_append, &line, message, args);
if (line.len > 0 && line.buf[line.len - 1] == '\n')
line.len--;
auto cur = percpu_get()->sched.current;
const struct log_record rec = {
.time_ns = clock_get_elapsed_ns(),
.tid = cur ? cur->id : 0,
.len = line.len,
.level = level,
};
spin_lock(&print_lock);
log_store(&rec, line.buf);
log_emit(printf_console, nullptr, &rec, line.buf);
spin_unlock(&print_lock);
}
void log_write(enum log_level level, const char* message, ...) {
va_list args;
va_start(args, message);
log_vwrite(level, message, args);
va_end(args);
}
core/random.c
-61
View File
@@ -1,61 +0,0 @@
#define pr_fmt(fmt) "random: " fmt
#include <zinnia/atomic.h>
#include <zinnia/clock.h>
#include <zinnia/print.h>
#include <zinnia/random.h>
#include <zinnia/spin.h>
#include <string.h>
static _Atomic(uint64_t) state = 0x9E37'79B9'7F4A'7C15;
static bool dumb_random(struct random_source*, uint64_t* out) {
uint64_t z = atomic_fetch_add(&state, 0x9E37'79B9'7F4A'7C15, ATOMIC_RELAXED);
z ^= clock_get_elapsed_ns() * 0x2545'F491'4F6C'DD1D;
z = (z ^ (z >> 30)) * 0xBF58'476D'1CE4'E5B9;
z = (z ^ (z >> 27)) * 0x94D0'49BB'1331'11EB;
*out = z ^ (z >> 31);
return true;
}
static struct random_source fallback_source = {
.name = "fallback",
.quality = 0,
.get_u64 = dumb_random,
};
static _Atomic(struct random_source*) active_source = &fallback_source;
static struct spinlock lock = {};
bool random_register(struct random_source* source) {
spin_lock(&lock);
struct random_source* active = atomic_load(&active_source, ATOMIC_RELAXED);
if (source->quality <= active->quality) {
spin_unlock(&lock);
return false;
}
atomic_store(&active_source, source, ATOMIC_RELEASE);
spin_unlock(&lock);
log("Switching to random source \"%s\"\n", source->name);
return true;
}
void random_fill(void* buf, size_t len) {
struct random_source* source = atomic_load(&active_source, ATOMIC_ACQUIRE);
uint8_t* out = buf;
while (len) {
uint64_t value;
source->get_u64(source, &value);
const size_t chunk = len < sizeof(value) ? len : sizeof(value);
memcpy(out, &value, chunk);
out += chunk;
len -= chunk;
}
}
INIT_TASK(random_stage, "core.random");
core/sched/core.c
-1
View File
@@ -1 +0,0 @@
#include <zinnia/sched.h>
core/sched/meson.build
-7
View File
@@ -1,7 +0,0 @@
kernel_sources += files(
'core.c',
)
if kconfig.get('CONFIG_SCHED_DUMB', 'n') == 'y'
kernel_sources += files('sched_dumb.c')
endif
core/sched/sched_dumb.c
-194
View File
@@ -1,194 +0,0 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/list.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/spin.h>
#include <zinnia/stack.h>
#include <zinnia/tailq.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <stdint.h>
static void idle_func(uintptr_t) {
while (1) {
irq_set_state(true);
irq_wait();
}
}
void sched_to_user(uintptr_t info_ptr) {
struct task_start_info* info = (struct task_start_info*)info_ptr;
uintptr_t ip = info->ip;
uintptr_t sp = info->sp;
uintptr_t arg = info->arg;
mem_free(info);
arch_sched_jump_to_user(ip, sp, arg);
}
void sched_to_user_context(uintptr_t ctx) {
arch_sched_jump_to_context((struct arch_context*)ctx);
}
void sched_init(struct scheduler* sched) {
struct task* idle_task;
assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!");
sched->idle_task = idle_task;
struct task* boot_task = mem_alloc(sizeof(struct task), 0);
assert(boot_task != nullptr, "Unable to create boot task!");
strncpy(boot_task->name, "boot", sizeof(boot_task->name));
boot_task->space = &kernel_vma;
sched->current = boot_task;
tailq_init(&sched->run_queue);
log("sched: Scheduler initialized\n");
}
void sched_add_task(struct scheduler* sched, struct task* task) {
assert(task != nullptr, "Tried to add NULL to the run queue");
task->state = TASK_STATE_READY;
tailq_insert_tail(&sched->run_queue, task, next);
}
static struct task* next_task(struct scheduler* sched) {
struct task* t = tailq_first(&sched->run_queue);
if (t == nullptr)
return sched->idle_task;
tailq_remove(&sched->run_queue, t, next);
t->state = TASK_STATE_RUNNING;
return t;
}
static void finish_switch(struct task* prev) {
if (prev->state == TASK_STATE_DEAD)
task_destroy(prev);
}
static void do_reschedule(struct scheduler* sched) {
struct task* from = sched->current;
struct task* to = next_task(sched);
// We'd end up on the same task, skip switching altogether.
if (from == to) {
irq_unlock();
return;
}
sched->current = to;
pmap_set(&to->space->pmap);
struct percpu* cpu = percpu_get();
from->user_stack = cpu->user_stack;
cpu->kernel_stack = kernel_stack_top(&to->kernel_stack);
cpu->user_stack = to->user_stack;
struct irq_percpu irq_state = *percpu_ref(irq_data);
struct task* prev = arch_sched_switch(from, to);
*percpu_ref(irq_data) = irq_state;
finish_switch(prev);
irq_unlock();
}
void sched_reschedule(struct scheduler* sched) {
irq_lock();
if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD)
sched_add_task(sched, sched->current);
do_reschedule(sched);
}
void sched_yield(struct scheduler* sched) {
irq_lock();
do_reschedule(sched);
}
void sched_block(struct scheduler* sched) {
irq_lock();
if (atomic_exchange(&sched->current->wake_pending, false, ATOMIC_ACQ_REL)) {
irq_unlock();
return;
}
sched->current->state = TASK_STATE_BLOCKED;
do_reschedule(sched);
}
void sched_wake(struct task* task) {
irq_lock();
if (task->state == TASK_STATE_BLOCKED)
sched_add_task(&percpu_get()->sched, task);
else
atomic_store(&task->wake_pending, true, ATOMIC_RELEASE);
irq_unlock();
}
errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
if (!out)
return EINVAL;
struct task* new_task = mem_alloc(sizeof(struct task), 0);
if (new_task == nullptr)
return ENOMEM;
static atomic_size_t last_tid = 0;
new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE);
new_task->space = space;
strncpy(new_task->name, name, sizeof(new_task->name));
// Allocate a kernel stack.
errno_t s = stack_init(&new_task->kernel_stack, KERNEL_STACK_SIZE);
if (s != 0) {
mem_free(new_task);
return s;
}
new_task->user_stack = 0;
// Initialize the arch-dependent parts.
s = arch_task_init(&new_task->context, entry, arg0, kernel_stack_top(&new_task->kernel_stack), true);
if (s != 0) {
stack_destroy(&new_task->kernel_stack);
mem_free(new_task);
return s;
}
*out = new_task;
return 0;
}
void task_destroy(struct task* task) {
arch_task_fini(&task->context);
stack_destroy(&task->kernel_stack);
mem_free(task);
}
void task_entry(task_fn_t entry, uintptr_t arg0, struct task* prev) {
finish_switch(prev);
*percpu_ref(irq_data) = (struct irq_percpu){
.depth = 1,
.saved_state = true,
};
irq_unlock();
entry(arg0);
struct scheduler* sched = &percpu_get()->sched;
sched->current->state = TASK_STATE_DEAD;
sched_yield(sched);
panic("unreachable");
}
[[__init]]
static void sched_stage_init(struct boot_info*) {
sched_init(&percpu_get()->sched);
}
INIT_TASK(sched_stage, "core.sched", sched_stage_init);
INIT_DEPENDS(sched_stage, percpu_stage);
core/spin.c
-24
View File
@@ -1,24 +0,0 @@
#include <zinnia/atomic.h>
#include <zinnia/irq.h>
#include <zinnia/spin.h>
#include <arch/cpu.h>
void spin_init(struct spinlock* spin) {
spin->next = 0;
spin->owner = 0;
}
void spin_lock(struct spinlock* spin) {
irq_lock();
uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED);
while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) {
arch_cpu_pause();
}
}
void spin_unlock(struct spinlock* spin) {
uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED);
atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE);
irq_unlock();
}
core/wait.c
-17
View File
@@ -1,17 +0,0 @@
#include <zinnia/waitq.h>
void waitq_init(struct waitq* wq) {
tailq_init(&wq->waiters);
}
void waitq_wait(struct waitq* wq) {
// TODO
}
void waitq_wake_one(struct waitq* wq) {
// TODO
}
void waitq_wake_all(struct waitq* wq) {
// TODO
}
dev/Kconfig
-19
View File
@@ -1,19 +0,0 @@
menu "Device Drivers"
config OF
bool "Support for OpenFirmware"
default y
help
Adds support for flattened device trees and OF platform drivers.
config ACPI
bool "Support for ACPI firmware"
default y
help
Adds support for ACPI firmware, tables and devices.
source "dev/block/Kconfig"
source "dev/pci/Kconfig"
source "dev/video/Kconfig"
endmenu
dev/acpi/include/uacpi_types.h
-39
View File
@@ -1,39 +0,0 @@
#pragma once
#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
typedef uint8_t uacpi_u8;
typedef uint16_t uacpi_u16;
typedef uint32_t uacpi_u32;
typedef uint64_t uacpi_u64;
typedef int8_t uacpi_i8;
typedef int16_t uacpi_i16;
typedef int32_t uacpi_i32;
typedef int64_t uacpi_i64;
#define UACPI_TRUE true
#define UACPI_FALSE false
typedef bool uacpi_bool;
#define UACPI_NULL nullptr
typedef uintptr_t uacpi_uintptr;
typedef uacpi_uintptr uacpi_virt_addr;
typedef size_t uacpi_size;
typedef va_list uacpi_va_list;
#define uacpi_va_start va_start
#define uacpi_va_end va_end
#define uacpi_va_arg va_arg
typedef char uacpi_char;
#define uacpi_offsetof offsetof
#define UACPI_PRIu64 "llu"
#define UACPI_PRIx64 "llx"
#define UACPI_PRIX64 "llX"
#define UACPI_FMT64(val) ((unsigned long long)(val))
dev/acpi/include/zinnia/acpi.h
-17
View File
@@ -1,17 +0,0 @@
#pragma once
#include <zinnia/init.h>
#include <zinnia/initgraph.h>
#include <zinnia/types.h>
#include <stdint.h>
INIT_EXTERN(acpi_stage);
extern phys_t acpi_rsdp_addr;
struct acpi_match {
const char* hid;
uintptr_t data;
};
#define ACPI_MATCH_HID(id) .hid = (id)
dev/acpi/init.c
-45
View File
@@ -1,45 +0,0 @@
#define pr_fmt(fmt) "acpi: " fmt
#include <zinnia/acpi.h>
#include <zinnia/alloc.h>
#include <zinnia/cmdline.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/types.h>
#include <uacpi/status.h>
#include <uacpi/tables.h>
#include <uacpi/uacpi.h>
phys_t acpi_rsdp_addr = 0;
static char table_buf[0x1000];
static bool use_acpi = true;
void use_acpi_option(bool use) {
use_acpi = use;
}
CMDLINE_OPTION("acpi", "Enable ACPI firmware subsystem", use_acpi_option);
[[__init]]
static bool acpi_stage_init(struct boot_info* info) {
if (!use_acpi) {
log_warn("ACPI disabled on the command line!\n");
return false;
}
if (!info->rsdp) {
log_warn("No RSDP was passed, unable to start ACPI subsystem!\n");
return false;
}
acpi_rsdp_addr = info->rsdp;
if (uacpi_setup_early_table_access(table_buf, sizeof(table_buf)) != UACPI_STATUS_OK) {
log_err("Failed to set up early table access!\n");
return false;
}
return true;
}
INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
INIT_DEPENDS(acpi_stage, mem_stage);
INIT_ENTAILS(acpi_stage, percpu_stage);
dev/acpi/mcfg.c
-32
View File
@@ -1,32 +0,0 @@
#define pr_fmt(fmt) "acpi: " fmt
#include <zinnia/acpi.h>
#include <zinnia/pci.h>
#include <zinnia/print.h>
#include <uacpi/acpi.h>
#include <uacpi/status.h>
#include <uacpi/tables.h>
[[__init]]
static void acpi_mcfg_stage_init(struct boot_info*) {
uacpi_table table = {};
if (uacpi_table_find_by_signature(ACPI_MCFG_SIGNATURE, &table) != UACPI_STATUS_OK)
return;
struct acpi_mcfg* mcfg = table.ptr;
size_t count = 0;
if (mcfg->hdr.length > sizeof(*mcfg))
count = (mcfg->hdr.length - sizeof(*mcfg)) / sizeof(struct acpi_mcfg_allocation);
for (size_t i = 0; i < count; i++) {
struct acpi_mcfg_allocation* entry = &mcfg->entries[i];
errno_t status = pci_ecam_register(entry->address, entry->segment, entry->start_bus, entry->end_bus);
if (status != 0)
log_warn("Failed to register ECAM for segment %hu: %i\n", entry->segment, status);
}
uacpi_table_unref(&table);
}
INIT_TASK(acpi_mcfg_stage, "dev.acpi.mcfg", acpi_mcfg_stage_init);
INIT_DEPENDS(acpi_mcfg_stage, acpi_stage);
INIT_ENTAILS(acpi_mcfg_stage, pci_stage);
dev/acpi/meson.build
-20
View File
@@ -1,20 +0,0 @@
uacpi = subproject('uacpi')
kernel_sources += uacpi.get_variable('sources')
kernel_sources += files(
'init.c',
'uacpi_glue.c',
)
if kconfig.get('CONFIG_PCI', 'n') == 'y'
kernel_sources += files('mcfg.c')
endif
kernel_includes += uacpi.get_variable('includes')
kernel_includes += include_directories('include')
kernel_c_args += [
'-DUACPI_FORMATTED_LOGGING',
'-DUACPI_BAREBONES_MODE', # TODO: Remove
'-DUACPI_OVERRIDE_TYPES',
]
dev/acpi/uacpi_glue.c
-73
View File
@@ -1,73 +0,0 @@
#include <zinnia/acpi.h>
#include <zinnia/alloc.h>
#include <zinnia/console.h>
#include <zinnia/init.h>
#include <zinnia/mmio.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <zinnia/vma.h>
#include <arch/mem.h>
#include <stdarg.h>
#include <uacpi/kernel_api.h>
#include <uacpi/status.h>
#include <uacpi/types.h>
uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) {
*out_rsdp_address = (uacpi_phys_addr)acpi_rsdp_addr;
return UACPI_STATUS_OK;
}
void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
uintptr_t aligned_addr = ALIGN_DOWN(addr, arch_mem_page_size());
uintptr_t difference = addr - aligned_addr;
size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
uintptr_t vaddr = vm_alloc(aligned_len);
if (!vaddr)
return nullptr;
for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) {
pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PTE_READ | PTE_WRITE, CACHE_ANY);
}
return (void*)(vaddr + difference);
}
void uacpi_kernel_unmap(void* addr, uacpi_size len) {
uintptr_t aligned_addr = ALIGN_DOWN((uintptr_t)addr, arch_mem_page_size());
uintptr_t difference = (uintptr_t)addr - aligned_addr;
size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
for (size_t i = 0; i < aligned_len; i += arch_mem_page_size())
pmap_unmap(&kernel_vma.pmap, aligned_addr + i);
vm_free(aligned_addr, aligned_len);
}
static enum log_level acpi_log_level(uacpi_log_level lvl) {
switch (lvl) {
case UACPI_LOG_ERROR:
return LOG_ERR;
case UACPI_LOG_WARN:
return LOG_WARN;
case UACPI_LOG_INFO:
return LOG_INFO;
default:
return LOG_DEBUG;
}
}
void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
va_list args;
va_start(args, msg);
log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
va_end(args);
}
void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
}
dev/block/Kconfig
-5
View File
@@ -1,5 +0,0 @@
menu "Block device drivers"
source "dev/block/nvme/Kconfig"
endmenu
dev/block/meson.build
-3
View File
@@ -1,3 +0,0 @@
if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
subdir('nvme')
endif
dev/block/nvme/Kconfig
-13
View File
@@ -1,13 +0,0 @@
menu "NVMe Support"
config NVME_CORE
bool
default y
config NVME_PCI
bool "NVMe via PCI transport"
default y
depends on PCI
depends on NVME_CORE
endmenu
dev/block/nvme/core.c
-41
View File
@@ -1,41 +0,0 @@
#define pr_fmt(fmt) "nvme: " fmt
#include <zinnia/fdt.h>
#include <zinnia/module.h>
#include <zinnia/pci.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <config.h>
#if CONFIG_NVME_PCI
#include "nvme_pci.h"
#endif
struct nvme_dev {
#if CONFIG_NVME_PCI
struct pci_device pci_dev;
#endif
};
errno_t nvme_init(struct module* mod) {
errno_t status = 0;
#if CONFIG_NVME_PCI
status = pci_driver_register(&nvme_driver);
#endif
return status;
}
errno_t nvme_exit(struct module* mod) {
// TODO: Unregister
return 0;
}
MODULE_ENTRY(nvme_init);
MODULE_EXIT(nvme_exit);
MODULE_DESC("NVMe Block Devices");
MODULE_AUTHOR("Marvin Friedrich");
MODULE_LICENSE("ISC");
MODULE_VERSION("1.0");
dev/block/nvme/meson.build
-12
View File
@@ -1,12 +0,0 @@
nvme_sources = files('core.c')
if kconfig.get('CONFIG_NVME_PCI', 'n') == 'y'
nvme_sources += files('pci.c')
endif
modules += {
'nvme': {
'modular': kconfig.get('CONFIG_NVME_CORE') == 'm',
'sources': nvme_sources,
},
}
dev/block/nvme/nvme_pci.h
-3
View File
@@ -1,3 +0,0 @@
#pragma once
extern struct pci_driver nvme_driver;
dev/block/nvme/pci.c
-23
View File
@@ -1,23 +0,0 @@
#include <zinnia/errno.h>
#include <zinnia/module.h>
#include <zinnia/pci.h>
static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
pci_log(dev, "Detected controller \n");
return 0;
}
static const struct pci_match nvme_match[] = {
// Generic NVMe device. Assume no quirks or any funny business.
{PCI_MATCH_CLASS(1, 8, 2), .data = 0},
};
MATCH_TABLE("pci", nvme_match);
struct pci_driver nvme_driver = {
.name = "nvme_pci",
.match = nvme_match,
.num_matches = ARRAY_SIZE(nvme_match),
.probe = nvme_pci_probe,
};
dev/common/meson.build
-3
View File
@@ -1,3 +0,0 @@
kernel_sources += files(
'memfiles.c',
)
dev/dt/fdt.c
-235
View File
@@ -1,235 +0,0 @@
#include <zinnia/fdt.h>
#include <string.h>
uint32_t fdt_read_u32(const void* data) {
const uint8_t* p = data;
return (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | p[3];
}
uint64_t fdt_read_u64(const void* data) {
const uint8_t* p = data;
return (uint64_t)fdt_read_u32(p) << 32 | fdt_read_u32(p + 4);
}
static bool range_valid(size_t offset, size_t length, size_t total) {
return offset <= total && length <= total - offset;
}
struct token {
uint32_t kind;
size_t next;
const char* name;
const void* data;
size_t length;
};
static bool read_token(struct fdt* tree, size_t offset, struct token* token) {
if (!range_valid(offset, 4, tree->structs_len) || offset % 4)
return false;
*token = (struct token){.kind = fdt_read_u32(tree->structs + offset), .next = offset + 4};
if (token->kind == FDT_BEGIN_NODE) {
token->name = (const char*)tree->structs + token->next;
size_t left = tree->structs_len - token->next;
size_t length = strnlen(token->name, left);
if (length == left)
return false;
token->next += (length + 1 + 3) & ~(size_t)3;
} else if (token->kind == FDT_PROP) {
if (!range_valid(token->next, 8, tree->structs_len))
return false;
token->length = fdt_read_u32(tree->structs + token->next);
size_t name_offset = fdt_read_u32(tree->structs + token->next + 4);
token->next += 8;
if (name_offset >= tree->strings_len || !range_valid(token->next, token->length, tree->structs_len))
return false;
token->name = (const char*)tree->strings + name_offset;
size_t left = tree->strings_len - name_offset;
if (strnlen(token->name, left) == left || !*token->name)
return false;
token->data = tree->structs + token->next;
token->next += (token->length + 3) & ~(size_t)3;
} else if (token->kind != FDT_END_NODE && token->kind != FDT_NOP && token->kind != FDT_END) {
return false;
}
return token->next <= tree->structs_len;
}
bool fdt_init(struct fdt* tree, const void* data, size_t length) {
if (!tree)
return false;
*tree = (struct fdt){};
if (!data || length < sizeof(struct fdt_header))
return false;
const uint8_t* blob = data;
size_t total = fdt_read_u32(blob + 4);
size_t structs = fdt_read_u32(blob + 8);
size_t strings = fdt_read_u32(blob + 12);
size_t reserves = fdt_read_u32(blob + 16);
size_t strings_len = fdt_read_u32(blob + 32);
size_t structs_len = fdt_read_u32(blob + 36);
uint32_t version = fdt_read_u32(blob + 20);
uint32_t compat = fdt_read_u32(blob + 24);
if (fdt_read_u32(blob) != FDT_MAGIC || total > length || total < sizeof(struct fdt_header) || version < 17 ||
compat > 17 || compat > version || structs % 4 || reserves % 8 || structs < sizeof(struct fdt_header) ||
strings < sizeof(struct fdt_header) || reserves < sizeof(struct fdt_header) ||
!range_valid(structs, structs_len, total) || !range_valid(strings, strings_len, total) ||
(structs < strings + strings_len && strings < structs + structs_len))
return false;
size_t reserved_end = reserves;
for (;;) {
if (!range_valid(reserved_end, 16, total))
return false;
bool end = !fdt_read_u64(blob + reserved_end) && !fdt_read_u64(blob + reserved_end + 8);
reserved_end += 16;
if (end)
break;
}
if ((reserves < structs + structs_len && structs < reserved_end) ||
(reserves < strings + strings_len && strings < reserved_end))
return false;
struct fdt candidate = {
.blob = blob,
.total_size = total,
.version = version,
.compat_version = compat,
.boot_cpuid = fdt_read_u32(blob + 28),
.structs = blob + structs,
.structs_len = structs_len,
.strings = blob + strings,
.strings_len = strings_len,
};
size_t depth = 0;
bool started = false;
bool children[64] = {};
for (size_t offset = 0; offset < structs_len;) {
struct token token;
if (!read_token(&candidate, offset, &token))
return false;
if (token.kind == FDT_BEGIN_NODE) {
if (depth == 64 || (!depth && (started || *token.name)) || (depth && !*token.name))
return false;
if (depth)
children[depth - 1] = true;
children[depth++] = false;
started = true;
} else if (token.kind == FDT_END_NODE) {
if (!depth)
return false;
depth--;
} else if (token.kind == FDT_PROP) {
if (!depth || children[depth - 1])
return false;
} else if (token.kind == FDT_END) {
if (!started || depth || token.next != structs_len)
return false;
*tree = candidate;
return true;
}
offset = token.next;
}
return false;
}
struct fdt_node fdt_root(struct fdt* tree) {
struct token token;
size_t offset = 0;
if (tree && tree->blob) {
while (read_token(tree, offset, &token)) {
if (token.kind == FDT_BEGIN_NODE)
return (struct fdt_node){tree, offset};
if (token.kind != FDT_NOP)
break;
offset = token.next;
}
}
return (struct fdt_node){};
}
static struct fdt_node find_child(struct fdt_node node, const char* name, size_t length) {
struct token token;
if (!node.tree || !read_token(node.tree, node.offset, &token) || token.kind != FDT_BEGIN_NODE)
return (struct fdt_node){};
size_t depth = 0;
for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
if (token.kind == FDT_BEGIN_NODE) {
if (!depth && strlen(token.name) == length && !memcmp(token.name, name, length))
return (struct fdt_node){node.tree, offset};
depth++;
} else if (token.kind == FDT_END_NODE) {
if (!depth)
break;
depth--;
}
}
return (struct fdt_node){};
}
struct fdt_node fdt_find(struct fdt* tree, const char* path) {
if (!path || *path != '/')
return (struct fdt_node){};
struct fdt_node node = fdt_root(tree);
path++;
while (*path && node.tree) {
size_t length = 0;
while (path[length] && path[length] != '/')
length++;
if (!length)
return (struct fdt_node){};
node = find_child(node, path, length);
path += length;
if (*path == '/')
path++;
}
return node;
}
struct fdt_prop fdt_property(struct fdt_node node, const char* name) {
struct token token;
if (!node.tree || !name || !read_token(node.tree, node.offset, &token) || token.kind != FDT_BEGIN_NODE)
return (struct fdt_prop){};
for (size_t offset = token.next; read_token(node.tree, offset, &token); offset = token.next) {
if (token.kind == FDT_PROP && !strcmp(token.name, name))
return (struct fdt_prop){node.tree, token.data, token.length};
if (token.kind == FDT_BEGIN_NODE || token.kind == FDT_END_NODE)
break;
}
return (struct fdt_prop){};
}
bool fdt_property_u32(struct fdt_node node, const char* name, uint32_t* value) {
struct fdt_prop property = fdt_property(node, name);
if (!property.tree || property.length != 4 || !value)
return false;
*value = fdt_read_u32(property.data);
return true;
}
bool fdt_property_u64(struct fdt_node node, const char* name, uint64_t* value) {
struct fdt_prop property = fdt_property(node, name);
if (!property.tree || property.length != 8 || !value)
return false;
*value = fdt_read_u64(property.data);
return true;
}
struct fdt_node fdt_find_compatible(struct fdt* tree, const char* compatible) {
if (!tree || !tree->blob || !compatible)
return (struct fdt_node){};
struct token token;
for (size_t offset = 0; read_token(tree, offset, &token); offset = token.next) {
if (token.kind != FDT_BEGIN_NODE)
continue;
struct fdt_node node = {tree, offset};
struct fdt_prop property = fdt_property(node, "compatible");
const char* data = property.data;
for (size_t i = 0; i < property.length;) {
size_t length = strnlen(data + i, property.length - i);
if (length == property.length - i)
break;
if (!strcmp(data + i, compatible))
return node;
i += length + 1;
}
}
return (struct fdt_node){};
}
dev/dt/meson.build
-1
View File
@@ -1 +0,0 @@
kernel_sources += files('fdt.c')
dev/meson.build
-14
View File
@@ -1,14 +0,0 @@
subdir('common')
subdir('block')
subdir('dt')
subdir('video')
if kconfig.get('CONFIG_PCI', 'n') == 'y'
subdir('pci')
endif
if kconfig.get('CONFIG_ACPI', 'n') == 'y'
subdir('acpi')
endif
module_lds = meson.current_source_dir() / 'module.ld'
dev/module.ld
-114
View File
@@ -1,114 +0,0 @@
/* Linker script for kernel modules */
ENTRY(__module_entry)
PHDRS
{
text PT_LOAD FLAGS(5);
rodata PT_LOAD FLAGS(4);
data PT_LOAD FLAGS(6);
dynamic PT_DYNAMIC FLAGS(6);
modversion 0x60000001;
modauthor 0x60000002;
moddesc 0x60000003;
modlicense 0x60000004;
modcmdline 0x60000005;
modexit 0x60000006;
modmatch 0x60000007;
}
SECTIONS
{
. = ALIGN(CONSTANT(MAXPAGESIZE));
/* Code */
.text : {
*(.init .init.*)
*(.text .text.*)
*(.fini .fini.*)
*(.plt .plt.*)
} :text
. = ALIGN(CONSTANT(MAXPAGESIZE));
/* Read-only data */
.rodata : {
*(.rodata .rodata.*)
*(.hash .hash.*)
*(.got .got.*)
*(.got.plt .got.plt.*)
} :rodata
.mod.version : {
KEEP(*(.mod.version))
} :rodata :modversion
.mod.author : {
KEEP(*(.mod.author))
} :rodata :modauthor
.mod.desc : {
KEEP(*(.mod.desc))
} :rodata :moddesc
.mod.license : {
KEEP(*(.mod.license))
} :rodata :modlicense
.cmdline : {
KEEP(*(.cmdline))
} :rodata :modcmdline
.mod.exit : {
KEEP(*(.mod.exit))
} :rodata :modexit
.mod.match : {
KEEP(*(.mod.match))
} :rodata :modmatch
. = ALIGN(CONSTANT(MAXPAGESIZE));
/* Read-Write data */
.data : {
*(.data)
*(.data.*)
} :data
/* Zero-initialized memory */
.bss : {
*(.*bss .*bss.*)
*(COMMON)
} :data
.dynsym : {
*(.dynsym .dynsym.*)
} :data
.dynstr : {
*(.dynstr .dynstr.*)
} :data
.rela.dyn : {
*(.rela.dyn .rela.dyn.*)
} :data
.relr.dyn : {
*(.relr.dyn .relr.dyn.*)
} :data
/* Dynamic tags */
.dynamic : {
*(.dynamic)
} :data :dynamic
/DISCARD/ :
{
*(.comment*)
*(.eh_frame*)
*(.rel.eh_frame*)
*(.gcc_except_table*)
*(.note*)
*(.interp)
}
}
dev/pci/Kconfig
-7
View File
@@ -1,7 +0,0 @@
menu "PCI bus drivers"
config PCI
bool "PCI core drivers"
default y
endmenu

Some files were not shown because too many files have changed in this diff Show More