marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

foo

Browse Source
This commit is contained in:
Marvin
2026-09-28 17:46:45 +02:00
committed by Marvin Friedrich
parent 09926f45ee
commit 4a20a5b0af
62 changed files with 1266 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/x86_64/apic.c
+32
View File
@@ -54,6 +54,10 @@ void lapic_init(struct local_apic* lapic) {
asm_wrmsr(0x1B, apic_msr);
lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
if (lapic->xapic_regs)
lapic->id >>= 24;
// Reset the TPR.
lapic_write_reg(lapic, APIC_REG_TPR, 0);
// Enable APIC bit in the SIVR.
@@ -84,6 +88,34 @@ void lapic_eoi(struct local_apic* lapic) {
lapic_write_reg(lapic, APIC_REG_EOI, 0);
}
void lapic_send_ipi(
struct local_apic* lapic,
uint32_t target,
uint8_t vector,
enum apic_delivery_mode delivery_mode,
enum apic_dest_mode dest_mode,
enum apic_level level,
enum apic_trigger_mode trigger_mode
) {
uint64_t icr = vector;
icr |= (uint64_t)delivery_mode << 8;
icr |= (uint64_t)dest_mode << 11;
icr |= (uint64_t)level << 14;
icr |= (uint64_t)trigger_mode << 15;
if (lapic->xapic_regs)
icr |= (uint64_t)(uint8_t)target << 56;
else
icr |= (uint64_t)target << 32;
lapic_write_reg(lapic, APIC_REG_ICR, icr);
if (lapic->xapic_regs) {
while (lapic_read_reg(lapic, APIC_REG_ICR) & (APIC_DELIVERY_STATUS_PENDING << 12))
asm volatile("pause");
}
}
constexpr uint16_t PIC1_COMMAND_PORT = 0x20;
constexpr uint16_t PIC1_DATA_PORT = 0x21;
constexpr uint16_t PIC2_COMMAND_PORT = 0xA0;
arch/x86_64/gdt.c
+3 -3
View File
@@ -1,13 +1,13 @@
#include <zinnia/percpu.h>
#include <stddef.h>
#include <zinnia/string.h>
#include <stddef.h>
#include <x86_64/asm.h>
#include <x86_64/defs.h>
#include <x86_64/gdt.h>
#include <x86_64/percpu.h>
#include <x86_64/tss.h>
static const struct gdt main_gdt = {
const struct gdt gdt_base = {
.null = {0},
.kernel_code32 = {.val = 0x00cf9b000000ffff},
.kernel_data32 = {.val = 0x00cf93000000ffff},
@@ -26,7 +26,7 @@ static const struct gdt main_gdt = {
void gdt_load() {
struct gdt* gdt = &percpu_ref(arch_data)->gdt;
memcpy(gdt, &main_gdt, sizeof(main_gdt));
memcpy(gdt, &gdt_base, sizeof(gdt_base));
struct tss* tss = &percpu_ref(arch_data)->tss;
tss->iopb = sizeof(struct tss);
arch/x86_64/hpet.c
+1 -1
View File
@@ -35,7 +35,7 @@ static void hpet_reset(struct clock* c) {
static uint64_t hpet_get_elapsed_ns(struct clock* c) {
struct hpet* hpet = containerof(c, struct hpet, clock);
uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
return (counter * (uint64_t)hpet->period / 1'000'000);
return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
}
static struct hpet hpet_clock = {
arch/x86_64/idt.c
+53 -11
View File
@@ -5,6 +5,9 @@
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/usercopy.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <bits/sched.h>
#include <stddef.h>
@@ -36,29 +39,68 @@ static_assert(sizeof(struct idtr) == 10);
void interrupt_handler(struct arch_context* ctx) {
struct percpu* cpu = percpu_get();
struct arch_percpu* arch_cpu = percpu_ref(arch_data);
bool old = irq_set_interrupted(true);
uint64_t isr = ctx->isr;
switch (isr) {
case IDT_PF:
uint64_t cr2;
asm volatile("mov %0, cr2" : "=r"(cr2));
panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
case IDT_IPI_RESCHED:
arch_sched_preempt_disable();
if (arch_sched_preempt_enable()) {
lapic_eoi(&arch_cpu->lapic);
sched_reschedule(&cpu->sched);
const struct vma_fault_info info = {
.ip = ctx->rip,
.addr = cr2,
.caused_by_user = (ctx->cs & CPL_USER) == CPL_USER,
.caused_by_write = ctx->error & (1 << 1),
.caused_by_fetch = ctx->error & (1 << 4),
.page_was_present = ctx->error & (1 << 0),
};
struct task* current = cpu->sched.current;
const bool irqs_were_on = ctx->rflags & RFLAGS_IF;
if (irqs_were_on)
irq_set_state(true);
const bool handled = current && current->space && vma_page_fault(current->space, &info);
if (irqs_were_on)
irq_set_state(false);
if (handled)
break;
if (current && current->usercopy_region) {
struct usercopy_region* region = current->usercopy_region;
if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) {
ctx->rip = (uintptr_t)region->fault_ip;
break;
}
}
panic(
"Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n",
info.caused_by_write ? "write to"
: info.caused_by_fetch ? "execute on"
: "read from",
info.page_was_present ? "present" : "non-present",
info.addr,
info.ip
);
case IDT_SPURIOUS:
break;
case IDT_IPI_RESCHED:
lapic_eoi(&arch_cpu->lapic);
arch_sched_preempt_disable();
if (arch_sched_preempt_enable())
sched_reschedule(&cpu->sched);
break;
default:
if (isr < 0x20)
panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu\n", isr);
struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
assert(line, "Unhandled interrupt on ISR %lu\n", isr);
// TODO: irq_raise(line);
irq_line_raise(line);
}
irq_set_interrupted(old);
}
#define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
arch/x86_64/include/x86_64/apic.h
+11
View File
@@ -55,4 +55,15 @@ enum apic_regs {
void lapic_init(struct local_apic* lapic);
void lapic_eoi(struct local_apic* lapic);
void lapic_send_ipi(
struct local_apic* lapic,
uint32_t target,
uint8_t vector,
enum apic_delivery_mode delivery_mode,
enum apic_dest_mode dest_mode,
enum apic_level level,
enum apic_trigger_mode trigger_mode
);
void pic_disable();
arch/x86_64/include/x86_64/asm.h
+2 -2
View File
@@ -55,13 +55,13 @@ static inline void asm_fxrstor(const void* mem) {
// Saves the FPU state to a region of memory using XSAVE.
// Pointer must be 16-byte aligned.
static inline void asm_xsave(void* mem) {
asm volatile("xsave [%0]" ::"r"(mem) : "memory");
asm volatile("xsave [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
}
// Restores the FPU state from a region of memory using XRSTOR.
// Pointer must be 16-byte aligned.
static inline void asm_xrstor(const void* mem) {
asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
asm volatile("xrstor [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
}
static inline uint8_t asm_inb(uint16_t port) {
arch/x86_64/include/x86_64/gdt.h
+2 -2
View File
@@ -1,7 +1,7 @@
#pragma once
#include <stdint.h>
#include <zinnia/compiler.h>
#include <stdint.h>
union gdt_desc {
uint64_t val;
@@ -50,7 +50,7 @@ struct [[__packed]] gdtr {
struct gdt* base;
};
struct percpu;
extern const struct gdt gdt_base;
// Initializes a GDT on the local CPU.
void gdt_load();
arch/x86_64/include/x86_64/percpu.h
+4
View File
@@ -1,6 +1,7 @@
#pragma once
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <stddef.h>
#include <x86_64/apic.h>
#include <x86_64/gdt.h>
@@ -18,3 +19,6 @@ struct arch_percpu {
};
extern struct arch_percpu arch_data;
// Initializes the calling CPU and makes `cpu` its per-CPU block.
void x86_64_cpu_setup(struct percpu* cpu);
arch/x86_64/include/x86_64/pmap.h
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include <zinnia/pmap.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
arch/x86_64/include/x86_64/smp.h
+6 -31
View File
@@ -1,23 +1,10 @@
#pragma once
#define SMP_GDTR_OFFSET 80
#define SMP_FARJMP_OFFSET 90
#define SMP_TEMP_STACK_OFFSET 98
#define SMP_TEMP_CR3_OFFSET 102
#define SMP_ENTRY_OFFSET 106
#define SMP_HHDM_OFFSET 114
#define SMP_KERNEL32_DS 16
#define SMP_KERNEL64_CS 24
#define SMP_KERNEL64_DS 32
#define SMP_INFO_SIZE 127
#ifndef __ASSEMBLER__
#include <zinnia/compiler.h>
#include <zinnia/utils.h>
#include <stddef.h>
#include <stdint.h>
#include <x86_64/gdt.h>
#include <zinnia/compiler.h>
#include <zinnia/utils.h>
struct [[__packed]] smp_info {
struct gdt gdt;
@@ -28,22 +15,10 @@ struct [[__packed]] smp_info {
uint32_t temp_cr3;
uint64_t entry;
uint64_t hhdm_offset;
uint64_t percpu;
uint32_t lapic_id;
uint8_t booted;
_Atomic(uint8_t) booted;
};
static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr));
static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset));
static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack));
static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3));
static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry));
static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset));
static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32));
static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64));
static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64));
static_assert(SMP_INFO_SIZE == sizeof(struct smp_info));
// Initializes a LAPIC.
void x86_64_smp_init(uint32_t id);
#endif
// Discovers all APs and starts them.
void x86_64_smp_init();
arch/x86_64/irq.c
+5 -3
View File
@@ -8,11 +8,13 @@ bool arch_irq_get_state() {
return flags & RFLAGS_IF;
}
void arch_irq_set_state(bool state) {
bool arch_irq_set_state(bool state) {
bool old = arch_irq_get_state();
if (state)
asm volatile("sti");
asm volatile("sti" ::: "memory");
else
asm volatile("cli");
asm volatile("cli" ::: "memory");
return old;
}
void arch_irq_wait() {
arch/x86_64/mem.c
+1 -1
View File
@@ -32,5 +32,5 @@ uintptr_t arch_mem_mapping_addr() {
}
bool arch_mem_addr_is_user(uintptr_t addr) {
return addr < 0x0000'8000'0000'0000;
return addr < 0x0000'7FFF'FFFF'F000;
}
arch/x86_64/meson.build
-1
View File
@@ -20,7 +20,6 @@ kernel_sources += files(
'smp.c',
'sys.c',
'syscall.c',
'trampoline.S',
'tsc.c',
'usercopy.c',
)
arch/x86_64/percpu.c
+4 -2
View File
@@ -9,6 +9,7 @@
#include <x86_64/hpet.h>
#include <x86_64/idt.h>
#include <x86_64/percpu.h>
#include <x86_64/smp.h>
#include <x86_64/syscall.h>
#include <x86_64/tsc.h>
@@ -43,7 +44,7 @@ static_assert(sizeof(union ia32_star) == 8);
[[__percpu]]
struct arch_percpu arch_data = {0};
static void setup_cpu(struct percpu* cpu) {
void x86_64_cpu_setup(struct percpu* cpu) {
asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
asm_wrmsr(MSR_FS_BASE, 0);
@@ -131,7 +132,8 @@ void arch_percpu_init() {
#endif
// Init the BSP.
setup_cpu(&percpu_bsp);
x86_64_cpu_setup(&percpu_bsp);
// Init any APs.
x86_64_smp_init();
}
arch/x86_64/pmap.c
+1 -17
View File
@@ -1,22 +1,6 @@
#include <zinnia/pmap.h>
#include <zinnia/types.h>
enum : pte_t {
ARCH_FLAG_PRESENT = 1 << 0,
ARCH_FLAG_READ_WRITE = 1 << 1,
ARCH_FLAG_USER_MODE = 1 << 2,
ARCH_FLAG_WRITE_THROUGH = 1 << 3,
ARCH_FLAG_CACHE_DISABLE = 1 << 4,
ARCH_FLAG_ACCESSED = 1 << 5,
ARCH_FLAG_DIRTY = 1 << 6,
ARCH_FLAG_SIZE = 1 << 7,
ARCH_FLAG_GLOBAL = 1 << 8,
ARCH_FLAG_AVAILABLE = 1 << 9,
ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
};
#include <x86_64/pmap.h>
void pte_clear(pte_t* pte) {
*pte = 0;
arch/x86_64/sched.c
+27 -29
View File
@@ -25,30 +25,30 @@ struct task_frame {
static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) {
asm volatile(
"sub rsp, 0x30\n" // Make room for all regs (except RIP).
"mov [rsp + %c0], rbx\n"
"mov [rsp + %c1], rbp\n"
"mov [rsp + %c2], r12\n"
"mov [rsp + %c3], r13\n"
"mov [rsp + %c4], r14\n"
"mov [rsp + %c5], r15\n"
"mov [rsp + %c[rbx]], rbx\n"
"mov [rsp + %c[rbp]], rbp\n"
"mov [rsp + %c[r12]], r12\n"
"mov [rsp + %c[r13]], r13\n"
"mov [rsp + %c[r14]], r14\n"
"mov [rsp + %c[r15]], r15\n"
"mov [rdi], rsp\n" // rdi = old_rsp
"mov rsp, rsi\n" // rsi = new_rsp
"mov rbx, [rsp + %c0]\n"
"mov rbp, [rsp + %c1]\n"
"mov r12, [rsp + %c2]\n"
"mov r13, [rsp + %c3]\n"
"mov r14, [rsp + %c4]\n"
"mov r15, [rsp + %c5]\n"
"mov rbx, [rsp + %c[rbx]]\n"
"mov rbp, [rsp + %c[rbp]]\n"
"mov r12, [rsp + %c[r12]]\n"
"mov r13, [rsp + %c[r13]]\n"
"mov r14, [rsp + %c[r14]]\n"
"mov r15, [rsp + %c[r15]]\n"
"add rsp, 0x30\n"
"mov rax, rdx\n"
"ret" // This will conveniently move us to the RIP we put at this stack entry.
:
: "i"(offsetof(struct task_frame, rbx)),
"i"(offsetof(struct task_frame, rbp)),
"i"(offsetof(struct task_frame, r12)),
"i"(offsetof(struct task_frame, r13)),
"i"(offsetof(struct task_frame, r14)),
"i"(offsetof(struct task_frame, r15))
: [rbx] "i"(offsetof(struct task_frame, rbx)),
[rbp] "i"(offsetof(struct task_frame, rbp)),
[r12] "i"(offsetof(struct task_frame, r12)),
[r13] "i"(offsetof(struct task_frame, r13)),
[r14] "i"(offsetof(struct task_frame, r14)),
[r15] "i"(offsetof(struct task_frame, r15))
: "memory"
);
}
@@ -60,7 +60,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
if (from->user_stack) {
if (from_ctx->fpu_region != nullptr) {
cpu->fpu_save(from_ctx->fpu_region);
from_ctx->ds = asm_read_ds();
from_ctx->es = asm_read_es();
@@ -70,7 +70,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
}
if (to->user_stack) {
if (to_ctx->fpu_region != nullptr) {
cpu->fpu_restore(to_ctx->fpu_region);
asm_write_ds(to_ctx->ds);
asm_write_es(to_ctx->es);
@@ -124,16 +124,14 @@ errno_t arch_task_init(
uintptr_t fpu_base = vm_alloc(fpu_size);
if (!fpu_base)
return ENOMEM;
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
phys_t page;
errno_t s = mem_phys_alloc(1, 0, &page);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
pmap_map(&kernel_vma.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
phys_t pages;
errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages);
if (s) {
vm_free(fpu_base, fpu_size);
return s;
}
for (size_t i = 0; i < fpu_size; i += arch_mem_page_size())
pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
context->fpu_region = (void*)fpu_base;
context->ds = asm_read_ds();
arch/x86_64/smp.c
+342 -7
View File
@@ -1,17 +1,352 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/clock.h>
#include <zinnia/compiler.h>
#include <zinnia/irq.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/string.h>
#include <zinnia/types.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>
#include <uacpi/acpi.h>
#include <uacpi/tables.h>
#include <x86_64/apic.h>
#include <x86_64/defs.h>
#include <x86_64/percpu.h>
#include <x86_64/pmap.h>
#include <x86_64/smp.h>
extern uint8_t smp_trampoline_start[];
extern uint8_t smp_trampoline_data[];
extern uint8_t smp_trampoline_end[];
[[noreturn]]
static void ap_entry(phys_t info) {
// TODO
[[__naked, __used]]
static void smp_trampoline() {
asm volatile(
".pushsection .rodata\n"
".code16\n"
".global smp_trampoline_start\n"
"smp_trampoline_start:\n"
" cli\n"
" cld\n"
" jmp .Lmode16\n"
while (1) {
asm volatile("hlt");
}
".global smp_trampoline_data\n"
"smp_trampoline_data:\n"
" .skip %c[info_size]\n"
".set .Ldata_offset, (smp_trampoline_data - smp_trampoline_start)\n"
".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
".Lmode16:\n"
" mov bx, cs\n"
" shl ebx, 4\n"
" lidtd cs:(.Linvalid_idtr - smp_trampoline_start)\n"
" lgdtd cs:.Lgdtr_offset\n"
" lea eax, [ebx + (.Lmode32 - smp_trampoline_start)]\n"
" mov dword ptr cs:.Lfarjmp_offset, eax\n"
" mov eax, %c[cr0]\n"
" mov cr0, eax\n"
" jmp fword ptr cs:.Lfarjmp_offset\n"
".code32\n"
".Lmode32:\n"
" mov ax, %c[kernel32_ds]\n"
" mov ds, ax\n"
" mov es, ax\n"
" mov fs, ax\n"
" mov gs, ax\n"
" mov ss, ax\n"
" xor eax, eax\n"
" lldt ax\n"
" mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n"
" xor eax, eax\n"
" mov cr4, eax\n"
" or eax, %c[cr4_pae]\n"
" mov cr4, eax\n"
" mov ecx, %c[msr_efer]\n"
" mov eax, %c[efer_lme]\n"
" xor edx, edx\n"
" wrmsr\n"
" mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n"
" mov cr3, eax\n"
" mov eax, cr0\n"
" or eax, %c[cr0_pg]\n"
" mov cr0, eax\n"
" lea eax, [ebx + (.Lmode64 - smp_trampoline_start)]\n"
" push %c[kernel64_cs]\n"
" push eax\n"
" retf\n"
".code64\n"
".Lmode64:\n"
" mov ax, %c[kernel64_ds]\n"
" mov ds, ax\n"
" mov es, ax\n"
" mov fs, ax\n"
" mov gs, ax\n"
" mov ss, ax\n"
" mov ecx, %c[msr_efer]\n"
" rdmsr\n"
" or eax, %c[efer_nxe]\n"
" wrmsr\n"
" xor ebp, ebp\n"
" lea rdi, [rbx + .Ldata_offset]\n"
" mov rax, [rbx + .Lentry_offset]\n"
" add rsp, [rbx + .Lhhdm_offset]\n"
" jmp rax\n"
".Linvalid_idtr:\n"
" .word 0\n"
" .quad 0\n"
".global smp_trampoline_end\n"
"smp_trampoline_end:\n"
".popsection\n"
" ud2\n"
:
: [info_size] "i"(sizeof(struct smp_info)),
[gdtr] "i"(offsetof(struct smp_info, gdtr)),
[farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
[temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
[temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)),
[entry] "i"(offsetof(struct smp_info, entry)),
[hhdm] "i"(offsetof(struct smp_info, hhdm_offset)),
[kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)),
[kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
[kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
[cr0] "i"(CR0_PE | CR0_ET),
[cr0_pg] "i"(CR0_PG),
[cr4_pae] "i"(CR4_PAE),
[msr_efer] "i"(MSR_EFER),
[efer_lme] "i"(MSR_EFER_LME),
[efer_nxe] "i"(MSR_EFER_NXE)
);
}
void x86_64_smp_init(uint32_t id) {}
constexpr size_t AP_BOOT_STACK_SIZE = 0x8000;
constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000;
[[noreturn]]
static void ap_entry(phys_t info_addr) {
// We're still running on the temporary page table.
pmap_set(&kernel_vma.pmap);
struct smp_info* info = HHDM_PTR(info_addr);
struct percpu* cpu = (struct percpu*)info->percpu;
x86_64_cpu_setup(cpu);
irq_lock();
// This task keeps running on the boot stack, but is never scheduled again after the first yield.
sched_init(&cpu->sched);
// Let the BSP know that we're alive.
// `info` must not be touched after this point.
atomic_store_explicit(&info->booted, 1, memory_order_release);
log("smp: CPU %zu online\n", cpu->id);
sched_yield(&cpu->sched);
unreachable();
}
static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
phys_t stack;
if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
log("smp: Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
return false;
}
struct percpu* cpu = percpu_new();
if (!cpu) {
log("smp: Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
return false;
}
log("smp: Starting CPU %zu (LAPIC %u)\n", cpu->id, lapic_id);
memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start);
struct smp_info* info = HHDM_PTR(info_addr);
memcpy(&info->gdt, &gdt_base, sizeof(info->gdt));
info->gdtr.limit = sizeof(struct gdt) - 1;
info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt));
info->farjmp_offset = 0; // Filled in by the trampoline.
info->farjmp_segment = offsetof(struct gdt, kernel_code32);
info->temp_stack = stack + AP_BOOT_STACK_SIZE;
info->temp_cr3 = temp_cr3;
info->entry = (uint64_t)ap_entry;
info->hhdm_offset = mem_hhdm_base;
info->percpu = (uint64_t)cpu;
info->lapic_id = lapic_id;
atomic_init(&info->booted, 0);
struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE);
clock_spin_ns(10'000'000);
lapic_send_ipi(
lapic,
lapic_id,
trampoline >> 12,
APIC_DELIVERY_STARTUP,
APIC_DEST_PHYSICAL,
APIC_LEVEL_ASSERT,
APIC_TRIGGER_EDGE
);
clock_spin_ns(10'000'000);
for (uint64_t waited = 0; !atomic_load_explicit(&info->booted, memory_order_acquire); waited += 1'000'000) {
if (waited >= AP_BOOT_TIMEOUT_NS) {
// The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
log("smp: CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
return false;
}
clock_spin_ns(1'000'000);
}
return true;
}
static errno_t build_temp_table(phys_t* out) {
const size_t pgsz = arch_mem_page_size();
phys_t mem;
errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem);
if (status)
return status;
uint64_t* l4 = HHDM_PTR(mem);
uint64_t* l3 = HHDM_PTR(mem + pgsz);
const phys_t l2_base = mem + 2 * pgsz;
// Use 2 MiB pages.
l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
for (size_t i = 0; i < 4; i++) {
uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz);
l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
for (size_t j = 0; j < 512; j++)
l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE;
}
// Copy over the higher half from the kernel page table.
const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root);
for (size_t i = 256; i < 512; i++)
l4[i] = kernel_l4[i];
*out = mem;
return 0;
}
void x86_64_smp_init() {
const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page\n");
const size_t max_cpus = percpu_max_cpus();
if (max_cpus <= 1)
return;
uacpi_table table;
if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
log("smp: No MADT found, not starting any APs\n");
return;
}
phys_t trampoline;
if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
log("smp: Failed to allocate 20-bit memory for the AP trampoline\n");
uacpi_table_unref(&table);
return;
}
phys_t temp_table;
if (build_temp_table(&temp_table)) {
log("smp: Failed to allocate a temporary page table\n");
mem_phys_free(trampoline, 1);
uacpi_table_unref(&table);
return;
}
const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id;
bool all_booted = true;
size_t num_cpus = 1;
const struct acpi_madt* madt = table.ptr;
const uint8_t* entry = (const uint8_t*)madt->entries;
const uint8_t* end = (const uint8_t*)madt + madt->hdr.length;
while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) {
const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry;
// Don't trust malformed firmware tables.
if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
log("smp: MADT contains a malformed entry\n");
break;
}
uint32_t lapic_id = 0;
bool enabled = false;
switch (hdr->type) {
case ACPI_MADT_ENTRY_TYPE_LAPIC:
const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr;
lapic_id = lapic->id;
enabled = lapic->flags & ACPI_PIC_ENABLED;
break;
case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC:
const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr;
lapic_id = x2apic->id;
enabled = x2apic->flags & ACPI_PIC_ENABLED;
break;
}
if (enabled && lapic_id != bsp_id) {
if (start_ap(trampoline, temp_table, lapic_id))
num_cpus++;
else
all_booted = false;
}
entry += hdr->length;
}
uacpi_table_unref(&table);
// An AP which didn't respond might still be using these.
if (all_booted) {
mem_phys_free(temp_table, 6);
mem_phys_free(trampoline, 1);
}
log("smp: %zu CPU(s) online\n", num_cpus);
}
arch/x86_64/syscall.c
+16 -17
View File
@@ -12,17 +12,17 @@
void arch_syscall_stub() {
asm volatile(
" swapgs\n"
" mov gs:%c0, rsp\n" // user_stack
" mov rsp, gs:%c1\n" // kernel_stack
" mov gs:%c[user_stack], rsp\n"
" mov rsp, gs:%c[kernel_stack]\n"
" cld\n"
"1:\n"
" push %c3\n" // ss
" push gs:%c0\n"
" push r11\n" // rflags
" push %c2\n" // cs
" push rcx\n" // rip
" push 0x00\n" // arch_context::error
" push 0x00\n" // arch_context::isr
" push %c[user_data]\n" // ss
" push gs:%c[user_stack]\n"
" push r11\n" // rflags
" push %c[user_code64]\n" // cs
" push rcx\n" // rip
" push 0x00\n" // arch_context::error
" push 0x00\n" // arch_context::isr
" push rax\n"
" push rbx\n"
" push rcx\n"
@@ -39,8 +39,8 @@ void arch_syscall_stub() {
" push r14\n"
" push r15\n"
" xor rbp, rbp\n"
" mov rdi, rsp\n" // SysV ABI first argument.
" call %4\n" // Call syscall handler.
" mov rdi, rsp\n" // SysV ABI first argument.
" call syscall_dispatch\n" // Call syscall handler.
" cli\n"
" pop r15\n"
" pop r14\n"
@@ -59,14 +59,13 @@ void arch_syscall_stub() {
" pop rax\n"
" add rsp, 0x10\n" // Skip .error and .isr fields
"2:\n"
" mov rsp, gs:%c0\n" // user_stack
" mov rsp, gs:%c[user_stack]\n"
" swapgs\n"
" sysretq\n"
:
: "i"(offsetof(struct percpu, user_stack)), // 0
"i"(offsetof(struct percpu, kernel_stack)), // 1
"i"(offsetof(struct gdt, user_code64) | CPL_USER), // 2
"i"(offsetof(struct gdt, user_data) | CPL_USER), // 3
"r"(syscall_dispatch) // 4
: [user_stack] "i"(offsetof(struct percpu, user_stack)),
[kernel_stack] "i"(offsetof(struct percpu, kernel_stack)),
[user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER),
[user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER)
);
}
arch/x86_64/trampoline.S
-107
View File
@@ -1,107 +0,0 @@
#include <x86_64/smp.h>
#include <x86_64/defs.h>
.intel_syntax noprefix
.global smp_trampoline_start
.global smp_trampoline_end
.section .rodata
.code16
smp_trampoline_start:
cli
cld
jmp 1f
smp_trampoline_data:
.skip SMP_INFO_SIZE
.set data_offset, (smp_trampoline_data - smp_trampoline_start)
.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET)
.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET)
.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET)
.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET)
.set entry_offset, (data_offset + SMP_ENTRY_OFFSET)
.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET)
1:
mov bx, cs
shl ebx, 4
.set idtr_offset, (invalid_idtr - smp_trampoline_start)
lidtd cs:idtr_offset
lgdtd cs:gdtr_offset
.set mode32_offset, (mode32 - smp_trampoline_start)
lea eax, [ebx + mode32_offset]
mov dword ptr cs:farjmp_offset, eax
mov eax, CR0_PE | CR0_ET
mov cr0, eax
jmp fword ptr cs:farjmp_offset
.code32
mode32:
mov ax, SMP_KERNEL32_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
xor eax, eax
lldt ax
mov esp, dword ptr [ebx + temp_stack_offset]
xor eax, eax
mov cr4, eax
or eax, CR4_PAE
mov cr4, eax
mov ecx, MSR_EFER
mov eax, MSR_EFER_LME
xor edx, edx
wrmsr
mov eax, dword ptr [ebx + temp_cr3_offset]
mov cr3, eax
mov eax, cr0
or eax, CR0_PG
mov cr0, eax
.set mode64_offset, (mode64 - smp_trampoline_start)
lea eax, [ebx + mode64_offset]
push SMP_KERNEL64_CS
push eax
retf
.code64
mode64:
mov ax, SMP_KERNEL64_DS
mov ds, ax
mov es, ax
mov fs, ax
mov gs, ax
mov ss, ax
mov ecx, MSR_EFER
rdmsr
or eax, MSR_EFER_NXE
wrmsr
xor ebp, ebp
lea rdi, [rbx + data_offset]
mov rax, [rbx + entry_offset]
add rsp, [rbx + hhdm_offset]
jmp rax
invalid_idtr:
.word 0
.quad 0
smp_trampoline_end:
arch/x86_64/tsc.c
+3 -2
View File
@@ -53,8 +53,9 @@ void tsc_init() {
// Check for the TSC info leaf.
bool has_tsc_leaf = false;
struct cpuid tsc_leaf = {0};
if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) {
if (asm_cpuid(0, 0).eax >= 0x15) {
tsc_leaf = asm_cpuid(0x15, 0);
has_tsc_leaf = true;
}
if (clock_available()) {
@@ -69,7 +70,7 @@ void tsc_init() {
// since at this point we have at least the HPET timer.
if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
log("x86_64/tsc: Calibrating TSC using CPUID 0x15");
tsc_freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
} else {
log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information");
return;
arch/x86_64/usercopy.c
+14 -8
View File
@@ -1,7 +1,12 @@
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <arch/usercopy.h>
#include <x86_64/percpu.h>
void arch_usercopy_allow(bool state) {
if (!percpu_ref(arch_data)->can_smap)
return;
if (state) {
asm volatile("stac" ::: "memory");
} else {
@@ -24,7 +29,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
asm volatile(
// Setup regs for `rep movsb`.
" xchg rcx, rdx\n"
" mov rax, [rip + %0]\n"
" lea rax, [rip + %0]\n"
" xchg [rdx], rax\n"
".global x86_64_read_start\n"
@@ -41,7 +46,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
".global x86_64_read_fault\n"
"x86_64_read_fault:\n"
" xor rax, rax\n"
" xchg [rdx], rax\n"
" mov [rdx], rax\n"
" ret\n"
:
: "i"(&read_region)
@@ -64,7 +69,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
asm volatile(
// Setup regs for `rep movsb`.
" xchg rcx, rdx\n"
" mov rax, [rip + %0]\n"
" lea rax, [rip + %0]\n"
" mov [rdx], rax\n"
".global x86_64_write_start\n"
@@ -81,7 +86,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
".global x86_64_write_fault\n"
"x86_64_write_fault:\n"
" xor rax, rax\n"
" xchg [rdx], rax\n"
" mov [rdx], rax\n"
" ret\n"
:
: "i"(&write_region)
@@ -102,18 +107,19 @@ static const struct usercopy_region strlen_region = {
[[__naked]]
bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
asm volatile(
" mov rax, [rip + %0]\n"
" lea rax, [rip + %0]\n"
" xchg [rcx], rax\n"
".global x86_64_strlen_start\n"
" x86_64_strlen_start:\n"
" xor r8, r8\n"
".Lloop:\n"
" cmp r8, rsi\n"
" je .Lleave\n"
" cmp byte ptr [rdi + r8], 0\n"
" je .Lleave\n"
" inc r8\n"
" cmp rsi, r8\n"
" jne .Lloop\n"
" jmp .Lloop\n"
".Lleave:\n"
" mov [rdx], r8\n"
@@ -127,7 +133,7 @@ bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struc
".global x86_64_strlen_fault\n"
"x86_64_strlen_fault:\n"
" xor rax, rax\n"
" xchg [rcx], rax\n"
" mov [rcx], rax\n"
" ret\n"
:
: "i"(&strlen_region)
boot/cmdline.c
+2
View File
@@ -65,6 +65,8 @@ void cmdline_parse(const char* cmdline) {
while (opt < __ld_cmdline_end) {
const struct cmdline_option* current = *opt;
if (!strcmp(current->name, name)) {
if (value == nullptr)
break;
switch (current->option_type) {
case CMDLINE_STRING:
current->func_str(value);
boot/limine/entry.c
+10 -3
View File
@@ -39,7 +39,7 @@ static struct boot_file files[8];
void kernel_entry() {
kernel_early_init();
struct boot_info info;
struct boot_info info = {};
struct limine_memmap_response* mm_res = memmap_request.response;
struct limine_module_response* module_res = module_request.response;
@@ -76,7 +76,8 @@ void kernel_entry() {
for (size_t i = 0; i < info.num_files; i++) {
files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset;
files[i].length = module_res->modules[i]->size;
strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path));
strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path) - 1);
files[i].path[sizeof(files[i].path) - 1] = 0;
}
}
@@ -85,7 +86,13 @@ void kernel_entry() {
info.virt_base = (uintptr_t)exec_res->virtual_base;
info.hhdm_base = (uintptr_t)hhdm_request.response->offset;
info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base;
#if CONFIG_ACPI
if (rsdp_request.response) {
info.rsdp = (phys_t)rsdp_request.response->address;
if (info.rsdp >= info.hhdm_base)
info.rsdp -= info.hhdm_base;
}
#endif
kernel_main(&info);
}
boot/main.c
+3 -3
View File
@@ -79,13 +79,13 @@ void kernel_main(struct boot_info* info) {
mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
percpu_init();
sched_init(&percpu_get()->sched);
#if CONFIG_ACPI
acpi_init(info);
#endif
percpu_init();
sched_init(&percpu_get()->sched);
struct task* main_task;
task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
sched_add_task(&percpu_get()->sched, main_task);
core/clock.c
+20 -10
View File
@@ -1,4 +1,3 @@
#include <zinnia/atomic.h>
#include <zinnia/clock.h>
#include <zinnia/compiler.h>
#include <zinnia/print.h>
@@ -9,16 +8,25 @@ static struct clock* active_clock = nullptr;
static uint64_t counter_base = 0;
static struct spinlock lock = {};
static uint64_t clock_get_elapsed_ns_locked() {
if (__unlikely(!active_clock))
return 0;
return counter_base + active_clock->get_elapsed_ns(active_clock);
}
bool clock_available() {
return active_clock != nullptr;
spin_lock(&lock);
bool available = active_clock != nullptr;
spin_unlock(&lock);
return available;
}
uint64_t clock_get_elapsed_ns() {
struct clock* active = atomic_load(&active_clock, ATOMIC_RELAXED);
if (__unlikely(!active))
return 0;
return atomic_load(&counter_base, ATOMIC_RELAXED) + active->get_elapsed_ns(active);
spin_lock(&lock);
uint64_t elapsed = clock_get_elapsed_ns_locked();
spin_unlock(&lock);
return elapsed;
}
bool clock_switch(struct clock* clock) {
@@ -33,16 +41,18 @@ bool clock_switch(struct clock* clock) {
log("clock: Switching to clock source \"%s\"\n", clock->name);
uint64_t elapsed = clock_get_elapsed_ns();
counter_base += elapsed;
uint64_t elapsed = clock_get_elapsed_ns_locked();
clock->reset(clock);
counter_base = elapsed;
active_clock = clock;
spin_unlock(&lock);
return true;
}
void clock_spin_ns(uint64_t ns) {
if (active_clock == nullptr) {
if (!clock_available()) {
log("clock: Unable to sleep for %lu ns, no clock available!\n", ns);
return;
}
core/console.c
+3 -1
View File
@@ -35,8 +35,10 @@ void console_init() {
active = __ld_console_start[0];
if (active) {
if (active->init(active) == false)
if (active->init(active) == false) {
active = nullptr;
return;
}
log("console: Logging on console \"%s\"\n", active->name);
}
core/elf.c
+89 -28
View File
@@ -7,11 +7,12 @@
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/string.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <zinnia/vmo.h>
#include <bits/elf.h>
#include <zinnia/string.h>
#include <stdckdint.h>
struct exec_info {
struct vma* space;
@@ -21,11 +22,15 @@ struct exec_info {
};
errno_t elf_load(struct exec_info* info, struct task** out) {
errno_t status;
errno_t err;
size_t read;
struct elf_ehdr ehdr = {};
vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
if (err)
return err;
if (read != sizeof(ehdr))
return ENOEXEC;
// Check if the file is an ELF.
if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
@@ -40,13 +45,48 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
return EINVAL;
if (ehdr.e_type != ET_EXEC)
return EINVAL;
if (!arch_mem_addr_is_user(ehdr.e_entry))
return ENOEXEC;
const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
uintptr_t phdrs_end;
if (ehdr.e_phentsize < sizeof(struct elf_phdr) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
return ENOEXEC;
if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size))
return ENOEXEC;
const size_t page_size = arch_mem_page_size();
uintptr_t phdr_addr = 0;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
struct elf_phdr phdr = {};
vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
if (err)
return err;
if (read != sizeof(phdr))
return ENOEXEC;
if (phdr.p_type == PT_PHDR)
phdr_addr = phdr.p_vaddr;
if (phdr.p_type == PT_LOAD) {
if (phdr.p_memsz == 0)
continue;
if (phdr.p_filesz > phdr.p_memsz)
return ENOEXEC;
uintptr_t seg_end;
if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz))
return ENOEXEC;
if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1))
return ENOEXEC;
if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
return ENOEXEC;
if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset);
enum prot_flags prot = 0;
if (phdr.p_flags & PF_R)
prot |= PROT_READ;
@@ -55,8 +95,6 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
if (phdr.p_flags & PF_X)
prot |= PROT_EXEC;
assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
const uintptr_t map_address = phdr.p_vaddr - misalign;
const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
@@ -64,18 +102,21 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
// Copy the file data into its own mapping.
struct paged_vmo* phdr_obj;
status = vmo_new_phys(&phdr_obj);
if (status)
return status;
err = vmo_new_phys(&phdr_obj);
if (err)
return err;
status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
if (status)
return status;
err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
if (err) {
vmo_unref(&phdr_obj->object);
return err;
}
// We map more than we copied so the rest is filled with zeroed pages.
status = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
if (status)
return status;
err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
vmo_unref(&phdr_obj->object);
if (err)
return err;
}
}
@@ -85,9 +126,9 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
// Allocate stack.
struct paged_vmo* stack;
status = vmo_new_phys(&stack);
if (status)
return status;
err = vmo_new_phys(&stack);
if (err)
return err;
// Fill in stack info.
uintptr_t stack_off = stack_size;
@@ -101,8 +142,21 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
num_argv++;
}
size_t strings_size = 0;
for (size_t env = 0; env < num_envp; env++)
strings_size += strlen(info->envp[env]) + 1;
for (size_t arg = 0; arg < num_argv; arg++)
strings_size += strlen(info->argv[arg]) + 1;
uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
mem_free(envp_offsets);
mem_free(argv_offsets);
vmo_unref(&stack->object);
return ENOMEM;
}
for (size_t env = 0; env < num_envp; env++) {
const char nul = 0;
@@ -153,22 +207,24 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
if (phdr_addr)
WRITE_AUXV(AT_PHDR, phdr_addr);
// envp pointers.
stack_off -= sizeof(uintptr_t);
const uintptr_t zero = 0;
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
for (size_t env = 0; env < num_envp; env++) {
for (size_t env = num_envp; env > 0; env--) {
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr);
vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr);
}
// argv pointers.
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
for (size_t arg = 0; arg < num_argv; arg++) {
for (size_t arg = num_argv; arg > 0; arg--) {
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr);
vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr);
}
mem_free(envp_offsets);
@@ -179,19 +235,24 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
uintptr_t argc = num_argv;
vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
status = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
if (status)
return status;
err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
vmo_unref(&stack->object);
if (err)
return err;
struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
if (!start_info)
return ENOMEM;
start_info->ip = ehdr.e_entry;
start_info->sp = highest - stack_size + stack_off;
start_info->arg = 0;
struct task* result;
status = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
if (status)
return status;
err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
if (err) {
mem_free(start_info);
return err;
}
*out = result;
return 0;
core/irq.c
+47 -19
View File
@@ -1,10 +1,15 @@
#include <zinnia/atomic.h>
#include <zinnia/assert.h>
#include <zinnia/irq.h>
#include <zinnia/list.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <arch/irq.h>
void irq_set_state(bool state) {
arch_irq_set_state(state);
[[__percpu]]
struct irq_percpu irq_data;
bool irq_set_state(bool state) {
return arch_irq_set_state(state);
}
bool irq_get_state() {
@@ -16,26 +21,49 @@ void irq_wait() {
}
void irq_lock() {
struct irq_percpu* cpu = &percpu_get()->irq;
if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE))
return;
arch_irq_set_state(false);
atomic_fetch_add(&percpu_get()->irq.level, 1, ATOMIC_ACQ_REL);
bool prev = arch_irq_set_state(false);
struct irq_percpu* state = percpu_ref(irq_data);
if (state->depth++ == 0)
state->saved_state = prev;
}
void irq_unlock() {
struct irq_percpu* cpu = &percpu_get()->irq;
if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE))
return;
uint32_t old_level = atomic_fetch_sub(&cpu->level, 1, ATOMIC_ACQ_REL);
// If it was 1, the new IRQ level is now 0.
if (old_level == 1) {
struct irq_percpu* state = percpu_ref(irq_data);
if (--state->depth == 0 && state->saved_state)
arch_irq_set_state(true);
}
}
bool irq_set_interrupted(bool is_interrupted) {
return atomic_exchange(&percpu_get()->irq.in_interrupt, is_interrupted, ATOMIC_RELEASE);
void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
irq_lock();
line->mode = line->set_config(line, trigger, polarity);
irq_unlock();
}
void irq_line_raise(struct irq_line* line) {
assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
irq_lock();
line->is_busy = true;
bool claimed = false;
struct irq_handler* handler;
list_foreach(handler, &line->handlers, next) {
if (handler->raise(handler) == IRQ_STATUS_HANDLED)
claimed = true;
}
if (!claimed)
log("irq: Spurious interrupt\n");
if ((line->mode & IRQ_MODE_EOI) && line->eoi)
line->eoi(line);
line->is_busy = false;
irq_unlock();
}
void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
irq_lock();
list_insert_head(&line->handlers, handler, next);
irq_unlock();
}
core/lib/string.c
+7 -6
View File
@@ -24,15 +24,14 @@ size_t strnlen(const char* str, size_t len) {
[[__hot]]
int memcmp(const void* s1, const void* s2, size_t size) {
int diff = 0;
const uint8_t* s1ptr = s1;
const uint8_t* s2ptr = s2;
for (size_t i = 0; i < size; i++) {
if (s1ptr[i] != s2ptr[i])
diff++;
return s1ptr[i] - s2ptr[i];
}
return diff;
return 0;
}
[[__hot]]
@@ -103,7 +102,7 @@ void* memset(void* dest, int value, size_t n) {
}
size_t* qword_dest = (size_t*)d;
size_t qword_value = value;
size_t qword_value = (uint8_t)value;
qword_value |= (qword_value << 8);
qword_value |= (qword_value << 16);
qword_value |= (qword_value << 32);
@@ -126,8 +125,10 @@ void* memset(void* dest, int value, size_t n) {
}
char* strncpy(char* restrict dst, const char* restrict src, size_t len) {
size_t src_len = strnlen(src, len) + 1;
return memcpy(dst, src, MIN(len, src_len));
size_t src_len = strnlen(src, len);
memcpy(dst, src, src_len);
memset(dst + src_len, 0, len - src_len);
return dst;
}
int strncmp(const char* str1, const char* str2, size_t len) {
core/mem/init.c
+3 -2
View File
@@ -64,10 +64,11 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
mem_hhdm_base = tmp_hhdm;
assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
tailq_init(&kernel_vma.mappings);
// text
log("mem: Mapping text segment at %p\n", __ld_text_start);
for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vma.pmap,
(uintptr_t)p,
@@ -116,7 +117,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
for (size_t i = 0; i < map_len; i++) {
if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
continue;
for (size_t p = 0; p <= map[i].length; p += pgsz) {
for (size_t p = 0; p < map[i].length; p += pgsz) {
uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
phys_t paddr = (phys_t)(map[i].address + p);
errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
core/mem/pmap.c
+11 -1
View File
@@ -17,6 +17,16 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
if (status)
return status;
pte_t* root = HHDM_PTR(addr);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = num_entries / 2; i < num_entries; i++) {
phys_t dir;
status = mem_phys_alloc(1, flags, &dir);
if (status)
return status;
root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
}
struct pmap result = {
.root = addr,
.lock = (struct spinlock){0},
@@ -71,7 +81,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
if (pte_is_present(pte)) {
// Get the next level.
*pte = pte_build(pte_address(pte), level_flags, CACHE_ANY);
*pte |= pte_build(0, level_flags, CACHE_ANY);
current_head = HHDM_PTR(pte_address(pte));
} else {
// If the current level isn't present, we can skip the rest.
core/mem/pmm.c
+53 -12
View File
@@ -25,7 +25,7 @@ static struct phys_mem* bump_region = nullptr;
static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
const size_t bytes = num_pages * arch_mem_page_size();
if (((intptr_t)bump_mem.length - bytes) <= 0)
if (bytes > bump_mem.length)
return ENOMEM;
*out = bump_mem.address;
@@ -76,7 +76,7 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
while (it) {
const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
// If the region is too high up or doesn't have enough pages left, continue searching.
if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) {
prev_it = it;
it = it->freelist.next;
continue;
@@ -96,11 +96,26 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
*out = page_addr;
}
// If not, we can just shrink the block.
else {
else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) {
it->freelist.count -= num_pages;
if (__likely(out))
*out = page_addr + (it->freelist.count * arch_mem_page_size());
} else {
struct page* rest = it + num_pages;
rest->freelist.count = it->freelist.count - num_pages;
rest->freelist.next = it->freelist.next;
if (prev_it)
prev_it->freelist.next = rest;
else
pmm_head = rest;
it->freelist.next = nullptr;
it->freelist.count = 0;
if (__likely(out))
*out = page_addr;
}
goto success;
}
@@ -118,13 +133,41 @@ success:
return 0;
}
static void freelist_insert(struct page* page, size_t num_pages) {
struct page* prev = nullptr;
struct page* next = pmm_head;
while (next && next < page) {
prev = next;
next = next->freelist.next;
}
page->freelist.count = num_pages;
page->freelist.next = next;
if (next && page + page->freelist.count == next) {
page->freelist.count += next->freelist.count;
page->freelist.next = next->freelist.next;
next->freelist.next = nullptr;
next->freelist.count = 0;
}
if (prev && prev + prev->freelist.count == page) {
prev->freelist.count += page->freelist.count;
prev->freelist.next = page->freelist.next;
page->freelist.next = nullptr;
page->freelist.count = 0;
} else if (prev) {
prev->freelist.next = page;
} else {
pmm_head = page;
}
}
static errno_t freelist_free(phys_t addr, size_t num_pages) {
spin_lock(&pmm_lock);
struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
page->freelist.count = num_pages;
page->freelist.next = pmm_head;
pmm_head = page;
freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages);
pmm_total_free += num_pages;
spin_unlock(&pmm_lock);
return 0;
@@ -141,11 +184,9 @@ void mem_phys_init(struct phys_mem* map, size_t length) {
if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
continue;
struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
page->freelist.count = map[i].length / arch_mem_page_size();
page->freelist.next = pmm_head;
pmm_head = page;
pmm_total_free += page->freelist.count;
const size_t num_pages = map[i].length / arch_mem_page_size();
freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages);
pmm_total_free += num_pages;
}
const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
core/mem/slab.c
+4 -4
View File
@@ -24,8 +24,6 @@ typedef struct {
static struct slab slabs[8] = {0};
static errno_t slab_new(struct slab* slab, size_t size) {
slab->lock = (struct spinlock){0};
// Allocate a new page for the head.
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
@@ -68,8 +66,10 @@ void slab_init(void) {
static void* slab_do_alloc(struct slab* slab) {
spin_lock(&slab->lock);
if (__unlikely(slab->head == nullptr))
slab_new(slab, slab->ent_size);
if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) {
spin_unlock(&slab->lock);
return nullptr;
}
void** old_free = slab->head;
slab->head = *old_free;
core/mem/usercopy.c
+20
View File
@@ -1,8 +1,18 @@
#include <zinnia/percpu.h>
#include <zinnia/usercopy.h>
#include <arch/mem.h>
#include <arch/usercopy.h>
static bool is_user_range(uintptr_t addr, size_t len) {
if (len == 0)
return arch_mem_addr_is_user(addr);
return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1);
}
bool usercopy_read(void* dst, const __user void* src, size_t len) {
if (!is_user_range((uintptr_t)src, len))
return false;
arch_usercopy_allow(true);
bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
@@ -10,6 +20,9 @@ bool usercopy_read(void* dst, const __user void* src, size_t len) {
}
bool usercopy_write(__user void* dst, const void* src, size_t len) {
if (!is_user_range((uintptr_t)dst, len))
return false;
arch_usercopy_allow(true);
bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
@@ -17,6 +30,13 @@ bool usercopy_write(__user void* dst, const void* src, size_t len) {
}
bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
if (!arch_mem_addr_is_user((uintptr_t)str))
return false;
size_t unused;
if (len == nullptr)
len = &unused;
arch_usercopy_allow(true);
bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
core/mem/virt.c
+4 -3
View File
@@ -1,18 +1,19 @@
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <stdatomic.h>
#include <stddef.h>
struct page* vm_pfndb = nullptr;
// TODO: Replace with actual allocator.
static size_t mapping_offset = 0;
static _Atomic(size_t) mapping_offset = 0;
uintptr_t vm_alloc(size_t length) {
uintptr_t base = arch_mem_mapping_addr();
size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE);
size_t offset =
atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire);
return base + offset;
}
core/mem/vma.c
+101 -41
View File
@@ -5,6 +5,7 @@
#include <zinnia/pmap.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <stdckdint.h>
// Try to create a new, empty virtual address space.
errno_t vma_new(struct vma** out) {
@@ -22,6 +23,8 @@ errno_t vma_new(struct vma** out) {
if (res)
goto err1;
tailq_init(&vmspace->mappings);
*out = vmspace;
return 0;
@@ -32,20 +35,90 @@ err0:
}
// Delete a virtual address space.
void vma_free(struct vma* v) {}
void vma_free(struct vma* v) {
struct vma_mapping* mapping;
while ((mapping = tailq_first(&v->mappings)) != nullptr) {
tailq_remove(&v->mappings, mapping, link);
vmo_unref(mapping->object);
mem_free(mapping);
}
}
// Remove any mappings in the given address range.
// May fail if there is no memory and a mapping needs to be split.
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size);
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
panic("TODO");
}
// Change the protection of an existing mapping.
// May fail if there is no memory and a mapping needs to be split.
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags);
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
panic("TODO");
}
static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (page >= m->start_page && page < m->end_page)
return m;
}
return nullptr;
}
static bool access_allowed(enum prot_flags prot, const struct vma_fault_info* info) {
if (info->caused_by_write)
return prot & PROT_WRITE;
if (info->caused_by_fetch)
return prot & PROT_EXEC;
return prot & (PROT_READ | PROT_WRITE);
}
static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
enum pte_flags pf = 0;
if (prot & PROT_READ)
pf |= PTE_READ;
if (prot & PROT_WRITE)
pf |= PTE_WRITE;
if (prot & PROT_EXEC)
pf |= PTE_EXEC;
if (v->pmap.is_user)
pf |= PTE_USER;
return pf;
}
// Try to handle a page fault.
bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
// TODO
return false;
const size_t page_size = arch_mem_page_size();
const size_t faulty_page = info->addr / page_size;
spin_lock(&v->lock);
struct vma_mapping* mapping = find_mapping(v, faulty_page);
if (!mapping || !access_allowed(mapping->prot, info)) {
spin_unlock(&v->lock);
return false;
}
struct vmo* object = mapping->object;
const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
const enum prot_flags prot = mapping->prot;
vmo_ref(object);
spin_unlock(&v->lock);
struct page* page;
if (object->get_page(object, object_page, &page)) {
vmo_unref(object);
return false;
}
spin_lock(&v->lock);
mapping = find_mapping(v, faulty_page);
const bool still_valid = mapping && mapping->object == object &&
(faulty_page - mapping->start_page) + mapping->offset_page == object_page &&
access_allowed(mapping->prot, info);
if (still_valid) {
errno_t status =
pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
assert(status == 0, "Failed to map a demand-loaded page\n");
}
spin_unlock(&v->lock);
vmo_unref(object);
return true;
}
errno_t vma_map(
@@ -56,44 +129,31 @@ errno_t vma_map(
enum prot_flags flags,
uintptr_t vmo_offset
) {
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
size_t start_page = addr / page_size;
const size_t addr_offset = addr % page_size;
const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size;
size_t end_page = start_page + pages;
if (!vma || !vmo)
return EINVAL;
const size_t first_vmo_page = vmo_offset / page_size;
uintptr_t end;
if (ckd_add(&end, addr, len))
return ENOMEM;
for (size_t p = start_page; p < end_page; ++p) {
const size_t page_index = p - start_page;
struct page* page_obj;
errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj);
if (status != 0)
return status;
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
phys_t paddr = (phys_t)vm_page_addr(page_obj);
struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
if (!mapping)
return ENOMEM;
enum pte_flags pf = 0;
if (flags & PROT_READ)
pf |= PTE_READ;
if (flags & PROT_WRITE)
pf |= PTE_WRITE;
if (flags & PROT_EXEC)
pf |= PTE_EXEC;
if (vma->pmap.is_user)
pf |= PTE_USER;
vmo_ref(vmo);
mapping->start_page = addr / page_size;
mapping->end_page = ALIGN_UP(end, page_size) / page_size;
mapping->offset_page = vmo_offset / page_size;
mapping->object = vmo;
mapping->prot = flags;
status = pmap_map(&vma->pmap, p * page_size, paddr, pf, CACHE_ANY);
if (status != 0)
return status;
}
spin_lock(&vma->lock);
tailq_insert_head(&vma->mappings, mapping, link);
spin_unlock(&vma->lock);
return 0;
}
core/mem/vmo.c
+53 -3
View File
@@ -5,6 +5,15 @@
#include <zinnia/vmo.h>
#include <zinnia/string.h>
void vmo_ref(struct vmo* obj) {
refcount_inc(&obj->rc);
}
void vmo_unref(struct vmo* obj) {
if (refcount_dec(&obj->rc))
obj->destroy(obj);
}
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
@@ -104,14 +113,17 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page
// Try to find the requested page in the cache.
struct page_list* iter;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
// We found a cached page!
*out = iter->value;
spin_unlock(&paged->lock);
return 0;
}
spin_unlock(&paged->lock);
// Get page from out pager if we don't have one already.
struct page* new_page;
@@ -120,26 +132,57 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page
return status;
struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
if (!entry)
if (!entry) {
paged->source.put_page(&paged->source, offset_idx, new_page);
return ENOMEM;
}
entry->offset = offset_idx;
entry->value = new_page;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
*out = iter->value;
spin_unlock(&paged->lock);
mem_free(entry);
paged->source.put_page(&paged->source, offset_idx, new_page);
return 0;
}
// Add page to the object cache.
list_insert_head(&paged->cache, entry, next);
spin_unlock(&paged->lock);
*out = new_page;
return 0;
}
static void paged_destroy(struct vmo* vmo) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
struct page_list* iter = list_first(&paged->cache);
while (iter) {
struct page_list* next = list_next(iter, next);
paged->source.put_page(&paged->source, iter->offset, iter->value);
mem_free(iter);
iter = next;
}
mem_free(paged);
}
errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
if (!result)
return ENOMEM;
refcount_init(&result->object.rc);
result->source = pager;
result->object.get_page = paged_get_page;
result->object.destroy = paged_destroy;
*out = result;
return 0;
@@ -156,8 +199,7 @@ static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct p
}
static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
// We do nothing here.
return 0;
return mem_phys_free(vm_page_addr(page), 1);
}
static const struct pager_ops phys_pager = {
@@ -177,16 +219,24 @@ struct phys_range_vmo {
static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
if (offset_idx >= p->len / arch_mem_page_size())
return EINVAL;
*out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
return 0;
}
static void phys_range_destroy(struct vmo* vmo) {
mem_free(containerof(vmo, struct phys_range_vmo, object));
}
errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
if (!vmo)
return ENOMEM;
refcount_init(&vmo->object.rc);
vmo->object.get_page = phys_range_get_page;
vmo->object.destroy = phys_range_destroy;
vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
vmo->len = ALIGN_UP(length, arch_mem_page_size());
core/meson.build
+1
View File
@@ -6,6 +6,7 @@ kernel_sources += files(
'panic.c',
'percpu.c',
'print.c',
'refcount.c',
'spin.c',
'wait.c',
)
core/percpu.c
+32 -2
View File
@@ -1,6 +1,22 @@
#include <zinnia/alloc.h>
#include <zinnia/cmdline.h>
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <zinnia/string.h>
#include <zinnia/utils.h>
#include <arch/percpu.h>
#include <stdatomic.h>
extern uint8_t __ld_percpu_start[];
extern uint8_t __ld_percpu_end[];
static _Atomic(size_t) num_cpus = 1;
static size_t max_cpus = SIZE_MAX;
static void smp_option(uintptr_t value) {
max_cpus = value;
}
CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option);
[[__used, __section(".percpu.init")]]
struct percpu percpu_bsp = {
@@ -10,8 +26,22 @@ struct percpu percpu_bsp = {
};
struct percpu* percpu_new() {
// TODO
return nullptr;
const size_t size = __ld_percpu_end - __ld_percpu_start;
const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
phys_t mem;
if (mem_phys_alloc(num_pages, 0, &mem))
return nullptr;
struct percpu* cpu = HHDM_PTR(mem);
cpu->self = cpu;
cpu->id = atomic_fetch_add_explicit(&num_cpus, 1, memory_order_relaxed);
cpu->online = false;
return cpu;
}
size_t percpu_max_cpus() {
return max_cpus;
}
struct percpu* percpu_get() {
core/print.c
+65 -42
View File
@@ -10,6 +10,11 @@
typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) {
for (size_t i = 0; i < count; i++)
callback(ctx, &c, 1);
}
static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
// Amount of bytes written.
int32_t written = 0;
@@ -37,7 +42,7 @@ static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_
bool blank_sign = false;
bool write_prefix = false;
bool pad_zero = false;
bool has_width = false;
bool left_align = false;
size_t width = 0;
bool has_precision = false;
size_t precision = 0;
@@ -68,6 +73,11 @@ check_fmt:
fmt++;
goto check_fmt;
}
case '-': {
left_align = true;
fmt++;
goto check_fmt;
}
// Width
case '1':
case '2':
@@ -78,14 +88,11 @@ check_fmt:
case '7':
case '8':
case '9': {
has_width = true;
char number[10 + 1] = {0};
size_t idx = 0;
width = 0;
while (*fmt >= '0' && *fmt <= '9') {
number[idx++] = *fmt;
width = width * 10 + (*fmt - '0');
fmt++;
}
width = atolu(number, 10);
goto check_fmt;
}
@@ -93,16 +100,19 @@ check_fmt:
case '.': {
has_precision = true;
fmt++;
precision = 0;
if (*fmt == '*') {
precision = (char)va_arg(args, int32_t);
const int32_t arg = va_arg(args, int32_t);
if (arg < 0)
has_precision = false;
else
precision = arg;
fmt++;
} else {
char number[10 + 1] = {0};
size_t idx = 0;
while (*fmt >= '0' && *fmt <= '9') {
number[idx++] = *fmt;
precision = precision * 10 + (*fmt - '0');
fmt++;
}
precision = atolu(number, 10);
}
goto check_fmt;
}
@@ -133,10 +143,15 @@ check_fmt:
// Character
case 'c': {
char c = (char)va_arg(args, int32_t);
const size_t pad = width > 1 ? width - 1 : 0;
if (!left_align)
print_pad(callback, ctx, ' ', pad);
callback(ctx, &c, sizeof(c));
if (left_align)
print_pad(callback, ctx, ' ', pad);
written++;
written += 1 + pad;
fmt++;
break;
}
@@ -150,10 +165,15 @@ check_fmt:
len = strnlen(str, precision);
else
len = strlen(str);
const size_t pad = width > len ? width - len : 0;
if (!left_align)
print_pad(callback, ctx, ' ', pad);
callback(ctx, str, len);
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += len;
written += len + pad;
fmt++;
break;
}
@@ -212,36 +232,38 @@ check_fmt:
break;
}
print_num:
size_t len = strlen(number);
if (has_precision) {
for (size_t i = 0; i < precision - len; i++)
callback(ctx, " ", 1);
const char* digits = number;
char sign = 0;
if (digits[0] == '-') {
sign = '-';
digits++;
} else if (force_sign) {
sign = '+';
} else if (blank_sign) {
sign = ' ';
}
if (has_width) {
char c = pad_zero ? '0' : ' ';
// We don't have to pad anything.
if (len < width) {
for (size_t i = 0; i < width - len; i++)
callback(ctx, &c, 1);
}
const size_t len = strlen(digits);
size_t zeros = (has_precision && precision > len) ? precision - len : 0;
const size_t total = (sign ? 1 : 0) + (write_prefix ? 2 : 0) + zeros + len;
size_t pad = width > total ? width - total : 0;
if (pad_zero && !left_align && !has_precision) {
zeros += pad;
pad = 0;
}
if (write_prefix) {
if (!left_align)
print_pad(callback, ctx, ' ', pad);
if (sign)
callback(ctx, &sign, 1);
if (write_prefix)
callback(ctx, "0x", 2);
}
if (force_sign) {
char c = '+';
if (number[0] != '-')
callback(ctx, &c, 1);
}
if (blank_sign) {
char c = ' ';
if (number[0] != '-')
callback(ctx, &c, 1);
}
callback(ctx, number, len);
print_pad(callback, ctx, '0', zeros);
callback(ctx, digits, len);
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += len;
written += width > total ? width : total;
fmt++;
break;
}
@@ -272,13 +294,14 @@ print_num:
}
// No format, just normal text.
default: {
fmt = format_begun_at;
size_t len = strlen(fmt);
size_t len = fmt - format_begun_at;
if (*fmt)
len++;
callback(ctx, fmt, len);
callback(ctx, format_begun_at, len);
written += len;
fmt += len;
fmt = format_begun_at + len;
break;
}
}
core/refcount.c
+17
View File
@@ -0,0 +1,17 @@
#include <zinnia/refcount.h>
#include <stdatomic.h>
void refcount_init(struct refcount* rc) {
atomic_store_explicit(&rc->count, 1, memory_order_relaxed);
}
void refcount_inc(struct refcount* rc) {
atomic_fetch_add_explicit(&rc->count, 1, memory_order_relaxed);
}
bool refcount_dec(struct refcount* rc) {
if (atomic_fetch_sub_explicit(&rc->count, 1, memory_order_release) != 1)
return false;
atomic_thread_fence(memory_order_acquire);
return true;
}
core/sched/sched_dumb.c
+29 -11
View File
@@ -1,6 +1,5 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/list.h>
@@ -9,11 +8,12 @@
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/spin.h>
#include <zinnia/string.h>
#include <zinnia/tailq.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <stdatomic.h>
#include <stdint.h>
#include <zinnia/string.h>
static void idle_func(uintptr_t) {
while (1) {
@@ -39,7 +39,12 @@ void sched_init(struct scheduler* sched) {
struct task* idle_task;
assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
sched->idle_task = idle_task;
sched->current = idle_task;
struct task* boot_task = mem_alloc(sizeof(struct task), 0);
assert(boot_task != nullptr, "Unable to create boot task!\n");
strncpy(boot_task->name, "boot", sizeof(boot_task->name));
boot_task->space = &kernel_vma;
sched->current = boot_task;
tailq_init(&sched->run_queue);
@@ -76,17 +81,19 @@ static void do_reschedule(struct scheduler* sched) {
pmap_set(&to->space->pmap);
struct percpu* cpu = percpu_get();
from->kernel_stack = cpu->kernel_stack;
from->user_stack = cpu->user_stack;
cpu->kernel_stack = to->kernel_stack;
cpu->kernel_stack = to->kernel_stack + KERNEL_STACK_SIZE;
cpu->user_stack = to->user_stack;
struct task* old = arch_sched_switch(from, to);
struct irq_percpu irq_state = *percpu_ref(irq_data);
arch_sched_switch(from, to);
*percpu_ref(irq_data) = irq_state;
irq_unlock();
}
void sched_reschedule(struct scheduler* sched) {
irq_lock();
if (sched->current->state != TASK_STATE_DEAD)
if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD)
sched_add_task(sched, sched->current);
do_reschedule(sched);
}
@@ -104,27 +111,38 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
if (new_task == nullptr)
return ENOMEM;
static size_t last_tid = 0;
new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE);
static _Atomic(size_t) last_tid = 0;
new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire);
new_task->space = space;
strncpy(new_task->name, name, sizeof(new_task->name));
// Allocate a kernel stack.
new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0);
if (!new_task->kernel_stack)
if (!new_task->kernel_stack) {
mem_free(new_task);
return ENOMEM;
}
new_task->user_stack = 0;
// Initialize the arch-dependent parts.
errno_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
if (s != 0)
if (s != 0) {
mem_free((void*)new_task->kernel_stack);
mem_free(new_task);
return s;
}
*out = new_task;
return 0;
}
void task_entry(task_fn_t entry, uintptr_t arg0) {
*percpu_ref(irq_data) = (struct irq_percpu){
.depth = 1,
.saved_state = true,
};
irq_unlock();
entry(arg0);
struct scheduler* sched = &percpu_get()->sched;
core/spin.c
+5 -5
View File
@@ -1,15 +1,15 @@
#include <zinnia/atomic.h>
#include <zinnia/spin.h>
#include <stdatomic.h>
void spin_lock(struct spinlock* spin) {
uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED);
uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed);
while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) {
while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) {
asm volatile("pause");
}
}
void spin_unlock(struct spinlock* spin) {
uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED);
atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE);
uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed);
atomic_store_explicit(&spin->owner, val + 1, memory_order_release);
}
dev/acpi/uacpi_glue.c
+7 -4
View File
@@ -23,12 +23,15 @@ void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
uintptr_t difference = addr - aligned_addr;
size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
uintptr_t vaddr = vm_alloc(len);
for (size_t i = 0; i < len; i += arch_mem_page_size()) {
pmap_map(&kernel_vma.pmap, vaddr + i, addr + i, PROT_READ | PROT_WRITE, CACHE_ANY);
uintptr_t vaddr = vm_alloc(aligned_len);
if (!vaddr)
return nullptr;
for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) {
pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PROT_READ | PROT_WRITE, CACHE_ANY);
}
return (void*)vaddr;
return (void*)(vaddr + difference);
}
void uacpi_kernel_unmap(void* addr, uacpi_size len) {
include/arch/irq.h
+1 -1
View File
@@ -2,6 +2,6 @@
#include <zinnia/irq.h>
void arch_irq_set_state(bool state);
bool arch_irq_set_state(bool state);
bool arch_irq_get_state();
void arch_irq_wait();
include/arch/mem.h
+9 -8
View File
@@ -1,28 +1,29 @@
#pragma once
#include <zinnia/compiler.h>
#include <stddef.h>
#include <stdint.h>
[[gnu::pure]]
[[__pure]]
bool arch_mem_addr_is_user(uintptr_t addr);
[[gnu::pure]]
[[__pure]]
size_t arch_mem_page_size();
[[gnu::pure]]
[[__pure]]
size_t arch_mem_page_bits();
[[gnu::pure]]
[[__pure]]
size_t arch_mem_level_bits();
[[gnu::pure]]
[[__pure]]
size_t arch_mem_num_levels();
[[gnu::pure]]
[[__pure]]
uintptr_t arch_mem_hhdm_addr();
[[gnu::pure]]
[[__pure]]
uintptr_t arch_mem_pfndb_addr();
[[gnu::pure]]
[[__pure]]
uintptr_t arch_mem_mapping_addr();
include/zinnia/atomic.h
-21
View File
@@ -1,21 +0,0 @@
#pragma once
typedef enum memory_order {
ATOMIC_RELAXED = __ATOMIC_RELAXED,
ATOMIC_CONSUME = __ATOMIC_CONSUME,
ATOMIC_ACQUIRE = __ATOMIC_ACQUIRE,
ATOMIC_RELEASE = __ATOMIC_RELEASE,
ATOMIC_ACQ_REL = __ATOMIC_ACQ_REL,
ATOMIC_SEQ_CST = __ATOMIC_SEQ_CST
} memory_order;
#define atomic(T) _Atomic(T)
#define atomic_store(ptr, des, order) __atomic_store_n(ptr, des, order)
#define atomic_load(ptr, order) __atomic_load_n(ptr, order)
#define atomic_exchange(ptr, des, order) __atomic_exchange_n(ptr, des, order)
#define atomic_fetch_add(ptr, op, order) __atomic_fetch_add(ptr, op, order)
#define atomic_fetch_sub(ptr, op, order) __atomic_fetch_sub(ptr, op, order)
#define atomic_fetch_or(ptr, op, order) __atomic_fetch_or(ptr, op, order)
#define atomic_fetch_xor(ptr, op, order) __atomic_fetch_xor(ptr, op, order)
#define atomic_fetch_and(ptr, op, order) __atomic_fetch_and(ptr, op, order)
#define atomic_thread_fence(order) __atomic_thread_fence(order)
include/zinnia/compiler.h
+1
View File
@@ -14,6 +14,7 @@
#define __format(like, ...) gnu::format(like, __VA_ARGS__)
#define __always_inline gnu::always_inline
#define __visibility gnu::visibility
#define __pure gnu::pure
// Only clang has these hints, GCC would throw a warning here.
// clang::noderef also don't support the C23 attribute format.
include/zinnia/hashmap.h
+29 -20
View File
@@ -46,26 +46,38 @@
// Internal: grow and rehash.
#define _HASHMAP_GROW(map, hash_fn, eq_fn) \
do { \
({ \
errno_t _grow_status = 0; \
size_t _old_cap = (map)->capacity; \
size_t _new_cap = _old_cap == 0 ? 16 : _old_cap * 2; \
typeof((map)->keys) _old_keys = (map)->keys; \
typeof((map)->values) _old_values = (map)->values; \
uint64_t* _old_hashes = (map)->hashes; \
(map)->keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \
(map)->values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \
(map)->hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \
(map)->capacity = _new_cap; \
(map)->count = 0; \
for (size_t _i = 0; _i < _old_cap; _i++) { \
if (_old_hashes[_i] != 0) { \
_HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \
typeof((map)->keys) _new_keys = mem_alloc(_new_cap * sizeof(*(map)->keys), ALLOC_NOZERO); \
typeof((map)->values) _new_values = mem_alloc(_new_cap * sizeof(*(map)->values), ALLOC_NOZERO); \
uint64_t* _new_hashes = mem_alloc(_new_cap * sizeof(uint64_t), 0); \
if (_new_keys == nullptr || _new_values == nullptr || _new_hashes == nullptr) { \
mem_free(_new_keys); \
mem_free(_new_values); \
mem_free(_new_hashes); \
_grow_status = ENOMEM; \
} else { \
typeof((map)->keys) _old_keys = (map)->keys; \
typeof((map)->values) _old_values = (map)->values; \
uint64_t* _old_hashes = (map)->hashes; \
(map)->keys = _new_keys; \
(map)->values = _new_values; \
(map)->hashes = _new_hashes; \
(map)->capacity = _new_cap; \
(map)->count = 0; \
for (size_t _i = 0; _i < _old_cap; _i++) { \
if (_old_hashes[_i] != 0) { \
_HASHMAP_INSERT_INNER(map, _old_keys[_i], _old_values[_i], _old_hashes[_i]); \
} \
} \
mem_free(_old_keys); \
mem_free(_old_values); \
mem_free(_old_hashes); \
} \
mem_free(_old_keys); \
mem_free(_old_values); \
mem_free(_old_hashes); \
} while (0)
_grow_status; \
})
// Internal: insert with a precomputed hash. Used by grow and the public INSERT macro.
#define _HASHMAP_INSERT_INNER(map, k, v, h) \
@@ -126,10 +138,7 @@
} \
if (!_found) { \
if ((map)->count * 4 >= (map)->capacity * 3 || (map)->capacity == 0) { \
_HASHMAP_GROW(map, hash_fn, eq_fn); \
if ((map)->keys == nullptr || (map)->values == nullptr || (map)->hashes == nullptr) { \
_status = ENOMEM; \
} \
_status = _HASHMAP_GROW(map, hash_fn, eq_fn); \
} \
if (_status == 0) { \
_HASHMAP_INSERT_INNER(map, _k, (val), _h); \
include/zinnia/init.h
+1 -4
View File
@@ -26,16 +26,13 @@ struct boot_info {
uintptr_t hhdm_base;
struct boot_file* files;
size_t num_files;
#if CONFIG_OF
void* dtb;
#endif
#if CONFIG_ACPI
phys_t rsdp;
#endif
};
extern struct boot_info boot_info;
[[__visibility("hidden")]]
extern uint8_t __ld_stack_top[];
[[__visibility("hidden")]]
include/zinnia/irq.h
+26 -10
View File
@@ -2,11 +2,13 @@
#include <zinnia/compiler.h>
#include <zinnia/list.h>
#include <zinnia/types.h>
#include <stddef.h>
#include <stdint.h>
enum irq_status : uint8_t {
IRQ_STATUS_HANDLED,
IRQ_STATUS_IGNORED,
IRQ_STATUS_HANDLED,
};
enum irq_polarity : uint8_t {
@@ -19,34 +21,48 @@ enum irq_trigger_mode : uint8_t {
IRQ_TRIGGER_LEVEL,
};
enum irq_mode : uint8_t {
IRQ_MODE_EOI = 1 << 0,
IRQ_MODE_MASKABLE = 1 << 1,
};
struct irq_handler {
enum irq_status (*handler)(void* self);
enum irq_status (*raise)(struct irq_handler* self);
list_link(struct irq_handler*) next;
};
struct irq_line {
int (*set_config)(enum irq_trigger_mode mode, enum irq_polarity polarity);
enum irq_mode (*set_config)(struct irq_line* self, enum irq_trigger_mode trigger, enum irq_polarity polarity);
void (*mask)(struct irq_line* self);
void (*unmask)(struct irq_line* self);
void (*eoi)(struct irq_line* self);
list_head(struct irq_handler*) handlers;
bool is_busy;
list_head(struct irq_handler*) handlers;
enum irq_mode mode;
};
struct msi_line {
struct irq_line line;
phys_t (*msg_addr)(struct msi_line* self);
uint32_t (*msg_data)(struct msi_line* self);
};
struct irq_percpu {
uint32_t level;
bool in_interrupt;
size_t depth;
bool saved_state;
};
void irq_line_attach(struct irq_line* self, struct irq_handler* handler);
extern struct irq_percpu irq_data;
void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity);
void irq_line_raise(struct irq_line* line);
void irq_line_attach(struct irq_line* line, struct irq_handler* handler);
void irq_lock();
void irq_unlock();
bool irq_set_interrupted(bool is_interrupted);
void irq_line_raise(struct irq_line* line);
// Hints to stop execution on this CPU until an interrupt happens.
void irq_wait();
bool irq_get_state();
void irq_set_state(bool state);
bool irq_set_state(bool state);
include/zinnia/percpu.h
+1 -2
View File
@@ -1,7 +1,6 @@
#pragma once
#include <zinnia/compiler.h>
#include <zinnia/irq.h>
#include <zinnia/sched.h>
#include <zinnia/usercopy.h>
#include <stddef.h>
@@ -33,11 +32,11 @@ struct percpu {
bool online; // Whether this CPU is initialized and active.
uintptr_t kernel_stack; // The kernel mode stack.
uintptr_t user_stack; // The user mode stack.
struct irq_percpu irq; // IRQ information.
struct scheduler sched; // Scheduler information.
};
struct percpu* percpu_new();
size_t percpu_max_cpus();
struct percpu* percpu_get();
void percpu_bsp_init();
void percpu_init();
include/zinnia/print.h
+1 -1
View File
@@ -16,5 +16,5 @@ void kvprintf(const char* message, va_list args);
auto _time = clock_get_elapsed_ns(); \
auto _cur = percpu_get()->sched.current; \
auto _tid = _cur ? _cur->id : 0; \
kprintf("[%5lu.%06lu] [%5lu] " message, _time / 1'000'000'000, _time % 1'000'000, _tid, ##__VA_ARGS__); \
kprintf("[%5lu.%06lu] [%5lu] " message, _time / 1'000'000'000, _time / 1'000 % 1'000'000, _tid, ##__VA_ARGS__); \
} while (0)
include/zinnia/refcount.h
+4 -3
View File
@@ -1,12 +1,13 @@
#pragma once
#include <zinnia/atomic.h>
#include <zinnia/utils.h>
#include <stdatomic.h>
#include <stddef.h>
struct refcount {
atomic(size_t) count;
_Atomic(size_t) count;
};
void refcount_init(struct refcount* rc);
void refcount_inc(struct refcount* rc);
void refcount_dec(struct refcount* rc);
bool refcount_dec(struct refcount* rc);
include/zinnia/sched.h
-1
View File
@@ -1,6 +1,5 @@
#pragma once
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/init.h>
#include <zinnia/tailq.h>
include/zinnia/spin.h
+3 -2
View File
@@ -1,12 +1,13 @@
#pragma once
#include <stdatomic.h>
#include <stdint.h>
// Busy-waits in a loop until the lock is freed.
// Does not put the CPU to sleep.
struct spinlock {
uint32_t owner;
uint32_t next;
_Atomic(uint32_t) owner;
_Atomic(uint32_t) next;
};
// Attempts to lock a spinlock. If it's already locked, waits until it's freed.
include/zinnia/syscall_list.inc
+2 -2
View File
@@ -1,11 +1,11 @@
#ifndef SYSCALL_DECLARE
#define SYSCALL_DECLARE(number, name, num_args) extern void sys_##name();
#define SYSCALL_DECLARE(number, name, num_args) extern void sys_##name(...);
#endif
// SYSCALL_DECLARE(0, exit, -1)
// SYSCALL_DECLARE(1, syslog, -1)
SYSCALL_DECLARE(2, getuname, 1)
// SYSCALL_DECLARE(3, setuname, -1)
SYSCALL_DECLARE(3, setuname, 1)
SYSCALL_DECLARE(4, archctl, 2)
// SYSCALL_DECLARE(5, reboot, -1)
// SYSCALL_DECLARE(6, mmap, -1)
include/zinnia/vec.h
+17 -17
View File
@@ -34,27 +34,30 @@
// Internal: grow to at least `needed` capacity.
#define _vec_grow(vec, needed) \
do { \
({ \
errno_t _grow_status = 0; \
size_t _new_cap = (vec)->capacity == 0 ? 8 : (vec)->capacity; \
while (_new_cap < (needed)) \
_new_cap *= 2; \
typeof((vec)->data) _new = mem_alloc(_new_cap * sizeof(*(vec)->data), ALLOC_NOZERO); \
if (_new != nullptr && (vec)->length > 0) \
memcpy(_new, (vec)->data, (vec)->length * sizeof(*(vec)->data)); \
mem_free((vec)->data); \
(vec)->data = _new; \
(vec)->capacity = _new_cap; \
} while (0)
if (_new == nullptr) { \
_grow_status = ENOMEM; \
} else { \
if ((vec)->length > 0) \
memcpy(_new, (vec)->data, (vec)->length * sizeof(*(vec)->data)); \
mem_free((vec)->data); \
(vec)->data = _new; \
(vec)->capacity = _new_cap; \
} \
_grow_status; \
})
// Append an element. Returns 0 or ENOMEM.
#define vec_push(vec, val) \
({ \
errno_t _status = 0; \
if ((vec)->length >= (vec)->capacity) { \
_vec_grow(vec, (vec)->length + 1); \
if ((vec)->data == nullptr) \
_status = ENOMEM; \
} \
if ((vec)->length >= (vec)->capacity) \
_status = _vec_grow(vec, (vec)->length + 1); \
if (_status == 0) \
(vec)->data[(vec)->length++] = (val); \
_status; \
@@ -68,11 +71,8 @@
({ \
errno_t _status = 0; \
size_t _idx = (i); \
if ((vec)->length >= (vec)->capacity) { \
_vec_grow(vec, (vec)->length + 1); \
if ((vec)->data == nullptr) \
_status = ENOMEM; \
} \
if ((vec)->length >= (vec)->capacity) \
_status = _vec_grow(vec, (vec)->length + 1); \
if (_status == 0) { \
memmove(&(vec)->data[_idx + 1], &(vec)->data[_idx], ((vec)->length - _idx) * sizeof(*(vec)->data)); \
(vec)->data[_idx] = (val); \
include/zinnia/vma.h
+17 -4
View File
@@ -4,12 +4,25 @@
#include <zinnia/mman.h>
#include <zinnia/mutex.h>
#include <zinnia/pmap.h>
#include <zinnia/spin.h>
#include <zinnia/tailq.h>
#include <zinnia/vmo.h>
struct vma_mapping {
size_t start_page;
size_t end_page;
size_t offset_page;
struct vmo* object;
enum prot_flags prot;
tailq_link(struct vma_mapping) link;
};
// Virtual address space.
struct vma {
// Architecture-specific virtual memory map implementation.
struct pmap pmap;
tailq_head(struct vma_mapping) mappings;
struct spinlock lock;
};
// Try to create a new, empty virtual address space.
@@ -36,10 +49,10 @@ errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags
struct vma_fault_info {
uintptr_t ip;
uintptr_t addr;
bool caused_by_user;
bool caused_by_write;
bool caused_by_fetch;
bool page_was_present;
bool caused_by_user:1;
bool caused_by_write:1;
bool caused_by_fetch:1;
bool page_was_present:1;
};
// Try to handle a page fault.
include/zinnia/vmo.h
+6
View File
@@ -2,15 +2,21 @@
#include <zinnia/errno.h>
#include <zinnia/list.h>
#include <zinnia/refcount.h>
#include <zinnia/spin.h>
#include <zinnia/virt.h>
#include <stddef.h>
#include <stdint.h>
struct vmo {
struct refcount rc;
errno_t (*get_page)(struct vmo* vmo, uintptr_t offset_idx, struct page** out);
void (*destroy)(struct vmo* vmo);
};
void vmo_ref(struct vmo* obj);
void vmo_unref(struct vmo* obj);
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read);
errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written);
errno_t vmo_copy(
include/zinnia/waitq.h
+3 -3
View File
@@ -1,14 +1,14 @@
#pragma once
#include <zinnia/atomic.h>
#include <zinnia/sched.h>
#include <zinnia/spin.h>
#include <zinnia/tailq.h>
#include <stdatomic.h>
// A queue of tasks waiting for an event.
struct waitq {
tailq_head(atomic(struct task)) waiters;
atomic(bool) woken;
tailq_head(_Atomic(struct task)) waiters;
_Atomic(bool) woken;
};
void waitq_init(struct waitq* wq);
syscalls/uname.c
+13 -11
View File
@@ -1,8 +1,8 @@
#include <zinnia/spin.h>
#include <zinnia/string.h>
#include <zinnia/syscall.h>
#include <zinnia/usercopy.h>
#include <config.h>
#include <zinnia/string.h>
#include <uapi/utsname.h>
static struct spinlock uname_lock = {};
@@ -13,9 +13,10 @@ static struct __utsname uname_buf = {
.version = CONFIG_KERNEL_FLAVOR,
.nodename = "localhost",
.machine = ZINNIA_ARCH,
.domainname = "none",
};
struct syscall_ret sys_setuname(__user const struct __utsname* ptr) {
struct syscall_ret sys_setuname(const __user struct __utsname* ptr) {
struct syscall_ret ret = {};
if (!ptr) {
@@ -33,11 +34,12 @@ struct syscall_ret sys_setuname(__user const struct __utsname* ptr) {
spin_lock(&uname_lock);
strncpy(uname_buf.sysname, read_buf.sysname, sizeof(uname_buf.sysname));
strncpy(uname_buf.release, read_buf.release, sizeof(uname_buf.release));
strncpy(uname_buf.version, read_buf.version, sizeof(uname_buf.version));
strncpy(uname_buf.nodename, read_buf.nodename, sizeof(uname_buf.nodename));
strncpy(uname_buf.machine, read_buf.machine, sizeof(uname_buf.machine));
strncpy(uname_buf.sysname, read_buf.sysname, sizeof(uname_buf.sysname) - 1);
strncpy(uname_buf.release, read_buf.release, sizeof(uname_buf.release) - 1);
strncpy(uname_buf.version, read_buf.version, sizeof(uname_buf.version) - 1);
strncpy(uname_buf.nodename, read_buf.nodename, sizeof(uname_buf.nodename) - 1);
strncpy(uname_buf.machine, read_buf.machine, sizeof(uname_buf.machine) - 1);
strncpy(uname_buf.domainname, read_buf.domainname, sizeof(uname_buf.domainname) - 1);
spin_unlock(&uname_lock);
@@ -53,11 +55,11 @@ struct syscall_ret sys_getuname(__user struct __utsname* ptr) {
}
spin_lock(&uname_lock);
if (!usercopy_write(ptr, &uname_buf, sizeof(uname_buf)))
ret.errno = EFAULT;
struct __utsname write_buf = uname_buf;
spin_unlock(&uname_lock);
if (!usercopy_write(ptr, &write_buf, sizeof(write_buf)))
ret.errno = EFAULT;
return ret;
}