marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

foo

Browse Source
This commit is contained in:
Marvin
2026-09-28 17:46:45 +02:00
committed by Marvin Friedrich
parent 09926f45ee
commit 4a20a5b0af
62 changed files with 1266 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/clock.c
+20 -10
View File
@@ -1,4 +1,3 @@
#include <zinnia/atomic.h>
#include <zinnia/clock.h>
#include <zinnia/compiler.h>
#include <zinnia/print.h>
@@ -9,16 +8,25 @@ static struct clock* active_clock = nullptr;
static uint64_t counter_base = 0;
static struct spinlock lock = {};
static uint64_t clock_get_elapsed_ns_locked() {
if (__unlikely(!active_clock))
return 0;
return counter_base + active_clock->get_elapsed_ns(active_clock);
}
bool clock_available() {
return active_clock != nullptr;
spin_lock(&lock);
bool available = active_clock != nullptr;
spin_unlock(&lock);
return available;
}
uint64_t clock_get_elapsed_ns() {
struct clock* active = atomic_load(&active_clock, ATOMIC_RELAXED);
if (__unlikely(!active))
return 0;
return atomic_load(&counter_base, ATOMIC_RELAXED) + active->get_elapsed_ns(active);
spin_lock(&lock);
uint64_t elapsed = clock_get_elapsed_ns_locked();
spin_unlock(&lock);
return elapsed;
}
bool clock_switch(struct clock* clock) {
@@ -33,16 +41,18 @@ bool clock_switch(struct clock* clock) {
log("clock: Switching to clock source \"%s\"\n", clock->name);
uint64_t elapsed = clock_get_elapsed_ns();
counter_base += elapsed;
uint64_t elapsed = clock_get_elapsed_ns_locked();
clock->reset(clock);
counter_base = elapsed;
active_clock = clock;
spin_unlock(&lock);
return true;
}
void clock_spin_ns(uint64_t ns) {
if (active_clock == nullptr) {
if (!clock_available()) {
log("clock: Unable to sleep for %lu ns, no clock available!\n", ns);
return;
}
core/console.c
+3 -1
View File
@@ -35,8 +35,10 @@ void console_init() {
active = __ld_console_start[0];
if (active) {
if (active->init(active) == false)
if (active->init(active) == false) {
active = nullptr;
return;
}
log("console: Logging on console \"%s\"\n", active->name);
}
core/elf.c
+89 -28
View File
@@ -7,11 +7,12 @@
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/string.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <zinnia/vmo.h>
#include <bits/elf.h>
#include <zinnia/string.h>
#include <stdckdint.h>
struct exec_info {
struct vma* space;
@@ -21,11 +22,15 @@ struct exec_info {
};
errno_t elf_load(struct exec_info* info, struct task** out) {
errno_t status;
errno_t err;
size_t read;
struct elf_ehdr ehdr = {};
vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
if (err)
return err;
if (read != sizeof(ehdr))
return ENOEXEC;
// Check if the file is an ELF.
if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
@@ -40,13 +45,48 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
return EINVAL;
if (ehdr.e_type != ET_EXEC)
return EINVAL;
if (!arch_mem_addr_is_user(ehdr.e_entry))
return ENOEXEC;
const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize;
uintptr_t phdrs_end;
if (ehdr.e_phentsize < sizeof(struct elf_phdr) || ehdr.e_phnum == 0 || phdrs_size > 64 * 1024)
return ENOEXEC;
if (ckd_add(&phdrs_end, ehdr.e_phoff, phdrs_size))
return ENOEXEC;
const size_t page_size = arch_mem_page_size();
uintptr_t phdr_addr = 0;
for (size_t i = 0; i < ehdr.e_phnum; i++) {
struct elf_phdr phdr = {};
vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
if (err)
return err;
if (read != sizeof(phdr))
return ENOEXEC;
if (phdr.p_type == PT_PHDR)
phdr_addr = phdr.p_vaddr;
if (phdr.p_type == PT_LOAD) {
if (phdr.p_memsz == 0)
continue;
if (phdr.p_filesz > phdr.p_memsz)
return ENOEXEC;
uintptr_t seg_end;
if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz))
return ENOEXEC;
if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1))
return ENOEXEC;
if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align)
return ENOEXEC;
if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz)
phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset);
enum prot_flags prot = 0;
if (phdr.p_flags & PF_R)
prot |= PROT_READ;
@@ -55,8 +95,6 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
if (phdr.p_flags & PF_X)
prot |= PROT_EXEC;
assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
const uintptr_t map_address = phdr.p_vaddr - misalign;
const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
@@ -64,18 +102,21 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
// Copy the file data into its own mapping.
struct paged_vmo* phdr_obj;
status = vmo_new_phys(&phdr_obj);
if (status)
return status;
err = vmo_new_phys(&phdr_obj);
if (err)
return err;
status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
if (status)
return status;
err = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
if (err) {
vmo_unref(&phdr_obj->object);
return err;
}
// We map more than we copied so the rest is filled with zeroed pages.
status = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
if (status)
return status;
err = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
vmo_unref(&phdr_obj->object);
if (err)
return err;
}
}
@@ -85,9 +126,9 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
// Allocate stack.
struct paged_vmo* stack;
status = vmo_new_phys(&stack);
if (status)
return status;
err = vmo_new_phys(&stack);
if (err)
return err;
// Fill in stack info.
uintptr_t stack_off = stack_size;
@@ -101,8 +142,21 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
num_argv++;
}
size_t strings_size = 0;
for (size_t env = 0; env < num_envp; env++)
strings_size += strlen(info->envp[env]) + 1;
for (size_t arg = 0; arg < num_argv; arg++)
strings_size += strlen(info->argv[arg]) + 1;
uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) {
mem_free(envp_offsets);
mem_free(argv_offsets);
vmo_unref(&stack->object);
return ENOMEM;
}
for (size_t env = 0; env < num_envp; env++) {
const char nul = 0;
@@ -153,22 +207,24 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
if (phdr_addr)
WRITE_AUXV(AT_PHDR, phdr_addr);
// envp pointers.
stack_off -= sizeof(uintptr_t);
const uintptr_t zero = 0;
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
for (size_t env = 0; env < num_envp; env++) {
for (size_t env = num_envp; env > 0; env--) {
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr);
vmo_write(&stack->object, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr);
}
// argv pointers.
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
for (size_t arg = 0; arg < num_argv; arg++) {
for (size_t arg = num_argv; arg > 0; arg--) {
stack_off -= sizeof(uintptr_t);
vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr);
vmo_write(&stack->object, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr);
}
mem_free(envp_offsets);
@@ -179,19 +235,24 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
uintptr_t argc = num_argv;
vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
status = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
if (status)
return status;
err = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
vmo_unref(&stack->object);
if (err)
return err;
struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
if (!start_info)
return ENOMEM;
start_info->ip = ehdr.e_entry;
start_info->sp = highest - stack_size + stack_off;
start_info->arg = 0;
struct task* result;
status = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
if (status)
return status;
err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
if (err) {
mem_free(start_info);
return err;
}
*out = result;
return 0;
core/irq.c
+47 -19
View File
@@ -1,10 +1,15 @@
#include <zinnia/atomic.h>
#include <zinnia/assert.h>
#include <zinnia/irq.h>
#include <zinnia/list.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <arch/irq.h>
void irq_set_state(bool state) {
arch_irq_set_state(state);
[[__percpu]]
struct irq_percpu irq_data;
bool irq_set_state(bool state) {
return arch_irq_set_state(state);
}
bool irq_get_state() {
@@ -16,26 +21,49 @@ void irq_wait() {
}
void irq_lock() {
struct irq_percpu* cpu = &percpu_get()->irq;
if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE))
return;
arch_irq_set_state(false);
atomic_fetch_add(&percpu_get()->irq.level, 1, ATOMIC_ACQ_REL);
bool prev = arch_irq_set_state(false);
struct irq_percpu* state = percpu_ref(irq_data);
if (state->depth++ == 0)
state->saved_state = prev;
}
void irq_unlock() {
struct irq_percpu* cpu = &percpu_get()->irq;
if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE))
return;
uint32_t old_level = atomic_fetch_sub(&cpu->level, 1, ATOMIC_ACQ_REL);
// If it was 1, the new IRQ level is now 0.
if (old_level == 1) {
struct irq_percpu* state = percpu_ref(irq_data);
if (--state->depth == 0 && state->saved_state)
arch_irq_set_state(true);
}
}
bool irq_set_interrupted(bool is_interrupted) {
return atomic_exchange(&percpu_get()->irq.in_interrupt, is_interrupted, ATOMIC_RELEASE);
void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
irq_lock();
line->mode = line->set_config(line, trigger, polarity);
irq_unlock();
}
void irq_line_raise(struct irq_line* line) {
assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
irq_lock();
line->is_busy = true;
bool claimed = false;
struct irq_handler* handler;
list_foreach(handler, &line->handlers, next) {
if (handler->raise(handler) == IRQ_STATUS_HANDLED)
claimed = true;
}
if (!claimed)
log("irq: Spurious interrupt\n");
if ((line->mode & IRQ_MODE_EOI) && line->eoi)
line->eoi(line);
line->is_busy = false;
irq_unlock();
}
void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
irq_lock();
list_insert_head(&line->handlers, handler, next);
irq_unlock();
}
core/lib/string.c
+7 -6
View File
@@ -24,15 +24,14 @@ size_t strnlen(const char* str, size_t len) {
[[__hot]]
int memcmp(const void* s1, const void* s2, size_t size) {
int diff = 0;
const uint8_t* s1ptr = s1;
const uint8_t* s2ptr = s2;
for (size_t i = 0; i < size; i++) {
if (s1ptr[i] != s2ptr[i])
diff++;
return s1ptr[i] - s2ptr[i];
}
return diff;
return 0;
}
[[__hot]]
@@ -103,7 +102,7 @@ void* memset(void* dest, int value, size_t n) {
}
size_t* qword_dest = (size_t*)d;
size_t qword_value = value;
size_t qword_value = (uint8_t)value;
qword_value |= (qword_value << 8);
qword_value |= (qword_value << 16);
qword_value |= (qword_value << 32);
@@ -126,8 +125,10 @@ void* memset(void* dest, int value, size_t n) {
}
char* strncpy(char* restrict dst, const char* restrict src, size_t len) {
size_t src_len = strnlen(src, len) + 1;
return memcpy(dst, src, MIN(len, src_len));
size_t src_len = strnlen(src, len);
memcpy(dst, src, src_len);
memset(dst + src_len, 0, len - src_len);
return dst;
}
int strncmp(const char* str1, const char* str2, size_t len) {
core/mem/init.c
+3 -2
View File
@@ -64,10 +64,11 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
mem_hhdm_base = tmp_hhdm;
assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
tailq_init(&kernel_vma.mappings);
// text
log("mem: Mapping text segment at %p\n", __ld_text_start);
for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
for (uint8_t* p = __ld_text_start; p < __ld_text_end; p += pgsz) {
errno_t status = pmap_map(
&kernel_vma.pmap,
(uintptr_t)p,
@@ -116,7 +117,7 @@ void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_
for (size_t i = 0; i < map_len; i++) {
if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
continue;
for (size_t p = 0; p <= map[i].length; p += pgsz) {
for (size_t p = 0; p < map[i].length; p += pgsz) {
uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
phys_t paddr = (phys_t)(map[i].address + p);
errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
core/mem/pmap.c
+11 -1
View File
@@ -17,6 +17,16 @@ errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
if (status)
return status;
pte_t* root = HHDM_PTR(addr);
const size_t num_entries = (size_t)1 << arch_mem_level_bits();
for (size_t i = num_entries / 2; i < num_entries; i++) {
phys_t dir;
status = mem_phys_alloc(1, flags, &dir);
if (status)
return status;
root[i] = pte_build(dir, PTE_DIR, CACHE_ANY);
}
struct pmap result = {
.root = addr,
.lock = (struct spinlock){0},
@@ -71,7 +81,7 @@ static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check
if (pte_is_present(pte)) {
// Get the next level.
*pte = pte_build(pte_address(pte), level_flags, CACHE_ANY);
*pte |= pte_build(0, level_flags, CACHE_ANY);
current_head = HHDM_PTR(pte_address(pte));
} else {
// If the current level isn't present, we can skip the rest.
core/mem/pmm.c
+53 -12
View File
@@ -25,7 +25,7 @@ static struct phys_mem* bump_region = nullptr;
static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
const size_t bytes = num_pages * arch_mem_page_size();
if (((intptr_t)bump_mem.length - bytes) <= 0)
if (bytes > bump_mem.length)
return ENOMEM;
*out = bump_mem.address;
@@ -76,7 +76,7 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
while (it) {
const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
// If the region is too high up or doesn't have enough pages left, continue searching.
if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
if (__unlikely(it->freelist.count < num_pages) || page_addr + bytes > limit) {
prev_it = it;
it = it->freelist.next;
continue;
@@ -96,11 +96,26 @@ static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t*
*out = page_addr;
}
// If not, we can just shrink the block.
else {
else if (page_addr + it->freelist.count * arch_mem_page_size() <= limit) {
it->freelist.count -= num_pages;
if (__likely(out))
*out = page_addr + (it->freelist.count * arch_mem_page_size());
} else {
struct page* rest = it + num_pages;
rest->freelist.count = it->freelist.count - num_pages;
rest->freelist.next = it->freelist.next;
if (prev_it)
prev_it->freelist.next = rest;
else
pmm_head = rest;
it->freelist.next = nullptr;
it->freelist.count = 0;
if (__likely(out))
*out = page_addr;
}
goto success;
}
@@ -118,13 +133,41 @@ success:
return 0;
}
static void freelist_insert(struct page* page, size_t num_pages) {
struct page* prev = nullptr;
struct page* next = pmm_head;
while (next && next < page) {
prev = next;
next = next->freelist.next;
}
page->freelist.count = num_pages;
page->freelist.next = next;
if (next && page + page->freelist.count == next) {
page->freelist.count += next->freelist.count;
page->freelist.next = next->freelist.next;
next->freelist.next = nullptr;
next->freelist.count = 0;
}
if (prev && prev + prev->freelist.count == page) {
prev->freelist.count += page->freelist.count;
prev->freelist.next = page->freelist.next;
page->freelist.next = nullptr;
page->freelist.count = 0;
} else if (prev) {
prev->freelist.next = page;
} else {
pmm_head = page;
}
}
static errno_t freelist_free(phys_t addr, size_t num_pages) {
spin_lock(&pmm_lock);
struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
page->freelist.count = num_pages;
page->freelist.next = pmm_head;
pmm_head = page;
freelist_insert(&vm_pfndb[addr / arch_mem_page_size()], num_pages);
pmm_total_free += num_pages;
spin_unlock(&pmm_lock);
return 0;
@@ -141,11 +184,9 @@ void mem_phys_init(struct phys_mem* map, size_t length) {
if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
continue;
struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
page->freelist.count = map[i].length / arch_mem_page_size();
page->freelist.next = pmm_head;
pmm_head = page;
pmm_total_free += page->freelist.count;
const size_t num_pages = map[i].length / arch_mem_page_size();
freelist_insert(&vm_pfndb[map[i].address / arch_mem_page_size()], num_pages);
pmm_total_free += num_pages;
}
const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
core/mem/slab.c
+4 -4
View File
@@ -24,8 +24,6 @@ typedef struct {
static struct slab slabs[8] = {0};
static errno_t slab_new(struct slab* slab, size_t size) {
slab->lock = (struct spinlock){0};
// Allocate a new page for the head.
phys_t addr;
errno_t status = mem_phys_alloc(1, 0, &addr);
@@ -68,8 +66,10 @@ void slab_init(void) {
static void* slab_do_alloc(struct slab* slab) {
spin_lock(&slab->lock);
if (__unlikely(slab->head == nullptr))
slab_new(slab, slab->ent_size);
if (__unlikely(slab->head == nullptr) && slab_new(slab, slab->ent_size) != 0) {
spin_unlock(&slab->lock);
return nullptr;
}
void** old_free = slab->head;
slab->head = *old_free;
core/mem/usercopy.c
+20
View File
@@ -1,8 +1,18 @@
#include <zinnia/percpu.h>
#include <zinnia/usercopy.h>
#include <arch/mem.h>
#include <arch/usercopy.h>
static bool is_user_range(uintptr_t addr, size_t len) {
if (len == 0)
return arch_mem_addr_is_user(addr);
return addr + len > addr && arch_mem_addr_is_user(addr) && arch_mem_addr_is_user(addr + len - 1);
}
bool usercopy_read(void* dst, const __user void* src, size_t len) {
if (!is_user_range((uintptr_t)src, len))
return false;
arch_usercopy_allow(true);
bool success = arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
@@ -10,6 +20,9 @@ bool usercopy_read(void* dst, const __user void* src, size_t len) {
}
bool usercopy_write(__user void* dst, const void* src, size_t len) {
if (!is_user_range((uintptr_t)dst, len))
return false;
arch_usercopy_allow(true);
bool success = arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
@@ -17,6 +30,13 @@ bool usercopy_write(__user void* dst, const void* src, size_t len) {
}
bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
if (!arch_mem_addr_is_user((uintptr_t)str))
return false;
size_t unused;
if (len == nullptr)
len = &unused;
arch_usercopy_allow(true);
bool success = arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
arch_usercopy_allow(false);
core/mem/virt.c
+4 -3
View File
@@ -1,18 +1,19 @@
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <stdatomic.h>
#include <stddef.h>
struct page* vm_pfndb = nullptr;
// TODO: Replace with actual allocator.
static size_t mapping_offset = 0;
static _Atomic(size_t) mapping_offset = 0;
uintptr_t vm_alloc(size_t length) {
uintptr_t base = arch_mem_mapping_addr();
size_t offset = atomic_fetch_add(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), ATOMIC_ACQUIRE);
size_t offset =
atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire);
return base + offset;
}
core/mem/vma.c
+101 -41
View File
@@ -5,6 +5,7 @@
#include <zinnia/pmap.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <stdckdint.h>
// Try to create a new, empty virtual address space.
errno_t vma_new(struct vma** out) {
@@ -22,6 +23,8 @@ errno_t vma_new(struct vma** out) {
if (res)
goto err1;
tailq_init(&vmspace->mappings);
*out = vmspace;
return 0;
@@ -32,20 +35,90 @@ err0:
}
// Delete a virtual address space.
void vma_free(struct vma* v) {}
void vma_free(struct vma* v) {
struct vma_mapping* mapping;
while ((mapping = tailq_first(&v->mappings)) != nullptr) {
tailq_remove(&v->mappings, mapping, link);
vmo_unref(mapping->object);
mem_free(mapping);
}
}
// Remove any mappings in the given address range.
// May fail if there is no memory and a mapping needs to be split.
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size);
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size) {
panic("TODO");
}
// Change the protection of an existing mapping.
// May fail if there is no memory and a mapping needs to be split.
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags);
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags) {
panic("TODO");
}
static struct vma_mapping* find_mapping(struct vma* v, size_t page) {
for (struct vma_mapping* m = tailq_first(&v->mappings); m; m = tailq_next(m, link)) {
if (page >= m->start_page && page < m->end_page)
return m;
}
return nullptr;
}
static bool access_allowed(enum prot_flags prot, const struct vma_fault_info* info) {
if (info->caused_by_write)
return prot & PROT_WRITE;
if (info->caused_by_fetch)
return prot & PROT_EXEC;
return prot & (PROT_READ | PROT_WRITE);
}
static enum pte_flags prot_to_pte(struct vma* v, enum prot_flags prot) {
enum pte_flags pf = 0;
if (prot & PROT_READ)
pf |= PTE_READ;
if (prot & PROT_WRITE)
pf |= PTE_WRITE;
if (prot & PROT_EXEC)
pf |= PTE_EXEC;
if (v->pmap.is_user)
pf |= PTE_USER;
return pf;
}
// Try to handle a page fault.
bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
// TODO
return false;
const size_t page_size = arch_mem_page_size();
const size_t faulty_page = info->addr / page_size;
spin_lock(&v->lock);
struct vma_mapping* mapping = find_mapping(v, faulty_page);
if (!mapping || !access_allowed(mapping->prot, info)) {
spin_unlock(&v->lock);
return false;
}
struct vmo* object = mapping->object;
const size_t object_page = (faulty_page - mapping->start_page) + mapping->offset_page;
const enum prot_flags prot = mapping->prot;
vmo_ref(object);
spin_unlock(&v->lock);
struct page* page;
if (object->get_page(object, object_page, &page)) {
vmo_unref(object);
return false;
}
spin_lock(&v->lock);
mapping = find_mapping(v, faulty_page);
const bool still_valid = mapping && mapping->object == object &&
(faulty_page - mapping->start_page) + mapping->offset_page == object_page &&
access_allowed(mapping->prot, info);
if (still_valid) {
errno_t status =
pmap_map(&v->pmap, faulty_page * page_size, vm_page_addr(page), prot_to_pte(v, prot), CACHE_ANY);
assert(status == 0, "Failed to map a demand-loaded page\n");
}
spin_unlock(&v->lock);
vmo_unref(object);
return true;
}
errno_t vma_map(
@@ -56,44 +129,31 @@ errno_t vma_map(
enum prot_flags flags,
uintptr_t vmo_offset
) {
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
size_t start_page = addr / page_size;
const size_t addr_offset = addr % page_size;
const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size;
size_t end_page = start_page + pages;
if (!vma || !vmo)
return EINVAL;
const size_t first_vmo_page = vmo_offset / page_size;
uintptr_t end;
if (ckd_add(&end, addr, len))
return ENOMEM;
for (size_t p = start_page; p < end_page; ++p) {
const size_t page_index = p - start_page;
struct page* page_obj;
errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj);
if (status != 0)
return status;
const size_t page_size = arch_mem_page_size();
if (addr % page_size != vmo_offset % page_size)
return EINVAL;
phys_t paddr = (phys_t)vm_page_addr(page_obj);
struct vma_mapping* mapping = mem_alloc(sizeof(struct vma_mapping), 0);
if (!mapping)
return ENOMEM;
enum pte_flags pf = 0;
if (flags & PROT_READ)
pf |= PTE_READ;
if (flags & PROT_WRITE)
pf |= PTE_WRITE;
if (flags & PROT_EXEC)
pf |= PTE_EXEC;
if (vma->pmap.is_user)
pf |= PTE_USER;
vmo_ref(vmo);
mapping->start_page = addr / page_size;
mapping->end_page = ALIGN_UP(end, page_size) / page_size;
mapping->offset_page = vmo_offset / page_size;
mapping->object = vmo;
mapping->prot = flags;
status = pmap_map(&vma->pmap, p * page_size, paddr, pf, CACHE_ANY);
if (status != 0)
return status;
}
spin_lock(&vma->lock);
tailq_insert_head(&vma->mappings, mapping, link);
spin_unlock(&vma->lock);
return 0;
}
core/mem/vmo.c
+53 -3
View File
@@ -5,6 +5,15 @@
#include <zinnia/vmo.h>
#include <zinnia/string.h>
void vmo_ref(struct vmo* obj) {
refcount_inc(&obj->rc);
}
void vmo_unref(struct vmo* obj) {
if (refcount_dec(&obj->rc))
obj->destroy(obj);
}
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
const size_t page_size = arch_mem_page_size();
size_t progress = 0;
@@ -104,14 +113,17 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page
// Try to find the requested page in the cache.
struct page_list* iter;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
// We found a cached page!
*out = iter->value;
spin_unlock(&paged->lock);
return 0;
}
spin_unlock(&paged->lock);
// Get page from out pager if we don't have one already.
struct page* new_page;
@@ -120,26 +132,57 @@ static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page
return status;
struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
if (!entry)
if (!entry) {
paged->source.put_page(&paged->source, offset_idx, new_page);
return ENOMEM;
}
entry->offset = offset_idx;
entry->value = new_page;
spin_lock(&paged->lock);
list_foreach(iter, &paged->cache, next) {
if (iter->offset != offset_idx)
continue;
*out = iter->value;
spin_unlock(&paged->lock);
mem_free(entry);
paged->source.put_page(&paged->source, offset_idx, new_page);
return 0;
}
// Add page to the object cache.
list_insert_head(&paged->cache, entry, next);
spin_unlock(&paged->lock);
*out = new_page;
return 0;
}
static void paged_destroy(struct vmo* vmo) {
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
struct page_list* iter = list_first(&paged->cache);
while (iter) {
struct page_list* next = list_next(iter, next);
paged->source.put_page(&paged->source, iter->offset, iter->value);
mem_free(iter);
iter = next;
}
mem_free(paged);
}
errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
if (!result)
return ENOMEM;
refcount_init(&result->object.rc);
result->source = pager;
result->object.get_page = paged_get_page;
result->object.destroy = paged_destroy;
*out = result;
return 0;
@@ -156,8 +199,7 @@ static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct p
}
static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
// We do nothing here.
return 0;
return mem_phys_free(vm_page_addr(page), 1);
}
static const struct pager_ops phys_pager = {
@@ -177,16 +219,24 @@ struct phys_range_vmo {
static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
if (offset_idx >= p->len / arch_mem_page_size())
return EINVAL;
*out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
return 0;
}
static void phys_range_destroy(struct vmo* vmo) {
mem_free(containerof(vmo, struct phys_range_vmo, object));
}
errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
if (!vmo)
return ENOMEM;
refcount_init(&vmo->object.rc);
vmo->object.get_page = phys_range_get_page;
vmo->object.destroy = phys_range_destroy;
vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
vmo->len = ALIGN_UP(length, arch_mem_page_size());
core/meson.build
+1
View File
@@ -6,6 +6,7 @@ kernel_sources += files(
'panic.c',
'percpu.c',
'print.c',
'refcount.c',
'spin.c',
'wait.c',
)
core/percpu.c
+32 -2
View File
@@ -1,6 +1,22 @@
#include <zinnia/alloc.h>
#include <zinnia/cmdline.h>
#include <zinnia/compiler.h>
#include <zinnia/percpu.h>
#include <zinnia/string.h>
#include <zinnia/utils.h>
#include <arch/percpu.h>
#include <stdatomic.h>
extern uint8_t __ld_percpu_start[];
extern uint8_t __ld_percpu_end[];
static _Atomic(size_t) num_cpus = 1;
static size_t max_cpus = SIZE_MAX;
static void smp_option(uintptr_t value) {
max_cpus = value;
}
CMDLINE_OPTION("smp", "Maximum number of CPUs to use", smp_option);
[[__used, __section(".percpu.init")]]
struct percpu percpu_bsp = {
@@ -10,8 +26,22 @@ struct percpu percpu_bsp = {
};
struct percpu* percpu_new() {
// TODO
return nullptr;
const size_t size = __ld_percpu_end - __ld_percpu_start;
const size_t num_pages = ALIGN_UP(size, arch_mem_page_size()) / arch_mem_page_size();
phys_t mem;
if (mem_phys_alloc(num_pages, 0, &mem))
return nullptr;
struct percpu* cpu = HHDM_PTR(mem);
cpu->self = cpu;
cpu->id = atomic_fetch_add_explicit(&num_cpus, 1, memory_order_relaxed);
cpu->online = false;
return cpu;
}
size_t percpu_max_cpus() {
return max_cpus;
}
struct percpu* percpu_get() {
core/print.c
+65 -42
View File
@@ -10,6 +10,11 @@
typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
static void print_pad(log_fn_t callback, void* ctx, char c, size_t count) {
for (size_t i = 0; i < count; i++)
callback(ctx, &c, 1);
}
static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
// Amount of bytes written.
int32_t written = 0;
@@ -37,7 +42,7 @@ static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_
bool blank_sign = false;
bool write_prefix = false;
bool pad_zero = false;
bool has_width = false;
bool left_align = false;
size_t width = 0;
bool has_precision = false;
size_t precision = 0;
@@ -68,6 +73,11 @@ check_fmt:
fmt++;
goto check_fmt;
}
case '-': {
left_align = true;
fmt++;
goto check_fmt;
}
// Width
case '1':
case '2':
@@ -78,14 +88,11 @@ check_fmt:
case '7':
case '8':
case '9': {
has_width = true;
char number[10 + 1] = {0};
size_t idx = 0;
width = 0;
while (*fmt >= '0' && *fmt <= '9') {
number[idx++] = *fmt;
width = width * 10 + (*fmt - '0');
fmt++;
}
width = atolu(number, 10);
goto check_fmt;
}
@@ -93,16 +100,19 @@ check_fmt:
case '.': {
has_precision = true;
fmt++;
precision = 0;
if (*fmt == '*') {
precision = (char)va_arg(args, int32_t);
const int32_t arg = va_arg(args, int32_t);
if (arg < 0)
has_precision = false;
else
precision = arg;
fmt++;
} else {
char number[10 + 1] = {0};
size_t idx = 0;
while (*fmt >= '0' && *fmt <= '9') {
number[idx++] = *fmt;
precision = precision * 10 + (*fmt - '0');
fmt++;
}
precision = atolu(number, 10);
}
goto check_fmt;
}
@@ -133,10 +143,15 @@ check_fmt:
// Character
case 'c': {
char c = (char)va_arg(args, int32_t);
const size_t pad = width > 1 ? width - 1 : 0;
if (!left_align)
print_pad(callback, ctx, ' ', pad);
callback(ctx, &c, sizeof(c));
if (left_align)
print_pad(callback, ctx, ' ', pad);
written++;
written += 1 + pad;
fmt++;
break;
}
@@ -150,10 +165,15 @@ check_fmt:
len = strnlen(str, precision);
else
len = strlen(str);
const size_t pad = width > len ? width - len : 0;
if (!left_align)
print_pad(callback, ctx, ' ', pad);
callback(ctx, str, len);
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += len;
written += len + pad;
fmt++;
break;
}
@@ -212,36 +232,38 @@ check_fmt:
break;
}
print_num:
size_t len = strlen(number);
if (has_precision) {
for (size_t i = 0; i < precision - len; i++)
callback(ctx, " ", 1);
const char* digits = number;
char sign = 0;
if (digits[0] == '-') {
sign = '-';
digits++;
} else if (force_sign) {
sign = '+';
} else if (blank_sign) {
sign = ' ';
}
if (has_width) {
char c = pad_zero ? '0' : ' ';
// We don't have to pad anything.
if (len < width) {
for (size_t i = 0; i < width - len; i++)
callback(ctx, &c, 1);
}
const size_t len = strlen(digits);
size_t zeros = (has_precision && precision > len) ? precision - len : 0;
const size_t total = (sign ? 1 : 0) + (write_prefix ? 2 : 0) + zeros + len;
size_t pad = width > total ? width - total : 0;
if (pad_zero && !left_align && !has_precision) {
zeros += pad;
pad = 0;
}
if (write_prefix) {
if (!left_align)
print_pad(callback, ctx, ' ', pad);
if (sign)
callback(ctx, &sign, 1);
if (write_prefix)
callback(ctx, "0x", 2);
}
if (force_sign) {
char c = '+';
if (number[0] != '-')
callback(ctx, &c, 1);
}
if (blank_sign) {
char c = ' ';
if (number[0] != '-')
callback(ctx, &c, 1);
}
callback(ctx, number, len);
print_pad(callback, ctx, '0', zeros);
callback(ctx, digits, len);
if (left_align)
print_pad(callback, ctx, ' ', pad);
written += len;
written += width > total ? width : total;
fmt++;
break;
}
@@ -272,13 +294,14 @@ print_num:
}
// No format, just normal text.
default: {
fmt = format_begun_at;
size_t len = strlen(fmt);
size_t len = fmt - format_begun_at;
if (*fmt)
len++;
callback(ctx, fmt, len);
callback(ctx, format_begun_at, len);
written += len;
fmt += len;
fmt = format_begun_at + len;
break;
}
}
core/refcount.c
+17
View File
@@ -0,0 +1,17 @@
#include <zinnia/refcount.h>
#include <stdatomic.h>
void refcount_init(struct refcount* rc) {
atomic_store_explicit(&rc->count, 1, memory_order_relaxed);
}
void refcount_inc(struct refcount* rc) {
atomic_fetch_add_explicit(&rc->count, 1, memory_order_relaxed);
}
bool refcount_dec(struct refcount* rc) {
if (atomic_fetch_sub_explicit(&rc->count, 1, memory_order_release) != 1)
return false;
atomic_thread_fence(memory_order_acquire);
return true;
}
core/sched/sched_dumb.c
+29 -11
View File
@@ -1,6 +1,5 @@
#include <zinnia/alloc.h>
#include <zinnia/assert.h>
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/list.h>
@@ -9,11 +8,12 @@
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/spin.h>
#include <zinnia/string.h>
#include <zinnia/tailq.h>
#include <zinnia/vma.h>
#include <arch/sched.h>
#include <stdatomic.h>
#include <stdint.h>
#include <zinnia/string.h>
static void idle_func(uintptr_t) {
while (1) {
@@ -39,7 +39,12 @@ void sched_init(struct scheduler* sched) {
struct task* idle_task;
assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
sched->idle_task = idle_task;
sched->current = idle_task;
struct task* boot_task = mem_alloc(sizeof(struct task), 0);
assert(boot_task != nullptr, "Unable to create boot task!\n");
strncpy(boot_task->name, "boot", sizeof(boot_task->name));
boot_task->space = &kernel_vma;
sched->current = boot_task;
tailq_init(&sched->run_queue);
@@ -76,17 +81,19 @@ static void do_reschedule(struct scheduler* sched) {
pmap_set(&to->space->pmap);
struct percpu* cpu = percpu_get();
from->kernel_stack = cpu->kernel_stack;
from->user_stack = cpu->user_stack;
cpu->kernel_stack = to->kernel_stack;
cpu->kernel_stack = to->kernel_stack + KERNEL_STACK_SIZE;
cpu->user_stack = to->user_stack;
struct task* old = arch_sched_switch(from, to);
struct irq_percpu irq_state = *percpu_ref(irq_data);
arch_sched_switch(from, to);
*percpu_ref(irq_data) = irq_state;
irq_unlock();
}
void sched_reschedule(struct scheduler* sched) {
irq_lock();
if (sched->current->state != TASK_STATE_DEAD)
if (sched->current != sched->idle_task && sched->current->state != TASK_STATE_DEAD)
sched_add_task(sched, sched->current);
do_reschedule(sched);
}
@@ -104,27 +111,38 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt
if (new_task == nullptr)
return ENOMEM;
static size_t last_tid = 0;
new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE);
static _Atomic(size_t) last_tid = 0;
new_task->id = atomic_fetch_add_explicit(&last_tid, 1, memory_order_acquire);
new_task->space = space;
strncpy(new_task->name, name, sizeof(new_task->name));
// Allocate a kernel stack.
new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0);
if (!new_task->kernel_stack)
if (!new_task->kernel_stack) {
mem_free(new_task);
return ENOMEM;
}
new_task->user_stack = 0;
// Initialize the arch-dependent parts.
errno_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
if (s != 0)
if (s != 0) {
mem_free((void*)new_task->kernel_stack);
mem_free(new_task);
return s;
}
*out = new_task;
return 0;
}
void task_entry(task_fn_t entry, uintptr_t arg0) {
*percpu_ref(irq_data) = (struct irq_percpu){
.depth = 1,
.saved_state = true,
};
irq_unlock();
entry(arg0);
struct scheduler* sched = &percpu_get()->sched;
core/spin.c
+5 -5
View File
@@ -1,15 +1,15 @@
#include <zinnia/atomic.h>
#include <zinnia/spin.h>
#include <stdatomic.h>
void spin_lock(struct spinlock* spin) {
uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED);
uint32_t my = atomic_fetch_add_explicit(&spin->next, 1, memory_order_relaxed);
while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) {
while (atomic_load_explicit(&spin->owner, memory_order_acquire) != my) {
asm volatile("pause");
}
}
void spin_unlock(struct spinlock* spin) {
uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED);
atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE);
uint32_t val = atomic_load_explicit(&spin->owner, memory_order_relaxed);
atomic_store_explicit(&spin->owner, val + 1, memory_order_release);
}