compiles
This commit is contained in:
+2
-2
@@ -31,7 +31,7 @@ bool clock_switch(struct clock* clock) {
|
||||
}
|
||||
}
|
||||
|
||||
kprintf("Switching to clock source \"%s\"\n", clock->name);
|
||||
log("clock", "Switching to clock source \"%s\"\n", clock->name);
|
||||
|
||||
uint64_t elapsed = clock_get_elapsed_ns();
|
||||
counter_base += elapsed;
|
||||
@@ -43,7 +43,7 @@ bool clock_switch(struct clock* clock) {
|
||||
|
||||
void clock_spin_ns(uint64_t ns) {
|
||||
if (active_clock == nullptr) {
|
||||
kprintf("Unable to sleep for %lu ns, no clock available!\n", ns);
|
||||
log("clock", "Unable to sleep for %lu ns, no clock available!\n", ns);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -37,7 +37,7 @@ void console_init() {
|
||||
if (active) {
|
||||
if (active->init(active) == false)
|
||||
active = nullptr;
|
||||
else
|
||||
kprintf("Logging on console \"%s\"\n", active->name);
|
||||
|
||||
log("console", "Logging on console \"%s\"\n", active->name);
|
||||
}
|
||||
}
|
||||
|
||||
+16
-18
@@ -1,18 +1,25 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/assert.h>
|
||||
#include <zinnia/elf.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/mman.h>
|
||||
#include <zinnia/percpu.h>
|
||||
#include <zinnia/pmap.h>
|
||||
#include <zinnia/print.h>
|
||||
#include <zinnia/sched.h>
|
||||
#include <zinnia/utils.h>
|
||||
#include <zinnia/vas.h>
|
||||
#include <zinnia/vdso.h>
|
||||
#include <zinnia/vma.h>
|
||||
#include <zinnia/vmo.h>
|
||||
#include <assert.h>
|
||||
#include <bits/elf.h>
|
||||
#include <string.h>
|
||||
|
||||
struct exec_info {
|
||||
struct vma* space;
|
||||
struct vmo* file_obj;
|
||||
const char** argv;
|
||||
const char** envp;
|
||||
};
|
||||
|
||||
errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
errno_t status;
|
||||
|
||||
@@ -40,13 +47,13 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
struct elf_phdr phdr = {};
|
||||
vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
|
||||
if (phdr.p_type == PT_LOAD) {
|
||||
enum zn_vm_flags prot = 0;
|
||||
enum prot_flags prot = 0;
|
||||
if (phdr.p_flags & PF_R)
|
||||
prot |= PROT_READ;
|
||||
if (phdr.p_flags & PF_W)
|
||||
prot |= ZN_VM_MAP_WRITE;
|
||||
prot |= PROT_WRITE;
|
||||
if (phdr.p_flags & PF_X)
|
||||
prot |= ZN_VM_MAP_EXEC;
|
||||
prot |= PROT_EXEC;
|
||||
|
||||
assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
|
||||
|
||||
@@ -66,7 +73,7 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
return status;
|
||||
|
||||
// We map more than we copied so the rest is filled with zeroed pages.
|
||||
status = vma_map_vmo(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
|
||||
status = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
|
||||
if (status)
|
||||
return status;
|
||||
}
|
||||
@@ -130,14 +137,6 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
|
||||
}
|
||||
|
||||
// Add the init handle to the process namespace.
|
||||
zn_handle_t init_handle = ZN_HANDLE_INVALID;
|
||||
if (info->init_handle) {
|
||||
status = namespace_add_desc(info->ns, *info->init_handle, &init_handle);
|
||||
if (status)
|
||||
return status;
|
||||
}
|
||||
|
||||
#define WRITE_AUXV(auxv, value) \
|
||||
do { \
|
||||
stack_off -= sizeof(uintptr_t); \
|
||||
@@ -154,7 +153,6 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
|
||||
WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
|
||||
WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
|
||||
WRITE_AUXV(AT_INIT_HANDLE, init_handle);
|
||||
|
||||
// envp pointers.
|
||||
stack_off -= sizeof(uintptr_t);
|
||||
@@ -181,7 +179,7 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
uintptr_t argc = num_argv;
|
||||
vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
|
||||
|
||||
status = vma_map_vmo(info->space, &stack->object, stack_start, stack_size, PROT_READ | ZN_VM_MAP_WRITE, 0);
|
||||
status = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
|
||||
if (status)
|
||||
return status;
|
||||
|
||||
@@ -191,7 +189,7 @@ errno_t elf_load(struct exec_info* info, struct task** out) {
|
||||
start_info->arg = 0;
|
||||
|
||||
struct task* result;
|
||||
status = task_create(info->argv[0], info->space, info->ns, sched_to_user, (uintptr_t)start_info, &result);
|
||||
status = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
|
||||
if (status)
|
||||
return status;
|
||||
|
||||
|
||||
+14
-1
@@ -1,6 +1,19 @@
|
||||
#include <stdatomic.h>
|
||||
#include <zinnia/irq.h>
|
||||
#include <zinnia/percpu.h>
|
||||
#include <arch/irq.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
void irq_set_state(bool state) {
|
||||
arch_irq_set_state(state);
|
||||
}
|
||||
|
||||
bool irq_get_state() {
|
||||
return arch_irq_get_state();
|
||||
}
|
||||
|
||||
void irq_wait() {
|
||||
arch_irq_wait();
|
||||
}
|
||||
|
||||
void irq_lock() {
|
||||
struct irq_percpu* cpu = &percpu_get()->irq;
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# Freestanding libc functions that are needed in the kernel
|
||||
|
||||
kernel_sources += files(
|
||||
'string.c',
|
||||
)
|
||||
@@ -0,0 +1,164 @@
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <zinnia/compiler.h>
|
||||
#include <zinnia/utils.h>
|
||||
|
||||
[[__hot]]
|
||||
size_t strlen(const char* str) {
|
||||
size_t result = 0;
|
||||
while (*str++) {
|
||||
result++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
[[__hot]]
|
||||
size_t strnlen(const char* str, size_t len) {
|
||||
size_t result = 0;
|
||||
while (result < len && *str++) {
|
||||
result++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
[[__hot]]
|
||||
int memcmp(const void* s1, const void* s2, size_t size) {
|
||||
int diff = 0;
|
||||
char* s1ptr = (char*)s1;
|
||||
char* s2ptr = (char*)s2;
|
||||
|
||||
for (size_t i = 0; i < size; i++) {
|
||||
if (s1ptr[i] != s2ptr[i])
|
||||
diff++;
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
|
||||
[[__hot]]
|
||||
void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
|
||||
if (__unlikely(n == 0))
|
||||
return dest;
|
||||
|
||||
size_t d = (size_t)dest;
|
||||
size_t s = (size_t)src;
|
||||
|
||||
if (d % sizeof(size_t) != 0 || s % sizeof(size_t) != 0) {
|
||||
while (n && (d % sizeof(size_t) != 0) && (s % sizeof(size_t) != 0)) {
|
||||
*((uint8_t*)d) = *((uint8_t*)s);
|
||||
d++;
|
||||
s++;
|
||||
n--;
|
||||
}
|
||||
}
|
||||
|
||||
size_t* qword_dest = (size_t*)d;
|
||||
const size_t* qword_src = (const size_t*)s;
|
||||
const size_t word_count = n / sizeof(size_t);
|
||||
|
||||
for (size_t i = 0; i < word_count; i++) {
|
||||
qword_dest[i] = qword_src[i];
|
||||
}
|
||||
|
||||
size_t remaining_bytes = n % sizeof(size_t);
|
||||
d = (size_t)(qword_dest + word_count);
|
||||
s = (size_t)(qword_src + word_count);
|
||||
|
||||
while (remaining_bytes--) {
|
||||
*((uint8_t*)d) = *((uint8_t*)s);
|
||||
d++;
|
||||
s++;
|
||||
}
|
||||
|
||||
return dest;
|
||||
}
|
||||
|
||||
[[__hot]]
|
||||
void* memmove(void* dstptr, const void* srcptr, size_t size) {
|
||||
uint8_t* dst = (uint8_t*)dstptr;
|
||||
const uint8_t* src = (const uint8_t*)srcptr;
|
||||
if (dst < src) {
|
||||
for (size_t i = 0; i < size; i++)
|
||||
dst[i] = src[i];
|
||||
} else {
|
||||
for (size_t i = size; i != 0; i--)
|
||||
dst[i - 1] = src[i - 1];
|
||||
}
|
||||
return dstptr;
|
||||
}
|
||||
|
||||
[[__hot]]
|
||||
void* memset(void* dest, int value, size_t n) {
|
||||
if (__unlikely(n == 0))
|
||||
return dest;
|
||||
|
||||
size_t d = (size_t)dest;
|
||||
|
||||
if (d % sizeof(size_t) != 0) {
|
||||
while (n && (d % sizeof(size_t) != 0)) {
|
||||
*((uint8_t*)d) = value;
|
||||
d++;
|
||||
n--;
|
||||
}
|
||||
}
|
||||
|
||||
size_t* qword_dest = (size_t*)d;
|
||||
size_t qword_value = value;
|
||||
qword_value |= (qword_value << 8);
|
||||
qword_value |= (qword_value << 16);
|
||||
qword_value |= (qword_value << 32);
|
||||
|
||||
const size_t word_count = n / sizeof(size_t);
|
||||
|
||||
for (size_t i = 0; i < word_count; i++) {
|
||||
qword_dest[i] = qword_value;
|
||||
}
|
||||
|
||||
size_t remaining_bytes = n % sizeof(size_t);
|
||||
d = (size_t)(qword_dest + word_count);
|
||||
|
||||
while (remaining_bytes--) {
|
||||
*((uint8_t*)d) = value;
|
||||
d++;
|
||||
}
|
||||
|
||||
return dest;
|
||||
}
|
||||
|
||||
char* strncpy(char* restrict dst, const char* restrict src, size_t len) {
|
||||
size_t src_len = strnlen(src, len) + 1;
|
||||
return memcpy(dst, src, MIN(len, src_len));
|
||||
}
|
||||
|
||||
int strncmp(const char* str1, const char* str2, size_t len) {
|
||||
while (len && *str1 && (*str1 == *str2)) {
|
||||
++str1;
|
||||
++str2;
|
||||
--len;
|
||||
}
|
||||
if (len == 0) {
|
||||
return 0;
|
||||
} else {
|
||||
return (*(unsigned char*)str1 - *(unsigned char*)str2);
|
||||
}
|
||||
}
|
||||
|
||||
int strcmp(const char* str1, const char* str2) {
|
||||
while (*str1 && (*str1 == *str2)) {
|
||||
++str1;
|
||||
++str2;
|
||||
}
|
||||
return (*(unsigned char*)str1 - *(unsigned char*)str2);
|
||||
}
|
||||
|
||||
char* strchr(const char* s, int c) {
|
||||
char ch = (char)c;
|
||||
while (true) {
|
||||
char cur = *s;
|
||||
if (cur == ch)
|
||||
return (char*)s;
|
||||
if (cur == 0)
|
||||
return nullptr;
|
||||
s++;
|
||||
}
|
||||
}
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/assert.h>
|
||||
#include <zinnia/init.h>
|
||||
#include <zinnia/panic.h>
|
||||
#include <zinnia/print.h>
|
||||
#include <zinnia/utils.h>
|
||||
#include <zinnia/vma.h>
|
||||
#include <string.h>
|
||||
|
||||
extern uint8_t __ld_text_start[];
|
||||
extern uint8_t __ld_text_end[];
|
||||
extern uint8_t __ld_rodata_start[];
|
||||
extern uint8_t __ld_rodata_end[];
|
||||
extern uint8_t __ld_data_start[];
|
||||
extern uint8_t __ld_data_end[];
|
||||
extern uint8_t __ld_kernel_start[];
|
||||
|
||||
struct vma kernel_vma = {0};
|
||||
|
||||
[[__init]]
|
||||
void mem_init(struct phys_mem* map, size_t map_len, uintptr_t kernel_virt, phys_t kernel_phys, uintptr_t tmp_hhdm) {
|
||||
// This function creates a kernel page table and initializes all memory managers.
|
||||
|
||||
const size_t pgsz = arch_mem_page_size();
|
||||
|
||||
log("mem", "Memory map:\n");
|
||||
for (size_t i = 0; i < map_len; i++) {
|
||||
if (map[i].length == 0)
|
||||
continue;
|
||||
|
||||
const char* label = nullptr;
|
||||
switch (map[i].usage) {
|
||||
case PHYS_RESERVED:
|
||||
label = "Reserved";
|
||||
break;
|
||||
case PHYS_USABLE:
|
||||
label = "Usable";
|
||||
break;
|
||||
case PHYS_STATIC:
|
||||
label = "Static";
|
||||
break;
|
||||
case PHYS_RECLAIMABLE:
|
||||
label = "Reclaimable";
|
||||
break;
|
||||
}
|
||||
|
||||
log("mem", "[%p - %p] %s\n", (void*)map[i].address, (void*)(map[i].address + map[i].length - 1), label);
|
||||
}
|
||||
|
||||
// Set up the bootstrap allocator.
|
||||
struct phys_mem* largest = map;
|
||||
for (size_t i = 0; i < map_len; i++) {
|
||||
if (map[i].length > largest->length && map[i].usage == PHYS_USABLE)
|
||||
largest = &map[i];
|
||||
}
|
||||
mem_phys_bootstrap(largest);
|
||||
log("mem",
|
||||
"Using region [%p - %p] for bootstrap allocator\n",
|
||||
(void*)largest->address,
|
||||
(void*)(largest->address + largest->length - 1));
|
||||
|
||||
// Set the HHDM base address to the address given by the loader.
|
||||
// We must not keep any virtual addresses to this region,
|
||||
// since we're likely going to map it at a different base address.
|
||||
mem_hhdm_base = tmp_hhdm;
|
||||
|
||||
assert(pmap_new_kernel(&kernel_vma.pmap, 0) == 0, "Unable to allocate the kernel page table\n");
|
||||
|
||||
// text
|
||||
log("mem", "Mapping text segment at %p\n", __ld_text_start);
|
||||
for (uint8_t* p = __ld_text_start; p <= __ld_text_end; p += pgsz) {
|
||||
errno_t status = pmap_map(
|
||||
&kernel_vma.pmap,
|
||||
(uintptr_t)p,
|
||||
(phys_t)(p - __ld_kernel_start + kernel_phys),
|
||||
PTE_READ | PTE_EXEC,
|
||||
CACHE_ANY
|
||||
);
|
||||
assert(!status, "Failed to map %p with error %i\n", p, status);
|
||||
}
|
||||
|
||||
// rodata
|
||||
log("mem", "Mapping rodata segment at %p\n", __ld_rodata_start);
|
||||
for (uint8_t* p = __ld_rodata_start; p < __ld_rodata_end; p += pgsz) {
|
||||
errno_t status = pmap_map(
|
||||
&kernel_vma.pmap,
|
||||
(uintptr_t)p,
|
||||
(phys_t)(p - __ld_kernel_start + kernel_phys),
|
||||
PTE_READ,
|
||||
CACHE_ANY
|
||||
);
|
||||
assert(!status, "Failed to map %p with error %i\n", p, status);
|
||||
}
|
||||
|
||||
// data
|
||||
log("mem", "Mapping data segment at %p\n", __ld_data_start);
|
||||
for (uint8_t* p = __ld_data_start; p < __ld_data_end; p += pgsz) {
|
||||
errno_t status = pmap_map(
|
||||
&kernel_vma.pmap,
|
||||
(uintptr_t)p,
|
||||
(phys_t)(p - __ld_kernel_start + kernel_phys),
|
||||
PTE_READ | PTE_WRITE,
|
||||
CACHE_ANY
|
||||
);
|
||||
assert(!status, "Failed to map %p with error %i\n", p, status);
|
||||
}
|
||||
|
||||
// The kernel address space is divided into 4 segments.
|
||||
// For example, they could live at:
|
||||
// HHDM FFFF'8000'0000'0000
|
||||
// PFNDB FFFF'A000'0000'0000
|
||||
// Mappings FFFF'C000'0000'0000
|
||||
// Kernel FFFF'FFFF'8000'0000
|
||||
|
||||
// Map all physical memory to the HHDM address.
|
||||
tmp_hhdm = arch_mem_hhdm_addr();
|
||||
for (size_t i = 0; i < map_len; i++) {
|
||||
if (map[i].usage != PHYS_USABLE && map[i].usage != PHYS_STATIC)
|
||||
continue;
|
||||
for (size_t p = 0; p <= map[i].length; p += pgsz) {
|
||||
uintptr_t vaddr = (uintptr_t)(map[i].address + p + tmp_hhdm);
|
||||
phys_t paddr = (phys_t)(map[i].address + p);
|
||||
errno_t status = pmap_map(&kernel_vma.pmap, vaddr, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
|
||||
assert(!status, "Failed to map HHDM page %p to %p with error %i\n", (void*)vaddr, (void*)paddr, status);
|
||||
}
|
||||
}
|
||||
mem_hhdm_base = tmp_hhdm;
|
||||
|
||||
// Switch to our own page table.
|
||||
pmap_set(&kernel_vma.pmap);
|
||||
|
||||
// We record metadata for every single page of available memory in a large array.
|
||||
// This array is contiguous in virtual memory, but is sparsely populated.
|
||||
// Only those array entries which represent usable memory are mapped.
|
||||
for (size_t i = 0; i < map_len; i++) {
|
||||
if (map[i].length == 0 || map[i].usage != PHYS_USABLE)
|
||||
continue;
|
||||
|
||||
const size_t num_pages = (map[i].length + pgsz - 1) / pgsz;
|
||||
const size_t metadata_size = num_pages * sizeof(struct page);
|
||||
const uintptr_t metadata_start_vaddr = arch_mem_pfndb_addr() + (map[i].address / pgsz * sizeof(struct page));
|
||||
const uintptr_t metadata_end_vaddr = metadata_start_vaddr + metadata_size;
|
||||
|
||||
// Find the page-aligned boundaries that contain this metadata.
|
||||
const uintptr_t aligned_start = ALIGN_DOWN(metadata_start_vaddr, pgsz);
|
||||
const uintptr_t aligned_end = ALIGN_UP(metadata_end_vaddr, pgsz);
|
||||
const size_t length = aligned_end - aligned_start;
|
||||
const uintptr_t vaddr = aligned_start;
|
||||
|
||||
for (size_t page = 0; page < length; page += pgsz) {
|
||||
phys_t paddr;
|
||||
assert(!mem_phys_alloc(1, 0, &paddr), "Failed to allocate memory for PFNDB!\n");
|
||||
pmap_map(&kernel_vma.pmap, vaddr + page, paddr, PTE_READ | PTE_WRITE, CACHE_ANY);
|
||||
}
|
||||
}
|
||||
vm_pfndb = (struct page*)arch_mem_pfndb_addr();
|
||||
|
||||
// We don't need the bootstrap allocator from this point on.
|
||||
// Initialize the real page allocator.
|
||||
mem_phys_init(map, map_len);
|
||||
|
||||
slab_init();
|
||||
|
||||
log("mem", "Memory initialization complete\n");
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
kernel_sources += files(
|
||||
'fault.c',
|
||||
'init.c',
|
||||
'pmap.c',
|
||||
'pmm.c',
|
||||
'slab.c',
|
||||
'usercopy.c',
|
||||
'virt.c',
|
||||
'vma.c',
|
||||
'vmo.c',
|
||||
)
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/pmap.h>
|
||||
#include <zinnia/spin.h>
|
||||
#include <zinnia/vma.h>
|
||||
#include <string.h>
|
||||
|
||||
errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
|
||||
flags &= ~ALLOC_NOZERO;
|
||||
|
||||
phys_t addr;
|
||||
errno_t status = mem_phys_alloc(1, flags, &addr);
|
||||
if (status)
|
||||
return status;
|
||||
|
||||
struct pmap result = {
|
||||
.root = addr,
|
||||
.lock = (spinlock_t){0},
|
||||
.is_user = false,
|
||||
};
|
||||
|
||||
*pt = result;
|
||||
return 0;
|
||||
}
|
||||
|
||||
errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
|
||||
phys_t user_l1;
|
||||
errno_t status = mem_phys_alloc(1, 0, &user_l1);
|
||||
if (status)
|
||||
return status;
|
||||
|
||||
void* user_l1_ptr = HHDM_PTR(user_l1);
|
||||
void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
|
||||
memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
|
||||
|
||||
struct pmap result = {
|
||||
.root = user_l1,
|
||||
.lock = (spinlock_t){0},
|
||||
.is_user = true,
|
||||
};
|
||||
|
||||
*pt = result;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Gets a reference to the PTE at the given virtual address.
|
||||
// If `check_only` is set, only checks if the PTE exists,
|
||||
// and doesn't allocate new levels if they don't already exist.
|
||||
// If it can't allocate a page if it has to, returns `nullptr`.
|
||||
static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) {
|
||||
pte_t* current_head = HHDM_PTR(pt->root);
|
||||
size_t index = 0;
|
||||
|
||||
for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) {
|
||||
const size_t addr_mask = (1 << arch_mem_level_bits()) - 1;
|
||||
const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level);
|
||||
const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0);
|
||||
|
||||
index = (vaddr >> addr_shift) & addr_mask;
|
||||
|
||||
// The last level is used to access the actual PTE, so break the loop then.
|
||||
// We still need to update the index beforehand, that's why we can't just end early.
|
||||
if (level == 0)
|
||||
break;
|
||||
|
||||
pte_t* pte = ¤t_head[index];
|
||||
|
||||
if (pte_is_present(pte)) {
|
||||
// Get the next level.
|
||||
*pte = pte_build(pte_address(pte), level_flags, CACHE_ANY);
|
||||
current_head = HHDM_PTR(pte_address(pte));
|
||||
} else {
|
||||
// If the current level isn't present, we can skip the rest.
|
||||
if (check_only)
|
||||
return nullptr;
|
||||
|
||||
phys_t addr;
|
||||
if (mem_phys_alloc(1, 0, &addr) != 0)
|
||||
return nullptr;
|
||||
|
||||
*pte = pte_build(addr, level_flags, CACHE_ANY);
|
||||
current_head = HHDM_PTR(addr);
|
||||
}
|
||||
}
|
||||
|
||||
return ¤t_head[index];
|
||||
}
|
||||
|
||||
errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
|
||||
spin_lock(&pt->lock);
|
||||
|
||||
errno_t status = 0;
|
||||
pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
|
||||
if (pte == nullptr) {
|
||||
status = ENOMEM;
|
||||
goto fail;
|
||||
}
|
||||
|
||||
*pte = pte_build(paddr, flags, cache);
|
||||
|
||||
fail:
|
||||
spin_unlock(&pt->lock);
|
||||
return status;
|
||||
}
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/compiler.h>
|
||||
#include <zinnia/panic.h>
|
||||
#include <zinnia/print.h>
|
||||
#include <zinnia/spin.h>
|
||||
#include <zinnia/virt.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
uintptr_t mem_hhdm_base = 0;
|
||||
|
||||
static errno_t (*alloc_fn)(size_t num_pages, enum alloc_flags flags, phys_t* out);
|
||||
static errno_t (*free_fn)(phys_t start, size_t num_pages);
|
||||
|
||||
errno_t mem_phys_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
|
||||
return alloc_fn(num_pages, flags, out);
|
||||
}
|
||||
|
||||
errno_t mem_phys_free(phys_t start, size_t num_pages) {
|
||||
return free_fn(start, num_pages);
|
||||
}
|
||||
|
||||
static struct phys_mem bump_mem = {0};
|
||||
static struct phys_mem* bump_region = nullptr;
|
||||
|
||||
static errno_t bump_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
|
||||
const size_t bytes = num_pages * arch_mem_page_size();
|
||||
if (((intptr_t)bump_mem.length - bytes) <= 0)
|
||||
return ENOMEM;
|
||||
|
||||
*out = bump_mem.address;
|
||||
bump_mem.length -= bytes;
|
||||
bump_mem.address += bytes;
|
||||
|
||||
if (!(flags & ALLOC_NOZERO)) {
|
||||
memset(HHDM_PTR(*out), 0, bytes);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static errno_t bump_free(phys_t addr, size_t num_pages) {
|
||||
// We don't free bootstrap memory.
|
||||
panic("Attempted to free bootstrap memory!");
|
||||
}
|
||||
|
||||
void mem_phys_bootstrap(struct phys_mem* mem) {
|
||||
alloc_fn = bump_alloc;
|
||||
free_fn = bump_free;
|
||||
|
||||
// Save the region by reference and value.
|
||||
// We can't directly modify the region by reference, because that
|
||||
// interferes with e.g. HHDM mapping logic.
|
||||
bump_mem = *mem;
|
||||
bump_region = mem;
|
||||
}
|
||||
|
||||
static struct page* pmm_head = nullptr;
|
||||
static spinlock_t pmm_lock = {0};
|
||||
static size_t pmm_total_free = 0;
|
||||
|
||||
static errno_t freelist_alloc(size_t num_pages, enum alloc_flags flags, phys_t* out) {
|
||||
spin_lock(&pmm_lock);
|
||||
|
||||
const size_t bytes = num_pages * arch_mem_page_size();
|
||||
phys_t limit = UINTPTR_MAX;
|
||||
if (flags & ALLOC_MEM20) {
|
||||
limit = (phys_t)1 << 20;
|
||||
} else if (flags & ALLOC_MEM32) {
|
||||
limit = (phys_t)1 << 32;
|
||||
}
|
||||
|
||||
struct page* it = pmm_head;
|
||||
struct page* prev_it = nullptr;
|
||||
|
||||
while (it) {
|
||||
const phys_t page_addr = (((uintptr_t)it - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
|
||||
// If the region is too high up or doesn't have enough pages left, continue searching.
|
||||
if (page_addr + bytes >= limit || __unlikely(it->freelist.count < num_pages)) {
|
||||
prev_it = it;
|
||||
it = it->freelist.next;
|
||||
continue;
|
||||
}
|
||||
|
||||
// If the current block has exactly enough pages left, consume it entirely.
|
||||
if (__unlikely(it->freelist.count == num_pages)) {
|
||||
if (prev_it) {
|
||||
prev_it->freelist.next = it->freelist.next;
|
||||
} else {
|
||||
pmm_head = it->freelist.next;
|
||||
}
|
||||
|
||||
it->freelist.next = nullptr;
|
||||
it->freelist.count = 0;
|
||||
|
||||
if (__likely(out))
|
||||
*out = page_addr;
|
||||
}
|
||||
// If not, we can just shrink the block.
|
||||
else {
|
||||
it->freelist.count -= num_pages;
|
||||
|
||||
if (__likely(out))
|
||||
*out = page_addr + (it->freelist.count * arch_mem_page_size());
|
||||
}
|
||||
goto success;
|
||||
}
|
||||
|
||||
spin_unlock(&pmm_lock);
|
||||
return ENOMEM;
|
||||
|
||||
success:
|
||||
// If the NOZERO flag is *not* specified, zero memory.
|
||||
if (__likely(out) && __likely(!(flags & ALLOC_NOZERO))) {
|
||||
memset(HHDM_PTR(*out), 0, bytes);
|
||||
}
|
||||
pmm_total_free -= num_pages;
|
||||
spin_unlock(&pmm_lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static errno_t freelist_free(phys_t addr, size_t num_pages) {
|
||||
spin_lock(&pmm_lock);
|
||||
|
||||
struct page* page = &vm_pfndb[addr / arch_mem_page_size()];
|
||||
page->freelist.count = num_pages;
|
||||
page->freelist.next = pmm_head;
|
||||
pmm_head = page;
|
||||
|
||||
spin_unlock(&pmm_lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void mem_phys_init(struct phys_mem* map, size_t length) {
|
||||
alloc_fn = freelist_alloc;
|
||||
free_fn = freelist_free;
|
||||
bump_region->address = bump_mem.address;
|
||||
bump_region->length = bump_mem.length;
|
||||
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
// Regions smaller than a page are useless.
|
||||
if (map[i].length < arch_mem_page_size() || map[i].usage != PHYS_USABLE)
|
||||
continue;
|
||||
|
||||
struct page* page = &vm_pfndb[map[i].address / arch_mem_page_size()];
|
||||
page->freelist.count = map[i].length / arch_mem_page_size();
|
||||
page->freelist.next = pmm_head;
|
||||
pmm_head = page;
|
||||
pmm_total_free += page->freelist.count;
|
||||
}
|
||||
|
||||
const size_t free_bytes = (pmm_total_free * arch_mem_page_size());
|
||||
log("pmm", "Total available memory: %zu MiB\n", free_bytes / 1024 / 1024);
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/compiler.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/spin.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
size_t num_pages; // Amount of pages connected to this slab.
|
||||
size_t size; // Size of this slab.
|
||||
} slab_info;
|
||||
|
||||
struct slab {
|
||||
spinlock_t lock; // Access lock.
|
||||
size_t ent_size; // Size of one entry.
|
||||
void** head;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
struct slab* slab;
|
||||
} slab_header;
|
||||
|
||||
// Initializes the SLAB structures.
|
||||
|
||||
static struct slab slabs[8] = {0};
|
||||
|
||||
static errno_t slab_new(struct slab* slab, size_t size) {
|
||||
slab->lock = (spinlock_t){0};
|
||||
|
||||
// Allocate a new page for the head.
|
||||
phys_t addr;
|
||||
errno_t status = mem_phys_alloc(1, 0, &addr);
|
||||
if (status)
|
||||
return status;
|
||||
|
||||
slab->head = (void**)(addr + arch_mem_hhdm_addr());
|
||||
slab->ent_size = size;
|
||||
|
||||
const size_t offset = ALIGN_UP(sizeof(slab_header), size);
|
||||
const size_t available_size = arch_mem_page_size() - offset;
|
||||
|
||||
slab_header* ptr = (slab_header*)slab->head;
|
||||
ptr->slab = slab;
|
||||
slab->head = (void**)((void*)slab->head + offset);
|
||||
|
||||
void** arr = slab->head;
|
||||
const size_t max = available_size / size - 1;
|
||||
const size_t fact = size / sizeof(void*);
|
||||
|
||||
for (size_t i = 0; i < max; i++) {
|
||||
arr[i * fact] = &arr[(i + 1) * fact];
|
||||
}
|
||||
arr[max * fact] = nullptr;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void slab_init(void) {
|
||||
slab_new(&slabs[0], 16);
|
||||
slab_new(&slabs[1], 32);
|
||||
slab_new(&slabs[2], 64);
|
||||
slab_new(&slabs[3], 128);
|
||||
slab_new(&slabs[4], 256);
|
||||
slab_new(&slabs[5], 512);
|
||||
slab_new(&slabs[6], 1024);
|
||||
slab_new(&slabs[7], 2048);
|
||||
}
|
||||
|
||||
static void* slab_do_alloc(struct slab* slab) {
|
||||
spin_lock(&slab->lock);
|
||||
|
||||
if (__unlikely(slab->head == nullptr))
|
||||
slab_new(slab, slab->ent_size);
|
||||
|
||||
void** old_free = slab->head;
|
||||
slab->head = *old_free;
|
||||
memset(old_free, 0, slab->ent_size);
|
||||
|
||||
spin_unlock(&slab->lock);
|
||||
return old_free;
|
||||
}
|
||||
|
||||
static void slab_do_free(struct slab* slab, void* addr) {
|
||||
spin_lock(&slab->lock);
|
||||
|
||||
if (__unlikely(addr == nullptr))
|
||||
goto cleanup;
|
||||
|
||||
void** new_head = addr;
|
||||
*new_head = slab->head;
|
||||
slab->head = new_head;
|
||||
|
||||
cleanup:
|
||||
spin_unlock(&slab->lock);
|
||||
}
|
||||
|
||||
static inline struct slab* slab_find_size(size_t size) {
|
||||
for (size_t i = 0; i < ARRAY_SIZE(slabs); i++) {
|
||||
if (slabs[i].ent_size >= size)
|
||||
return &slabs[i];
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void* mem_alloc(size_t size, enum alloc_flags flags) {
|
||||
if (__unlikely(size == 0))
|
||||
return nullptr;
|
||||
|
||||
struct slab* slab = slab_find_size(size);
|
||||
if (slab != nullptr) {
|
||||
return slab_do_alloc(slab);
|
||||
}
|
||||
|
||||
size_t num_pages = ROUND_UP(size, arch_mem_page_size());
|
||||
|
||||
// Allocate the pages plus an additional page for metadata.
|
||||
phys_t ret;
|
||||
errno_t status = mem_phys_alloc(num_pages + 1, 0, &ret);
|
||||
if (__unlikely(status != 0))
|
||||
return nullptr;
|
||||
|
||||
ret = ret + (phys_t)arch_mem_hhdm_addr();
|
||||
// Write metadata into the first page.
|
||||
slab_info* info = (slab_info*)ret;
|
||||
info->num_pages = num_pages;
|
||||
info->size = size;
|
||||
// Skip the first page and return the next one.
|
||||
return (void*)(ret + arch_mem_page_size());
|
||||
}
|
||||
|
||||
void mem_free(void* addr) {
|
||||
if (__unlikely(addr == nullptr))
|
||||
return;
|
||||
|
||||
// If the address is page aligned.
|
||||
if ((size_t)addr == ALIGN_DOWN((size_t)addr, arch_mem_page_size())) {
|
||||
slab_info* info = (slab_info*)(addr - arch_mem_page_size());
|
||||
mem_phys_free(((phys_t)info - (phys_t)arch_mem_hhdm_addr()), info->num_pages + 1);
|
||||
return;
|
||||
}
|
||||
|
||||
slab_header* header = (slab_header*)(ALIGN_DOWN((size_t)addr, arch_mem_page_size()));
|
||||
slab_do_free(header->slab, addr);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
#include <zinnia/percpu.h>
|
||||
#include <zinnia/usercopy.h>
|
||||
#include <arch/usercopy.h>
|
||||
|
||||
bool usercopy_read(void* dst, const __user void* src, size_t len) {
|
||||
return arch_usercopy_read(dst, src, len, &percpu_get()->sched.current->usercopy_region);
|
||||
}
|
||||
|
||||
bool usercopy_write(__user void* dst, const void* src, size_t len) {
|
||||
return arch_usercopy_write(dst, src, len, &percpu_get()->sched.current->usercopy_region);
|
||||
}
|
||||
|
||||
bool usercopy_strlen(const __user char* str, size_t max, size_t* len) {
|
||||
return arch_usercopy_strlen(str, max, len, &percpu_get()->sched.current->usercopy_region);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/utils.h>
|
||||
#include <zinnia/virt.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stddef.h>
|
||||
|
||||
struct page* vm_pfndb = nullptr;
|
||||
|
||||
// TODO: Replace with actual allocator.
|
||||
|
||||
static size_t mapping_offset = 0;
|
||||
|
||||
uintptr_t vm_alloc(size_t length) {
|
||||
uintptr_t base = arch_mem_mapping_addr();
|
||||
size_t offset =
|
||||
atomic_fetch_add_explicit(&mapping_offset, ALIGN_UP(length, arch_mem_page_size()), memory_order_acquire);
|
||||
|
||||
return base + offset;
|
||||
}
|
||||
|
||||
void vm_free(uintptr_t ptr, size_t length) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
uintptr_t vm_page_addr(struct page* page) {
|
||||
return (((uintptr_t)page - (uintptr_t)vm_pfndb) / sizeof(struct page)) * arch_mem_page_size();
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/assert.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/mman.h>
|
||||
#include <zinnia/pmap.h>
|
||||
#include <zinnia/utils.h>
|
||||
#include <zinnia/vma.h>
|
||||
|
||||
// Try to create a new, empty virtual address space.
|
||||
errno_t vma_new(struct vma** out) {
|
||||
debug_assert(out != nullptr, "Missing out");
|
||||
|
||||
errno_t res;
|
||||
if (!out)
|
||||
return EINVAL;
|
||||
|
||||
struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
|
||||
if (!vmspace)
|
||||
return ENOMEM;
|
||||
|
||||
res = pmap_new_user(&vmspace->pmap, 0);
|
||||
if (res)
|
||||
goto err1;
|
||||
|
||||
*out = vmspace;
|
||||
return 0;
|
||||
|
||||
err1:
|
||||
mem_free(vmspace);
|
||||
err0:
|
||||
return res;
|
||||
}
|
||||
|
||||
// Delete a virtual address space.
|
||||
void vma_free(struct vma* v) {}
|
||||
|
||||
// Remove any mappings in the given address range.
|
||||
// May fail if there is no memory and a mapping needs to be split.
|
||||
errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size);
|
||||
|
||||
// Change the protection of an existing mapping.
|
||||
// May fail if there is no memory and a mapping needs to be split.
|
||||
errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags);
|
||||
|
||||
// Try to handle a page fault.
|
||||
bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
|
||||
// TODO
|
||||
return false;
|
||||
}
|
||||
|
||||
errno_t vma_map(
|
||||
struct vma* vma,
|
||||
struct vmo* vmo,
|
||||
uintptr_t addr,
|
||||
size_t len,
|
||||
enum prot_flags flags,
|
||||
uintptr_t vmo_offset
|
||||
) {
|
||||
const size_t page_size = arch_mem_page_size();
|
||||
|
||||
if (addr % page_size != vmo_offset % page_size)
|
||||
return EINVAL;
|
||||
|
||||
size_t start_page = addr / page_size;
|
||||
const size_t addr_offset = addr % page_size;
|
||||
const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size;
|
||||
size_t end_page = start_page + pages;
|
||||
|
||||
if (!vma || !vmo)
|
||||
return EINVAL;
|
||||
|
||||
const size_t first_vmo_page = vmo_offset / page_size;
|
||||
|
||||
for (size_t p = start_page; p < end_page; ++p) {
|
||||
const size_t page_index = p - start_page;
|
||||
struct page* page_obj;
|
||||
errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj);
|
||||
if (status != 0)
|
||||
return status;
|
||||
|
||||
phys_t paddr = (phys_t)vm_page_addr(page_obj);
|
||||
|
||||
enum pte_flags pf = 0;
|
||||
if (flags & PROT_READ)
|
||||
pf |= PTE_READ;
|
||||
if (flags & PROT_WRITE)
|
||||
pf |= PTE_WRITE;
|
||||
if (flags & PROT_EXEC)
|
||||
pf |= PTE_EXEC;
|
||||
if (vma->pmap.is_user)
|
||||
pf |= PTE_USER;
|
||||
|
||||
status = pmap_map(&vma->pmap, p * page_size, paddr, pf, CACHE_ANY);
|
||||
if (status != 0)
|
||||
return status;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/list.h>
|
||||
#include <zinnia/utils.h>
|
||||
#include <zinnia/vmo.h>
|
||||
#include <string.h>
|
||||
|
||||
errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
|
||||
const size_t page_size = arch_mem_page_size();
|
||||
size_t progress = 0;
|
||||
errno_t status = 0;
|
||||
|
||||
while (progress < len) {
|
||||
const size_t misalign = (progress + offset) % page_size;
|
||||
const size_t page_index = (progress + offset) / page_size;
|
||||
const size_t copy_size = MIN(page_size - misalign, len - progress);
|
||||
|
||||
struct page* p;
|
||||
status = obj->get_page(obj, page_index, &p);
|
||||
if (status != 0)
|
||||
break;
|
||||
|
||||
uintptr_t page_addr = vm_page_addr(p);
|
||||
|
||||
memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
|
||||
progress += copy_size;
|
||||
}
|
||||
|
||||
if (actual_read)
|
||||
*actual_read = progress;
|
||||
return status;
|
||||
}
|
||||
|
||||
errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
|
||||
const size_t page_size = arch_mem_page_size();
|
||||
size_t progress = 0;
|
||||
errno_t status = 0;
|
||||
|
||||
while (progress < len) {
|
||||
const size_t misalign = (progress + offset) % page_size;
|
||||
const size_t page_index = (progress + offset) / page_size;
|
||||
const size_t copy_size = MIN(page_size - misalign, len - progress);
|
||||
|
||||
struct page* p;
|
||||
status = obj->get_page(obj, page_index, &p);
|
||||
if (status != 0)
|
||||
break;
|
||||
|
||||
uintptr_t page_addr = vm_page_addr(p);
|
||||
|
||||
memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
|
||||
progress += copy_size;
|
||||
}
|
||||
|
||||
if (actual_written)
|
||||
*actual_written = progress;
|
||||
return status;
|
||||
}
|
||||
|
||||
errno_t vmo_copy(
|
||||
struct vmo* target,
|
||||
uintptr_t target_offset,
|
||||
struct vmo* src,
|
||||
uintptr_t src_offset,
|
||||
size_t len,
|
||||
size_t* actual_copied
|
||||
) {
|
||||
const size_t page_size = arch_mem_page_size();
|
||||
size_t progress = 0;
|
||||
errno_t status = 0;
|
||||
|
||||
while (progress < len) {
|
||||
const size_t target_misalign = (progress + target_offset) % page_size;
|
||||
const size_t src_misalign = (progress + src_offset) % page_size;
|
||||
const size_t target_page_index = (progress + target_offset) / page_size;
|
||||
const size_t src_page_index = (progress + src_offset) / page_size;
|
||||
|
||||
const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
|
||||
|
||||
struct page* tpage;
|
||||
status = target->get_page(target, target_page_index, &tpage);
|
||||
if (status != 0)
|
||||
break;
|
||||
|
||||
struct page* spage;
|
||||
status = src->get_page(src, src_page_index, &spage);
|
||||
if (status != 0)
|
||||
break;
|
||||
|
||||
uintptr_t taddr = vm_page_addr(tpage);
|
||||
uintptr_t saddr = vm_page_addr(spage);
|
||||
|
||||
memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
|
||||
progress += copy_size;
|
||||
}
|
||||
|
||||
if (actual_copied)
|
||||
*actual_copied = progress;
|
||||
return status;
|
||||
}
|
||||
|
||||
static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
|
||||
struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
|
||||
|
||||
// Try to find the requested page in the cache.
|
||||
struct page_list* iter;
|
||||
list_foreach(iter, &paged->cache, next) {
|
||||
if (iter->offset != offset_idx)
|
||||
continue;
|
||||
|
||||
// We found a cached page!
|
||||
*out = iter->value;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Get page from out pager if we don't have one already.
|
||||
struct page* new_page;
|
||||
errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
|
||||
if (status)
|
||||
return status;
|
||||
|
||||
struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
|
||||
if (!entry)
|
||||
return ENOMEM;
|
||||
|
||||
entry->offset = offset_idx;
|
||||
entry->value = new_page;
|
||||
|
||||
// Add page to the object cache.
|
||||
list_insert_head(&paged->cache, entry, next);
|
||||
|
||||
*out = new_page;
|
||||
return 0;
|
||||
}
|
||||
|
||||
errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
|
||||
struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
|
||||
if (!result)
|
||||
return ENOMEM;
|
||||
|
||||
result->source = pager;
|
||||
result->object.get_page = paged_get_page;
|
||||
|
||||
*out = result;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
|
||||
phys_t addr;
|
||||
errno_t status = mem_phys_alloc(1, 0, &addr);
|
||||
if (status != 0)
|
||||
return status;
|
||||
size_t idx = addr / arch_mem_page_size();
|
||||
*out = &vm_pfndb[idx];
|
||||
return 0;
|
||||
}
|
||||
|
||||
static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
|
||||
// We do nothing here.
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct pager_ops phys_pager = {
|
||||
.get_page = phys_get_page,
|
||||
.put_page = phys_put_page,
|
||||
};
|
||||
|
||||
errno_t vmo_new_phys(struct paged_vmo** out) {
|
||||
return vmo_new_paged(phys_pager, out);
|
||||
}
|
||||
|
||||
struct phys_range_vmo {
|
||||
struct vmo object;
|
||||
uintptr_t start;
|
||||
size_t len;
|
||||
};
|
||||
|
||||
static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
|
||||
struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
|
||||
*out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
|
||||
return 0;
|
||||
}
|
||||
|
||||
errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
|
||||
struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
|
||||
if (!vmo)
|
||||
return ENOMEM;
|
||||
|
||||
vmo->object.get_page = phys_range_get_page;
|
||||
vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
|
||||
vmo->len = ALIGN_UP(length, arch_mem_page_size());
|
||||
|
||||
*out = &vmo->object;
|
||||
return 0;
|
||||
}
|
||||
+4
-1
@@ -10,4 +10,7 @@ kernel_sources += files(
|
||||
'spin.c',
|
||||
)
|
||||
|
||||
kernel_sources += files('sched_dumb.c')
|
||||
subdir('lib')
|
||||
subdir('mem')
|
||||
subdir('sched')
|
||||
subdir('syscalls')
|
||||
|
||||
+11
-7
@@ -4,19 +4,21 @@
|
||||
#include <zinnia/print.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
#define panic_log(msg, ...) kprintf("[ !!!! ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
|
||||
|
||||
[[noreturn]]
|
||||
void panic(const char* msg, ...) {
|
||||
// TODO: Stop all other CPUs with IPI.
|
||||
arch_irq_set_state(false);
|
||||
irq_set_state(false);
|
||||
|
||||
struct percpu* cpu = percpu_get();
|
||||
struct task* current = cpu->sched.current;
|
||||
kprintf(
|
||||
"----[ Kernel panic ]----\n"
|
||||
"In task \"%s\" (TID %zu) on CPU %zu!\n",
|
||||
current ? current->name : nullptr,
|
||||
"[ !!!! ] \e[31m"
|
||||
"Kernel panic on CPU %zu (T%zu, \"%s\"): ",
|
||||
cpu->id,
|
||||
current ? current->id : 0,
|
||||
cpu->id
|
||||
current ? current->name : nullptr
|
||||
);
|
||||
|
||||
va_list args;
|
||||
@@ -24,9 +26,11 @@ void panic(const char* msg, ...) {
|
||||
kvprintf(msg, args);
|
||||
va_end(args);
|
||||
|
||||
kprintf("----[ End of panic ]----\n");
|
||||
kprintf("\e[0m");
|
||||
|
||||
panic_log("End of panic message\n");
|
||||
|
||||
while (1) {
|
||||
arch_irq_wait();
|
||||
irq_wait();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include <zinnia/compiler.h>
|
||||
#include <zinnia/percpu.h>
|
||||
#include <arch/percpu.h>
|
||||
|
||||
[[__used, __section(".percpu")]]
|
||||
struct percpu percpu_bsp = {
|
||||
@@ -12,3 +13,15 @@ struct percpu* percpu_new() {
|
||||
// TODO
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
struct percpu* percpu_get() {
|
||||
return arch_percpu_get();
|
||||
}
|
||||
|
||||
void percpu_bsp_init() {
|
||||
arch_percpu_bsp_init();
|
||||
}
|
||||
|
||||
void percpu_init() {
|
||||
arch_percpu_init();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
kernel_sources += files(
|
||||
'sched_dumb.c',
|
||||
)
|
||||
@@ -1,4 +1,5 @@
|
||||
#include <zinnia/alloc.h>
|
||||
#include <zinnia/assert.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/irq.h>
|
||||
#include <zinnia/list.h>
|
||||
@@ -8,17 +9,15 @@
|
||||
#include <zinnia/sched.h>
|
||||
#include <zinnia/spin.h>
|
||||
#include <zinnia/tailq.h>
|
||||
#include <zinnia/vas.h>
|
||||
#include <assert.h>
|
||||
#include <bits/irq.h>
|
||||
#include <zinnia/vma.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
static void idle_func(uintptr_t) {
|
||||
while (1) {
|
||||
arch_irq_set_state(true);
|
||||
arch_irq_wait();
|
||||
irq_set_state(true);
|
||||
irq_wait();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,13 +36,13 @@ void sched_to_user_context(uintptr_t ctx) {
|
||||
|
||||
void sched_init(struct sched_percpu* sched) {
|
||||
struct task* idle_task;
|
||||
assert(task_create("idle", &kernel_vas, nullptr, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
|
||||
assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
|
||||
sched->idle_task = idle_task;
|
||||
sched->current = idle_task;
|
||||
|
||||
tailq_init(&sched->run_queue);
|
||||
|
||||
kprintf("Scheduler initialized\n");
|
||||
log("sched", "Scheduler initialized\n");
|
||||
}
|
||||
|
||||
void sched_add_task(struct sched_percpu* sched, struct task* task) {
|
||||
@@ -96,14 +95,7 @@ void sched_yield(struct sched_percpu* sched) {
|
||||
do_reschedule(sched);
|
||||
}
|
||||
|
||||
errno_t task_create(
|
||||
const char* name,
|
||||
struct vas* space,
|
||||
struct namespace* ns,
|
||||
task_fn_t entry,
|
||||
uintptr_t arg0,
|
||||
struct task** out
|
||||
) {
|
||||
errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
|
||||
if (!out)
|
||||
return EINVAL;
|
||||
|
||||
@@ -114,7 +106,6 @@ errno_t task_create(
|
||||
static size_t last_tid = 0;
|
||||
new_task->id = atomic_fetch_add(&last_tid, 1);
|
||||
new_task->space = space;
|
||||
new_task->namespace = ns;
|
||||
strncpy(new_task->name, name, sizeof(new_task->name));
|
||||
|
||||
// Allocate a kernel stack.
|
||||
@@ -0,0 +1,6 @@
|
||||
#include <zinnia/syscall.h>
|
||||
#include <arch/archctl.h>
|
||||
|
||||
void sys_archctl(struct arch_context* ctx) {
|
||||
ctx->ARCH_CTX_RET0 = arch_archctl(ctx->ARCH_CTX_A0, ctx->ARCH_CTX_A1);
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
#include <zinnia/compiler.h>
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/print.h>
|
||||
#include <zinnia/sched.h>
|
||||
#include <zinnia/syscall.h>
|
||||
#include <zinnia/syscall_list.h>
|
||||
#include <zinnia/types.h>
|
||||
#include <zinnia/utils.h>
|
||||
#include <bits/sched.h>
|
||||
|
||||
const struct syscall syscall_table[256] = {
|
||||
// [0] = {sys_exit, 0},
|
||||
// [1] = {sys_syslog, 0},
|
||||
[2] = {sys_getuname, 0},
|
||||
// [3] = {sys_setuname, 0},
|
||||
[4] = {sys_archctl, 0},
|
||||
// [5] = {sys_reboot, 0},
|
||||
// [6] = {sys_mmap, 0},
|
||||
// [7] = {sys_munmap, 0},
|
||||
// [8] = {sys_mprotect, 0},
|
||||
// [9] = {sys_madvise, 0},
|
||||
// [10] = {sys_sigprocmask, 0},
|
||||
// [11] = {sys_sigsuspend, 0},
|
||||
// [12] = {sys_sigpending, 0},
|
||||
// [13] = {sys_sigaction, 0},
|
||||
// [14] = {sys_sigtimedwait, 0},
|
||||
// [15] = {sys_sigaltstack, 0},
|
||||
// [16] = {sys_sigreturn, 0},
|
||||
// [17] = {sys_sigqueue, 0},
|
||||
// [18] = {sys_execve, 0},
|
||||
// [19] = {sys_fexecve, 0},
|
||||
// [20] = {sys_fork, 0},
|
||||
// [21] = {sys_kill, 0},
|
||||
// [22] = {sys_gettid, 0},
|
||||
// [23] = {sys_getpid, 0},
|
||||
// [24] = {sys_getppid, 0},
|
||||
// [25] = {sys_waitid, 0},
|
||||
// [26] = {sys_waitpid, 0},
|
||||
// [27] = {sys_pread, 0},
|
||||
// [28] = {sys_readv, 0},
|
||||
// [29] = {sys_pwrite, 0},
|
||||
// [30] = {sys_writev, 0},
|
||||
// [31] = {sys_seek, 0},
|
||||
// [32] = {sys_ioctl, 0},
|
||||
// [33] = {sys_openat, 0},
|
||||
// [34] = {sys_close, 0},
|
||||
// [35] = {sys_fstat, 0},
|
||||
// [36] = {sys_fstatat, 0},
|
||||
// [37] = {sys_statvfs, 0},
|
||||
// [38] = {sys_fstatvfs, 0},
|
||||
// [39] = {sys_faccessat, 0},
|
||||
// [40] = {sys_fcntl, 0},
|
||||
// [41] = {sys_ftruncate, 0},
|
||||
// [42] = {sys_truncate, 0},
|
||||
// [43] = {sys_fallocate, 0},
|
||||
// [44] = {sys_fadvise, 0},
|
||||
// [45] = {sys_utimensat, 0},
|
||||
// [46] = {sys_mknodat, 0},
|
||||
// [47] = {sys_getcwd, 0},
|
||||
// [48] = {sys_chdir, 0},
|
||||
// [49] = {sys_fchdir, 0},
|
||||
// [50] = {sys_mkdirat, 0},
|
||||
// [51] = {sys_rmdirat, 0},
|
||||
// [52] = {sys_getdents, 0},
|
||||
// [53] = {sys_renameat, 0},
|
||||
// [54] = {sys_fchmod, 0},
|
||||
// [55] = {sys_fchmodat, 0},
|
||||
// [56] = {sys_fchownat, 0},
|
||||
// [57] = {sys_linkat, 0},
|
||||
// [58] = {sys_symlinkat, 0},
|
||||
// [59] = {sys_unlinkat, 0},
|
||||
// [60] = {sys_readlinkat, 0},
|
||||
// [61] = {sys_flock, 0},
|
||||
// [62] = {sys_ppoll, 0},
|
||||
// [63] = {sys_dup, 0},
|
||||
// [64] = {sys_dup3, 0},
|
||||
// [65] = {sys_sync, 0},
|
||||
// [66] = {sys_fsync, 0},
|
||||
// [67] = {sys_fdatasync, 0},
|
||||
// [68] = {sys_getgroups, 0},
|
||||
// [69] = {sys_setgroups, 0},
|
||||
// [70] = {sys_getsid, 0},
|
||||
// [71] = {sys_setsid, 0},
|
||||
// [72] = {sys_setuid, 0},
|
||||
// [73] = {sys_getuid, 0},
|
||||
// [74] = {sys_setgid, 0},
|
||||
// [75] = {sys_getgid, 0},
|
||||
// [76] = {sys_geteuid, 0},
|
||||
// [77] = {sys_seteuid, 0},
|
||||
// [78] = {sys_getegid, 0},
|
||||
// [79] = {sys_setegid, 0},
|
||||
// [80] = {sys_getpgid, 0},
|
||||
// [81] = {sys_setpgid, 0},
|
||||
// [82] = {sys_getresuid, 0},
|
||||
// [83] = {sys_setresuid, 0},
|
||||
// [84] = {sys_getresgid, 0},
|
||||
// [85] = {sys_setresgid, 0},
|
||||
// [86] = {sys_setreuid, 0},
|
||||
// [87] = {sys_setregid, 0},
|
||||
// [88] = {sys_umask, 0},
|
||||
// [89] = {sys_pipe, 0},
|
||||
// [90] = {sys_futex_wait, 0},
|
||||
// [91] = {sys_futex_wake, 0},
|
||||
// [92] = {sys_thread_create, 0},
|
||||
// [93] = {sys_thread_kill, 0},
|
||||
// [94] = {sys_thread_exit, 0},
|
||||
// [95] = {sys_thread_setname, 0},
|
||||
// [96] = {sys_thread_getname, 0},
|
||||
// [97] = {sys_timer_create, 0},
|
||||
// [98] = {sys_timer_set, 0},
|
||||
// [99] = {sys_timer_get, 0},
|
||||
// [100] = {sys_timer_getoverrun, 0},
|
||||
// [101] = {sys_timer_delete, 0},
|
||||
// [102] = {sys_itimer_get, 0},
|
||||
// [103] = {sys_itimer_set, 0},
|
||||
// [104] = {sys_clock_get, 0},
|
||||
// [105] = {sys_clock_getres, 0},
|
||||
// [106] = {sys_clock_set, 0},
|
||||
// [107] = {sys_sleep, 0},
|
||||
// [108] = {sys_yield, 0},
|
||||
// [109] = {sys_chroot, 0},
|
||||
// [110] = {sys_mount, 0},
|
||||
// [111] = {sys_umount, 0},
|
||||
// [112] = {sys_socket, 0},
|
||||
// [113] = {sys_socketpair, 0},
|
||||
// [114] = {sys_shutdown, 0},
|
||||
// [115] = {sys_bind, 0},
|
||||
// [116] = {sys_connect, 0},
|
||||
// [117] = {sys_accept, 0},
|
||||
// [118] = {sys_listen, 0},
|
||||
// [119] = {sys_getpeername, 0},
|
||||
// [120] = {sys_getsockname, 0},
|
||||
// [121] = {sys_getsockopt, 0},
|
||||
// [122] = {sys_setsockopt, 0},
|
||||
// [123] = {sys_sendmsg, 0},
|
||||
// [124] = {sys_recvmsg, 0},
|
||||
// [125] = {sys_getentropy, 0},
|
||||
// [126] = {sys_getrusage, 0},
|
||||
// [127] = {sys_getrlimit, 0},
|
||||
// [128] = {sys_setrlimit, 0},
|
||||
// [129] = {sys_getpriority, 0},
|
||||
// [130] = {sys_setpriority, 0},
|
||||
// [131] = {sys_sched_getparam, 0},
|
||||
// [132] = {sys_sched_setparam, 0},
|
||||
// [133] = {sys_module_insert, 0},
|
||||
// [134] = {sys_module_remove, 0},
|
||||
// [135] = {sys_epoll_create, 0},
|
||||
// [136] = {sys_epoll_ctl, 0},
|
||||
// [137] = {sys_epoll_pwait, 0},
|
||||
// [138] = {sys_timerfd_create, 0},
|
||||
// [139] = {sys_timerfd_gettime, 0},
|
||||
// [140] = {sys_timerfd_settime, 0},
|
||||
// [141] = {sys_signalfd_create, 0},
|
||||
// [142] = {sys_eventfd_create, 0},
|
||||
// [143] = {sys_msync, 0},
|
||||
// [144] = {sys_sysconf, 0},
|
||||
};
|
||||
|
||||
[[gnu::no_caller_saved_registers]]
|
||||
struct syscall_ret syscall_dispatch_fast(
|
||||
uintptr_t a0,
|
||||
uintptr_t a1,
|
||||
uintptr_t a2,
|
||||
uintptr_t a3,
|
||||
uintptr_t a4,
|
||||
uintptr_t a5,
|
||||
size_t num
|
||||
) {
|
||||
// No need to check the table size here. In order to get to this point,
|
||||
// we already have to check if this is a fast syscall.
|
||||
if (syscall_table[num].func == nullptr) {
|
||||
return (struct syscall_ret){.errno = ENOSYS};
|
||||
}
|
||||
|
||||
struct syscall_ret (*func)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t) =
|
||||
syscall_table[num].func;
|
||||
|
||||
return func(a0, a1, a2, a3, a4, a5);
|
||||
}
|
||||
|
||||
void syscall_dispatch(struct arch_context* ctx) {
|
||||
const size_t num = ctx->ARCH_CTX_NUM;
|
||||
|
||||
if (num >= ARRAY_SIZE(syscall_table) || syscall_table[num].func == nullptr) {
|
||||
ctx->ARCH_CTX_RET0 = ENOSYS;
|
||||
return;
|
||||
}
|
||||
|
||||
void (*func)(struct arch_context* ctx) = syscall_table[num].func;
|
||||
func(ctx);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#include <zinnia/errno.h>
|
||||
#include <zinnia/futex.h>
|
||||
#include <zinnia/syscall.h>
|
||||
|
||||
errno_t syscall_futex_wait(struct arch_context* ctx) {
|
||||
__user int* addr = (__user int*)ctx->ARCH_CTX_A0;
|
||||
int expected = (int)ctx->ARCH_CTX_A1;
|
||||
|
||||
return futex_wait(addr, expected);
|
||||
}
|
||||
|
||||
errno_t syscall_futex_wake(struct arch_context* ctx) {
|
||||
__user int* addr = (__user int*)ctx->ARCH_CTX_A0;
|
||||
int count = (int)ctx->ARCH_CTX_A1;
|
||||
|
||||
return futex_wake(addr, count);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#include <zinnia/spin.h>
|
||||
#include <zinnia/syscall.h>
|
||||
#include <zinnia/usercopy.h>
|
||||
#include <config.h>
|
||||
#include <uapi/utsname.h>
|
||||
|
||||
static spinlock_t uname_lock = {};
|
||||
|
||||
static struct utsname uname_buf = {
|
||||
.sysname = "Zinnia",
|
||||
.release = ZINNIA_VERSION,
|
||||
.version = CONFIG_KERNEL_FLAVOR,
|
||||
.nodename = "localhost",
|
||||
.machine = ZINNIA_ARCH,
|
||||
};
|
||||
|
||||
struct syscall_ret sys_getuname(__user struct utsname* ptr) {
|
||||
struct syscall_ret ret = {};
|
||||
|
||||
if (!ptr) {
|
||||
ret.errno = EINVAL;
|
||||
return ret;
|
||||
}
|
||||
|
||||
spin_lock(&uname_lock);
|
||||
|
||||
if (!usercopy_write(ptr, &uname_buf, sizeof(uname_buf)))
|
||||
ret.errno = EFAULT;
|
||||
|
||||
spin_unlock(&uname_lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
kernel_sources += files(
|
||||
'archctl.c',
|
||||
'dispatch.c',
|
||||
'futex.c',
|
||||
'getuname.c',
|
||||
)
|
||||
Reference in New Issue
Block a user