diff --git a/boot/main.c b/boot/main.c index e309a20..8965175 100644 --- a/boot/main.c +++ b/boot/main.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: ISC + #include #include #include @@ -5,15 +7,17 @@ #include #include #include -#include #include #include +#include #include #include #include #include +#include #include #include +#include #include #include #include @@ -22,52 +26,41 @@ static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")"; +// Path to the init executable. +static const char* init_path = "/init"; +static void init_option(const char* value) { + init_path = value; +} +CMDLINE_OPTION("init", "Path to the init executable", init_option); + static void kernel_main_task(uintptr_t arg0) { errno_t status = 0; struct boot_info* info = (struct boot_info*)arg0; - log("main: Hello from main task!\n"); + log("Hello from main task!\n"); // Finish the initgraph. initgraph_run(nullptr, info); + // After everything is loaded, load all builtin drivers. module_init(); - // TODO: Try to run init. + // Load all initramfs files. + initramfs_load_boot_modules(info); - assert(info->num_files >= 1, "No init executable provided"); + const char* init_argv[] = {init_path, nullptr}; + const char* init_envp[] = {"HOME=/", "TERM=console", nullptr}; - struct vma* init_space; - assert(vma_new(&init_space) == 0, ""); + log("Starting init \"%s\"\n", init_path); + log("With arguments:\n"); + for (const char** arg = init_argv; *arg; arg++) + log(" %s\n", *arg); + log("With environment:\n"); + for (const char** env = init_envp; *env; env++) + log(" %s\n", *env); - struct boot_file* file_data = &info->files[0]; - struct task* init_task = nullptr; - - struct vmo* init_file; - status = vmo_new_anon(&init_file); - assert(status == 0, "Failed to allocate a memory object"); - - vmo_write(init_file, 0, HHDM_PTR(file_data->data), file_data->length, nullptr); - - // Load init. - const char* init_argv[] = {"init", nullptr}; - const char* init_envp[] = {nullptr}; - - struct exec_info exec = { - .space = init_space, - .file_obj = init_file, - .argv = init_argv, - .envp = init_envp, - }; - - vmo_unref(init_file); - - assert(status == 0, "Failed to load init process (%i)", status); - sched_add_task(&percpu_get()->sched, init_task); - - while (true) { - irq_wait(); - } + status = process_spawn_init(init_path, init_argv, init_envp); + assert(status == 0, "Failed to start init \"%s\" (%i)", init_path, status); } [[__init]] @@ -87,7 +80,7 @@ void kernel_main(struct boot_info* info) { initgraph_run(sched_stage, info); struct task* main_task; - task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task); + task_create("main", process_kernel(), kernel_main_task, (uintptr_t)info, &main_task); sched_add_task(&percpu_get()->sched, main_task); irq_unlock(); diff --git a/fs/exec/elf.c b/fs/exec/elf.c index 34692d4..9ffe5c6 100644 --- a/fs/exec/elf.c +++ b/fs/exec/elf.c @@ -1,8 +1,11 @@ +// SPDX-License-Identifier: ISC + #include #include #include -#include #include +#include +#include #include #include #include @@ -14,13 +17,59 @@ #include #include #include +#include -errno_t elf_load(struct exec_info* info, struct task** out) { +constexpr uintptr_t ELF_DYN_BASE = 0x10000; + +struct elf_image { + uintptr_t base; + uintptr_t entry; + uintptr_t phdr; + size_t phnum; + size_t phent; +}; + +static bool elf_identify(struct vmo* obj) { + elf_ehdr_t ehdr = {}; + size_t read; + if (vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read) || read != sizeof(ehdr)) + return false; + + return memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) == 0; +} + +static errno_t elf_read_interp(struct vmo* obj, const elf_phdr_t* phdr, char** out) { + if (phdr->p_filesz < 2 || phdr->p_filesz > __PATH_MAX) + return ENOEXEC; + + char* path = mem_alloc(phdr->p_filesz, ALLOC_NOZERO); + if (!path) + return ENOMEM; + + size_t read; + errno_t err = vmo_read(obj, phdr->p_offset, path, phdr->p_filesz, &read); + if (!err && (read != phdr->p_filesz || path[phdr->p_filesz - 1] != '\0')) + err = ENOEXEC; + if (err) { + mem_free(path); + return err; + } + *out = path; + return 0; +} + +static errno_t elf_map( + struct vma* space, + struct vmo* obj, + uintptr_t dyn_base, + struct elf_image* out, + char** out_interp +) { errno_t err; size_t read; elf_ehdr_t ehdr = {}; - err = vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read); + err = vmo_read(obj, 0, &ehdr, sizeof(ehdr), &read); if (err) return err; if (read != sizeof(ehdr)) @@ -37,9 +86,12 @@ errno_t elf_load(struct exec_info* info, struct task** out) { return EINVAL; if (ehdr.e_machine != ELF_ARCH_MACHINE) return EINVAL; - if (ehdr.e_type != ET_EXEC) + if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) return EINVAL; - if (!arch_mem_addr_is_user(ehdr.e_entry)) + + const uintptr_t base = ehdr.e_type == ET_DYN ? dyn_base : 0; + uintptr_t entry; + if (ckd_add(&entry, ehdr.e_entry, base) || !arch_mem_addr_is_user(entry)) return ENOEXEC; const size_t phdrs_size = (size_t)ehdr.e_phnum * ehdr.e_phentsize; @@ -54,14 +106,22 @@ errno_t elf_load(struct exec_info* info, struct task** out) { for (size_t i = 0; i < ehdr.e_phnum; i++) { elf_phdr_t phdr = {}; - err = vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read); + err = vmo_read(obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read); if (err) return err; if (read != sizeof(phdr)) return ENOEXEC; if (phdr.p_type == PT_PHDR) - phdr_addr = phdr.p_vaddr; + phdr_addr = phdr.p_vaddr + base; + + if (phdr.p_type == PT_INTERP && out_interp) { + if (*out_interp) + return ENOEXEC; + err = elf_read_interp(obj, &phdr, out_interp); + if (err) + return err; + } if (phdr.p_type == PT_LOAD) { if (phdr.p_memsz == 0) @@ -69,17 +129,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if (phdr.p_filesz > phdr.p_memsz) return ENOEXEC; - uintptr_t seg_end; - if (ckd_add(&seg_end, phdr.p_vaddr, phdr.p_memsz)) + uintptr_t vaddr, seg_end; + if (ckd_add(&vaddr, phdr.p_vaddr, base) || ckd_add(&seg_end, vaddr, phdr.p_memsz)) return ENOEXEC; - if (!arch_mem_addr_is_user(phdr.p_vaddr) || !arch_mem_addr_is_user(seg_end - 1)) + if (!arch_mem_addr_is_user(vaddr) || !arch_mem_addr_is_user(seg_end - 1)) return ENOEXEC; if (phdr.p_align > 1 && phdr.p_offset % phdr.p_align != phdr.p_vaddr % phdr.p_align) return ENOEXEC; if (phdr_addr == 0 && ehdr.e_phoff >= phdr.p_offset && phdrs_end - phdr.p_offset <= phdr.p_filesz) - phdr_addr = phdr.p_vaddr + (ehdr.e_phoff - phdr.p_offset); + phdr_addr = vaddr + (ehdr.e_phoff - phdr.p_offset); enum prot_flags prot = 0; if (phdr.p_flags & PF_R) @@ -89,44 +149,102 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if (phdr.p_flags & PF_X) prot |= PROT_EXEC; - const uintptr_t misalign = phdr.p_vaddr & (page_size - 1); - const uintptr_t map_address = phdr.p_vaddr - misalign; + const uintptr_t misalign = vaddr & (page_size - 1); + const uintptr_t map_address = vaddr - misalign; const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1); const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1); - // Copy the file data into its own mapping. - struct vmo* phdr_obj; - err = vmo_new_anon(&phdr_obj); - if (err) - return err; - - err = vmo_copy(phdr_obj, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr); - if (err) { - vmo_unref(phdr_obj); - return err; + if (backed_map_size) { + err = vma_map(space, obj, map_address, backed_map_size, prot, 0, phdr.p_offset - misalign); + if (err) + return err; + } + if (total_map_size > backed_map_size) { + err = vma_map( + space, + nullptr, + map_address + backed_map_size, + total_map_size - backed_map_size, + prot, + 0, + 0 + ); + if (err) + return err; } - // We map more than we copied so the rest is filled with zeroed pages. - err = vma_map(info->space, phdr_obj, phdr.p_vaddr, phdr.p_memsz, prot, 0, phdr.p_offset); - vmo_unref(phdr_obj); - if (err) - return err; + const uintptr_t file_end = vaddr + phdr.p_filesz; + if (phdr.p_memsz > phdr.p_filesz && file_end % page_size != 0) { + const uintptr_t zero_end = MIN(ALIGN_UP(file_end, page_size), seg_end); + err = vma_zero(space, file_end, zero_end - file_end); + if (err) + return err; + } } } + *out = (struct elf_image){ + .base = base, + .entry = entry, + .phdr = phdr_addr, + .phnum = ehdr.e_phnum, + .phent = ehdr.e_phentsize, + }; + return 0; +} + +static errno_t elf_load(struct process* proc, struct exec_info* info, uintptr_t* out_ip, uintptr_t* out_sp) { + struct elf_image exe = {}; + char* interp_path = nullptr; + errno_t err = elf_map(info->space, info->file_obj, ELF_DYN_BASE, &exe, &interp_path); + + uintptr_t entry = exe.entry; + uintptr_t interp_base = 0; + if (!err && interp_path) { + struct vmo* interp; + err = exec_open(proc, interp_path, &interp); + if (!err) { + struct elf_image ld; + if (!elf_identify(interp)) + err = ELIBBAD; + else + err = elf_map(info->space, interp, (uintptr_t)1 << (mem_high_shift() - 2), &ld, nullptr); + if (!err) { + entry = ld.entry; + interp_base = ld.base; + } + vmo_unref(interp); + } + } + mem_free(interp_path); + if (err) + return err; + + const size_t page_size = arch_mem_page_size(); + const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size; const size_t stack_size = 2 * 1024 * 1024; // 2MiB const uintptr_t stack_start = highest - stack_size; - // Allocate stack. - struct vmo* stack; - err = vmo_new_anon(&stack); + err = vma_map(info->space, nullptr, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0); if (err) return err; // Fill in stack info. uintptr_t stack_off = stack_size; + uintptr_t execfn = 0; + if (info->exec_path) { + const char nul = 0; + stack_off -= 1; + vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul)); + + const size_t len = strlen(info->exec_path); + stack_off -= len; + vma_write(info->space, stack_start + stack_off, info->exec_path, len); + execfn = stack_start + stack_off; + } + // TODO: Instead of allocating, calculate it on the fly. size_t num_envp = 0; for (const char** ptr = info->envp; *ptr != nullptr; ptr++) { @@ -148,18 +266,17 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if ((num_envp && !envp_offsets) || (num_argv && !argv_offsets)) { mem_free(envp_offsets); mem_free(argv_offsets); - vmo_unref(stack); return ENOMEM; } for (size_t env = 0; env < num_envp; env++) { const char nul = 0; stack_off -= 1; - vmo_write(stack, stack_off, &nul, sizeof(nul), nullptr); + vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul)); const size_t len = strlen(info->envp[env]); stack_off -= len; - vmo_write(stack, stack_off, info->envp[env], len, nullptr); + vma_write(info->space, stack_start + stack_off, info->envp[env], len); envp_offsets[env] = stack_start + stack_off; } @@ -167,11 +284,11 @@ errno_t elf_load(struct exec_info* info, struct task** out) { for (size_t arg = 0; arg < num_argv; arg++) { const char nul = 0; stack_off -= 1; - vmo_write(stack, stack_off, &nul, sizeof(nul), nullptr); + vma_write(info->space, stack_start + stack_off, &nul, sizeof(nul)); const size_t len = strlen(info->argv[arg]); stack_off -= len; - vmo_write(stack, stack_off, info->argv[arg], len, nullptr); + vma_write(info->space, stack_start + stack_off, info->argv[arg], len); argv_offsets[arg] = stack_start + stack_off; } @@ -182,43 +299,48 @@ errno_t elf_load(struct exec_info* info, struct task** out) { if ((1 + num_argv + num_envp) & 1) { stack_off -= sizeof(uintptr_t); uintptr_t zero = 0; - vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr); + vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero)); } #define WRITE_AUXV(auxv, value) \ do { \ stack_off -= sizeof(uintptr_t); \ uintptr_t auxv_val = value; \ - vmo_write(stack, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ + vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \ stack_off -= sizeof(uintptr_t); \ auxv_val = auxv; \ - vmo_write(stack, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \ + vma_write(info->space, stack_start + stack_off, &auxv_val, sizeof(uintptr_t)); \ } while (0) // Write auxiliary values. WRITE_AUXV(AT_NULL, 0); WRITE_AUXV(AT_SECURE, 0); - WRITE_AUXV(AT_PHNUM, ehdr.e_phnum); - WRITE_AUXV(AT_PHENT, ehdr.e_phentsize); - WRITE_AUXV(AT_ENTRY, ehdr.e_entry); - if (phdr_addr) - WRITE_AUXV(AT_PHDR, phdr_addr); + WRITE_AUXV(AT_PAGESZ, page_size); + WRITE_AUXV(AT_PHNUM, exe.phnum); + WRITE_AUXV(AT_PHENT, exe.phent); + WRITE_AUXV(AT_ENTRY, exe.entry); + if (exe.phdr) + WRITE_AUXV(AT_PHDR, exe.phdr); + if (interp_base) + WRITE_AUXV(AT_BASE, interp_base); + if (execfn) + WRITE_AUXV(AT_EXECFN, execfn); // envp pointers. stack_off -= sizeof(uintptr_t); const uintptr_t zero = 0; - vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr); + vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero)); for (size_t env = num_envp; env > 0; env--) { stack_off -= sizeof(uintptr_t); - vmo_write(stack, stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1]), nullptr); + vma_write(info->space, stack_start + stack_off, &envp_offsets[env - 1], sizeof(envp_offsets[env - 1])); } // argv pointers. stack_off -= sizeof(uintptr_t); - vmo_write(stack, stack_off, &zero, sizeof(zero), nullptr); + vma_write(info->space, stack_start + stack_off, &zero, sizeof(zero)); for (size_t arg = num_argv; arg > 0; arg--) { stack_off -= sizeof(uintptr_t); - vmo_write(stack, stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1]), nullptr); + vma_write(info->space, stack_start + stack_off, &argv_offsets[arg - 1], sizeof(argv_offsets[arg - 1])); } mem_free(envp_offsets); @@ -227,27 +349,22 @@ errno_t elf_load(struct exec_info* info, struct task** out) { // argc stack_off -= sizeof(uintptr_t); uintptr_t argc = num_argv; - vmo_write(stack, stack_off, &argc, sizeof(argc), nullptr); + vma_write(info->space, stack_start + stack_off, &argc, sizeof(argc)); - err = vma_map(info->space, stack, stack_start, stack_size, PROT_READ | PROT_WRITE, 0, 0); - vmo_unref(stack); - if (err) - return err; - - struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0); - if (!start_info) - return ENOMEM; - start_info->ip = ehdr.e_entry; - start_info->sp = highest - stack_size + stack_off; - start_info->arg = 0; - - struct task* result; - err = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result); - if (err) { - mem_free(start_info); - return err; - } - - *out = result; + *out_ip = entry; + *out_sp = highest - stack_size + stack_off; return 0; } + +static struct exec_format elf_format = { + .name = "elf", + .identify = elf_identify, + .load = elf_load, +}; + +static void elf_stage_init(struct boot_info*) { + exec_register(&elf_format); +} +INIT_TASK(elf_stage, "fs.exec.elf", elf_stage_init); +INIT_DEPENDS(elf_stage, mem_stage); +INIT_ENTAILS(elf_stage, vfs_stage); diff --git a/fs/exec/exec.c b/fs/exec/exec.c new file mode 100644 index 0000000..813b5be --- /dev/null +++ b/fs/exec/exec.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include +#include +#include + +static mutex_t formats_lock; +static tailq_head(struct exec_format) formats = {nullptr, &formats.tqh_first}; + +struct exec_format* exec_identify(struct vmo* obj) { + mutex_lock(&formats_lock); + struct exec_format* format = tailq_first(&formats); + while (format != nullptr && !format->identify(obj)) + format = tailq_next(format, link); + mutex_unlock(&formats_lock); + return format; +} + +void exec_register(struct exec_format* format) { + mutex_lock(&formats_lock); + tailq_insert_tail(&formats, format, link); + mutex_unlock(&formats_lock); + + log("exec: Registered new exec format \"%s\"\n", format->name); +} + +errno_t exec_open(struct process* proc, const char* path, struct vmo** out) { + mutex_lock(&proc->lock); + struct vfs_path root = vfs_path_clone(proc->root); + struct vfs_path cwd = vfs_path_clone(proc->cwd); + mutex_unlock(&proc->lock); + + struct vfs_file* file; + errno_t status = vfs_file_open(root, cwd, path, O_RDONLY | O_EXEC, 0, &proc->identity, &file); + vfs_path_drop(root); + vfs_path_drop(cwd); + if (status) + return status; + + status = exec_file_object(proc, file, out); + vfs_file_unref(file); + return status; +} + +errno_t exec_file_object(struct process* proc, struct vfs_file* file, struct vmo** out) { + errno_t status = vfs_node_check_exec(file->vnode, &proc->identity); + if (!status) + status = vfs_file_get_object(file, out); + return status; +} diff --git a/fs/exec/meson.build b/fs/exec/meson.build index a389904..cc1ef98 100644 --- a/fs/exec/meson.build +++ b/fs/exec/meson.build @@ -1,3 +1,4 @@ kernel_sources += files( 'elf.c', + 'exec.c', ) diff --git a/include/zinnia/process.h b/include/zinnia/process.h new file mode 100644 index 0000000..d8dcb9d --- /dev/null +++ b/include/zinnia/process.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t PROCESS_DEFAULT_FDS = 1024; +constexpr size_t PROCESS_MAX_FDS = 1 << 20; + +struct task; +struct vma; + +struct process_rlimit { + _Atomic(rlim_t) cur; + rlim_t max; +}; + +struct fd_entry { + struct vfs_file* file; + bool cloexec; +}; + +struct process { + struct refcount rc; + pid_t pid; + + struct spinlock tasks_lock; + tailq_head(struct task) tasks; + size_t thread_count; + + mutex_t lock; + struct vma* space; + struct vfs_path root; + struct vfs_path cwd; + struct ident identity; + mode_t umask; + int8_t nice; + + struct process_rlimit limit_nofile; + + mutex_t files_lock; + hashmap(int, struct fd_entry) files; +}; + +struct process* process_kernel(); +void process_ref(struct process* proc); +void process_unref(struct process* proc); + +uint8_t process_priority(struct process* proc); +void process_attach_task(struct process* proc, struct task* task); +void process_detach_task(struct process* proc, struct task* task); + +errno_t process_spawn_init(const char* path, const char** argv, const char** envp); + +size_t process_fd_limit(struct process* proc); + +// Allocate a file drescriptor for `file` on `proc`. +errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloexec, int* out_fd); + +// Allocate a file descriptor for `file` on `proc`, starting at at least `min_fd`. +errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int min_fd, bool cloexec, int* out_fd); + +// Replaces the current file indentified by `fd` in `proc` with a new `file`. +errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, bool cloexec); +errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec); +errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec); +struct vfs_file* process_fd_get(struct process* proc, int fd); +errno_t process_fd_close(struct process* proc, int fd); diff --git a/include/zinnia/sched.h b/include/zinnia/sched.h index e1fd62f..f4bf633 100644 --- a/include/zinnia/sched.h +++ b/include/zinnia/sched.h @@ -40,6 +40,7 @@ constexpr size_t SCHED_TIME_SLICE_NS = 10'000'000; constexpr uint8_t SCHED_FEEDBACK_DEPTH = 8; constexpr uint64_t SCHED_BOOST_INTERVAL_NS = 1'000'000'000; +struct process; struct turnstile; constexpr size_t TASK_NAME_MAX = 128; @@ -49,6 +50,8 @@ struct task { size_t id; char name[TASK_NAME_MAX]; struct vma* space; + struct process* process; + tailq_link(struct task) process_link; enum task_state state; struct arch_task_context context; struct usercopy_region* usercopy_region; @@ -105,7 +108,7 @@ struct scheduler { typedef void (*task_fn_t)(uintptr_t arg0); -errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out); +errno_t task_create(const char* name, struct process* proc, task_fn_t entry, uintptr_t arg0, struct task** out); void task_destroy(struct task* task); void task_lock(struct task* task); void task_unlock(struct task* task); @@ -127,6 +130,8 @@ void sched_wake(struct task* task); // Same as sched_wake(), but the task must be locked with task_lock(). void sched_wake_locked(struct task* task); +void sched_notify(struct task* task); + [[noreturn]] void sched_exit(); diff --git a/lib/meson.build b/lib/meson.build index 3b4893b..b56e508 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -14,6 +14,7 @@ kernel_sources += files( 'park.c', 'percpu.c', 'print.c', + 'process.c', 'radix.c', 'random.c', 'ringbuf.c', diff --git a/lib/process.c b/lib/process.c new file mode 100644 index 0000000..99649ef --- /dev/null +++ b/lib/process.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct process kernel_process; +static atomic_bool kernel_process_ready; +static atomic_int next_pid = 1; + +static void process_init_common(struct process* proc) { + refcount_init(&proc->rc); + spin_init(&proc->tasks_lock); + tailq_init(&proc->tasks); + mutex_init(&proc->lock); + mutex_init(&proc->files_lock); + hashmap_init(&proc->files); + vec_init(&proc->identity.groups); +} + +struct process* process_kernel() { + if (!atomic_load(&kernel_process_ready, ATOMIC_ACQUIRE)) { + process_init_common(&kernel_process); + kernel_process.pid = 0; + kernel_process.space = &kernel_vma; + atomic_store(&kernel_process_ready, true, ATOMIC_RELEASE); + } + return &kernel_process; +} + +static void close_files(struct process* proc) { + mutex_lock(&proc->files_lock); + size_t i = 0; + while (i < proc->files.capacity) { + struct fd_entry entry = proc->files.values[i]; + if (proc->files.hashes[i] == 0) { + i++; + continue; + } + hashmap_remove(&proc->files, proc->files.keys[i], hashmap_hash_default, hashmap_eq_default); + vfs_file_unref(entry.file); + } + mutex_unlock(&proc->files_lock); +} + +void process_ref(struct process* proc) { + refcount_inc(&proc->rc); +} + +void process_unref(struct process* proc) { + if (proc == &kernel_process || !refcount_dec(&proc->rc)) + return; + close_files(proc); + vfs_path_drop(proc->root); + vfs_path_drop(proc->cwd); + if (proc->space) + vma_unref(proc->space); + vec_destroy(&proc->identity.groups); + hashmap_destroy(&proc->files); + mem_free(proc); +} + +uint8_t process_priority(struct process* proc) { + if (proc == &kernel_process) + return SCHED_PRIO_DEFAULT; + + // Normalize the priority from "nice" ranges to our priorities. + return 16 + (19 - proc->nice) * 15 / 39; +} + +void process_attach_task(struct process* proc, struct task* task) { + spin_lock(&proc->tasks_lock); + tailq_insert_tail(&proc->tasks, task, process_link); + proc->thread_count++; + spin_unlock(&proc->tasks_lock); +} + +void process_detach_task(struct process* proc, struct task* task) { + spin_lock(&proc->tasks_lock); + tailq_remove(&proc->tasks, task, process_link); + spin_unlock(&proc->tasks_lock); +} + +static void rlimit_init(struct process_rlimit* limit, rlim_t cur, rlim_t max) { + atomic_init(&limit->cur, cur); + limit->max = max; +} + +static errno_t process_alloc(struct process** out) { + struct process* proc = mem_alloc(sizeof(struct process), 0); + if (!proc) + return ENOMEM; + process_init_common(proc); + proc->pid = atomic_fetch_add(&next_pid, 1, ATOMIC_RELAXED); + proc->root = vfs_get_root(); + proc->cwd = vfs_get_root(); + proc->umask = 022; + rlimit_init(&proc->limit_nofile, PROCESS_DEFAULT_FDS, PROCESS_MAX_FDS); + + *out = proc; + return 0; +} + +static errno_t load_image( + struct process* proc, + struct vmo* object, + const char* path, + const char** argv, + const char** envp, + struct vma** out_space, + uintptr_t* out_ip, + uintptr_t* out_sp +) { + struct vma* space; + errno_t status = vma_new(&space); + if (status) + return status; + + struct exec_info info = { + .space = space, + .file_obj = object, + .exec_path = path, + .argv = argv, + .envp = envp, + }; + struct exec_format* format = exec_identify(object); + status = format ? format->load(proc, &info, out_ip, out_sp) : ENOEXEC; + if (status) { + vma_unref(space); + return status; + } + + *out_space = space; + return 0; +} + +errno_t process_spawn_init(const char* path, const char** argv, const char** envp) { + struct process* proc; + errno_t status = process_alloc(&proc); + if (status) + return status; + + struct task_start_info* start = mem_alloc(sizeof(struct task_start_info), 0); + if (!start) { + process_unref(proc); + return ENOMEM; + } + + struct vmo* object; + status = exec_open(proc, path, &object); + if (!status) { + status = load_image(proc, object, path, argv, envp, &proc->space, &start->ip, &start->sp); + vmo_unref(object); + } + if (status) { + mem_free(start); + process_unref(proc); + return status; + } + + struct vfs_file* console; + status = vfs_file_open(proc->root, proc->cwd, "/dev/console", O_RDWR, 0, ident_kernel(), &console); + if (status) { + log_warn("process: Unable to open /dev/console for init (%i)\n", status); + } else { + for (int fd = 0; fd <= 2; fd++) + process_fd_replace(proc, fd, console, false); + vfs_file_unref(console); + } + + struct task* task; + status = task_create(argv[0], proc, sched_to_user, (uintptr_t)start, &task); + process_unref(proc); + if (status) { + mem_free(start); + return status; + } + + sched_add_task(&percpu_get()->sched, task); + return 0; +} + +errno_t process_fd_alloc(struct process* proc, struct vfs_file* file, bool cloexec, int* out_fd) { + return process_fd_alloc_min(proc, file, 0, cloexec, out_fd); +} + +static struct fd_entry* fd_lookup(struct process* proc, int fd) { + return hashmap_get(&proc->files, fd, hashmap_hash_default, hashmap_eq_default); +} + +size_t process_fd_limit(struct process* proc) { + return MIN((size_t)PROCESS_MAX_FDS, (size_t)atomic_load(&proc->limit_nofile.cur, ATOMIC_RELAXED)); +} + +errno_t process_fd_alloc_min(struct process* proc, struct vfs_file* file, int min_fd, bool cloexec, int* out_fd) { + if (min_fd < 0 || (size_t)min_fd >= process_fd_limit(proc)) + return EINVAL; + + mutex_lock(&proc->files_lock); + int fd = min_fd; + while ((size_t)fd < process_fd_limit(proc) && fd_lookup(proc, fd)) + fd++; + + if ((size_t)fd >= process_fd_limit(proc)) { + mutex_unlock(&proc->files_lock); + return EMFILE; + } + + const struct fd_entry entry = {.file = file, .cloexec = cloexec}; + errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default); + if (status) { + mutex_unlock(&proc->files_lock); + return status; + } + vfs_file_ref(file); + mutex_unlock(&proc->files_lock); + + *out_fd = fd; + return 0; +} + +errno_t process_fd_replace(struct process* proc, int fd, struct vfs_file* file, bool cloexec) { + if (fd < 0 || (size_t)fd >= process_fd_limit(proc)) + return EBADF; + + struct vfs_file* displaced = nullptr; + const struct fd_entry entry = {.file = file, .cloexec = cloexec}; + mutex_lock(&proc->files_lock); + struct fd_entry* slot = fd_lookup(proc, fd); + if (slot) { + displaced = slot->file; + *slot = entry; + } else { + errno_t status = hashmap_insert(&proc->files, fd, entry, hashmap_hash_default, hashmap_eq_default); + if (status) { + mutex_unlock(&proc->files_lock); + return status; + } + } + vfs_file_ref(file); + mutex_unlock(&proc->files_lock); + + if (displaced) + vfs_file_unref(displaced); + return 0; +} + +errno_t process_fd_get_cloexec(struct process* proc, int fd, bool* out_cloexec) { + errno_t status = EBADF; + mutex_lock(&proc->files_lock); + struct fd_entry* entry = fd_lookup(proc, fd); + if (entry) { + *out_cloexec = entry->cloexec; + status = 0; + } + mutex_unlock(&proc->files_lock); + return status; +} + +errno_t process_fd_set_cloexec(struct process* proc, int fd, bool cloexec) { + errno_t status = EBADF; + mutex_lock(&proc->files_lock); + struct fd_entry* entry = fd_lookup(proc, fd); + if (entry) { + entry->cloexec = cloexec; + status = 0; + } + mutex_unlock(&proc->files_lock); + return status; +} + +struct vfs_file* process_fd_get(struct process* proc, int fd) { + struct vfs_file* file = nullptr; + mutex_lock(&proc->files_lock); + struct fd_entry* entry = fd_lookup(proc, fd); + if (entry) { + file = entry->file; + vfs_file_ref(file); + } + mutex_unlock(&proc->files_lock); + return file; +} + +errno_t process_fd_close(struct process* proc, int fd) { + struct vfs_file* file = nullptr; + mutex_lock(&proc->files_lock); + struct fd_entry* entry = fd_lookup(proc, fd); + if (entry) { + file = entry->file; + hashmap_remove(&proc->files, fd, hashmap_hash_default, hashmap_eq_default); + } + mutex_unlock(&proc->files_lock); + + if (!file) + return EBADF; + vfs_file_unref(file); + return 0; +} diff --git a/sched/core.c b/sched/core.c index 280e82c..2f13a10 100644 --- a/sched/core.c +++ b/sched/core.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,9 @@ void sched_to_user(uintptr_t info_ptr) { } void sched_to_user_context(uintptr_t ctx) { - arch_sched_jump_to_context((struct arch_context*)ctx); + struct arch_context context = *(struct arch_context*)ctx; + mem_free((void*)ctx); + arch_sched_jump_to_context(&context); } static struct spinlock task_cache_lock; @@ -54,20 +57,25 @@ static void task_release(struct task* task) { spin_unlock(&task_cache_lock); } -static void task_init_common(struct task* task, const char* name, struct vma* space) { - task->space = space; +static void task_init_common(struct task* task, const char* name, struct process* proc) { + task->process = proc; + task->space = proc->space; + process_ref(proc); + if (task->space != &kernel_vma) + vma_ref(task->space); spin_init(&task->lock); atomic_store(&task->lockp, &task->lock, ATOMIC_RELAXED); strncpy(task->name, name, sizeof(task->name)); - task->base_prio = SCHED_PRIO_DEFAULT; + task->base_prio = process_priority(proc); task->time_slice = SCHED_TIME_SLICE_NS; task->penalty = 0; task->feedback_epoch = 0; - atomic_store(&task->priority, SCHED_PRIO_DEFAULT, ATOMIC_RELAXED); + atomic_store(&task->priority, task->base_prio, ATOMIC_RELAXED); tailq_init(&task->pi_lenders); + process_attach_task(proc, task); } -errno_t task_create_current(const char* name, struct vma* space, struct task** out) { +errno_t task_create_current(const char* name, struct process* proc, struct task** out) { struct task* task = task_alloc(); if (task == nullptr) return ENOMEM; @@ -78,14 +86,14 @@ errno_t task_create_current(const char* name, struct vma* space, struct task** o return ENOMEM; } - task_init_common(task, name, space); + task_init_common(task, name, proc); *out = task; return 0; } constexpr size_t KERNEL_STACK_SIZE = 16 * 1024; -errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) { +errno_t task_create(const char* name, struct process* proc, task_fn_t entry, uintptr_t arg0, struct task** out) { if (!out) return EINVAL; @@ -119,7 +127,7 @@ errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintpt return s; } - task_init_common(new_task, name, space); + task_init_common(new_task, name, proc); *out = new_task; return 0; } @@ -129,6 +137,10 @@ void task_destroy(struct task* task) { turnstile_free(task->turnstile); arch_task_fini(&task->context); stack_destroy(&task->kernel_stack); + process_detach_task(task->process, task); + if (task->space != &kernel_vma) + vma_unref(task->space); + process_unref(task->process); task_release(task); } diff --git a/sched/internal.h b/sched/internal.h index 29e9f81..d0935b4 100644 --- a/sched/internal.h +++ b/sched/internal.h @@ -4,6 +4,6 @@ #include -errno_t task_create_current(const char* name, struct vma* space, struct task** out); +errno_t task_create_current(const char* name, struct process* proc, struct task** out); bool sched_update_priority_locked(struct task* task); diff --git a/sched/sched_dumb.c b/sched/sched_dumb.c index e2e0cc1..4844175 100644 --- a/sched/sched_dumb.c +++ b/sched/sched_dumb.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -57,12 +58,12 @@ void sched_init(struct scheduler* sched) { tailq_init(&sched->run_queue[i]); struct task* idle_task; - assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!"); + assert(task_create("idle", process_kernel(), idle_func, 0, &idle_task) == 0, "Unable to create idle task!"); idle_task->last_rq = sched; sched->idle_task = idle_task; struct task* boot_task; - assert(task_create_current("boot", &kernel_vma, &boot_task) == 0, "Unable to create boot task!"); + assert(task_create_current("boot", process_kernel(), &boot_task) == 0, "Unable to create boot task!"); boot_task->state = TASK_STATE_RUNNING; boot_task->last_rq = sched; atomic_store(&boot_task->on_cpu, true, ATOMIC_RELAXED); @@ -326,6 +327,18 @@ void sched_wake(struct task* task) { task_unlock(task); } +void sched_notify(struct task* task) { + task_lock(task); + if (task->state == TASK_STATE_BLOCKED) { + enqueue(select_cpu(task), task); + } else { + atomic_store(&task->wake_pending, true, ATOMIC_RELEASE); + if (task->state == TASK_STATE_RUNNING && task->last_rq && task->last_rq != &percpu_get()->sched) + kick(task->last_rq); + } + task_unlock(task); +} + bool sched_update_priority_locked(struct task* task) { struct scheduler* sched = task->last_rq; if (sched == nullptr) {