From 9e065ee023f37b496f2f136e31cf96064cd4c54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Sat, 21 Mar 2026 22:46:54 -0400 Subject: [PATCH] initial commit --- .gitignore | 8 + Kconfig | 78 +++ build.zig | 195 +++++++ build.zig.zon | 24 + build/config.zig | 83 +++ build/image.zig | 111 ++++ build/ksyms.py | 66 +++ build/limine.conf | 11 + build/linker-scripts/aarch64.lds | 75 +++ build/linker-scripts/loongarch64.lds | 75 +++ build/linker-scripts/riscv64.lds | 75 +++ build/linker-scripts/x86_64.lds | 75 +++ build/menuconfig | 23 + build/run.zig | 69 +++ scripts/genconfig.py | 37 ++ src/kern/amd64/impl.zig | 0 src/kern/cpu.zig | 88 +++ src/kern/dpc.zig | 133 +++++ src/kern/init.zig | 54 ++ src/kern/ipl.zig | 96 ++++ src/kern/log.zig | 57 ++ src/kern/panic.zig | 19 + src/kern/private.zig | 31 ++ src/kern/root.zig | 68 +++ src/kern/sched.zig | 764 +++++++++++++++++++++++++++ src/kern/spinlock.zig | 60 +++ src/kern/thread.zig | 128 +++++ src/kern/timecounter.zig | 143 +++++ src/kern/timer.zig | 175 ++++++ src/kern/um/impl.zig | 106 ++++ src/kern/wait.zig | 235 ++++++++ src/limine.zig | 70 +++ src/main.zig | 57 ++ src/platform/amd64/entry.zig | 3 + src/platform/amd64/impl.zig | 16 + src/platform/root.zig | 75 +++ src/platform/um/entry.zig | 81 +++ src/platform/um/impl.zig | 200 +++++++ src/platform/um/timer.zig | 73 +++ src/root.zig | 38 ++ src/rtl/barrier.zig | 57 ++ src/rtl/list.zig | 204 +++++++ src/rtl/pairing_heap.zig | 331 ++++++++++++ src/rtl/root.zig | 11 + src/rtl/seqlock.zig | 52 ++ 45 files changed, 4430 insertions(+) create mode 100644 .gitignore create mode 100644 Kconfig create mode 100644 build.zig create mode 100644 build.zig.zon create mode 100644 build/config.zig create mode 100644 build/image.zig create mode 100755 build/ksyms.py create mode 100644 build/limine.conf create mode 100644 build/linker-scripts/aarch64.lds create mode 100644 build/linker-scripts/loongarch64.lds create mode 100644 build/linker-scripts/riscv64.lds create mode 100644 build/linker-scripts/x86_64.lds create mode 100755 build/menuconfig create mode 100644 build/run.zig create mode 100755 scripts/genconfig.py create mode 100644 src/kern/amd64/impl.zig create mode 100644 src/kern/cpu.zig create mode 100644 src/kern/dpc.zig create mode 100644 src/kern/init.zig create mode 100644 src/kern/ipl.zig create mode 100644 src/kern/log.zig create mode 100644 src/kern/panic.zig create mode 100644 src/kern/private.zig create mode 100644 src/kern/root.zig create mode 100644 src/kern/sched.zig create mode 100644 src/kern/spinlock.zig create mode 100644 src/kern/thread.zig create mode 100644 src/kern/timecounter.zig create mode 100644 src/kern/timer.zig create mode 100644 src/kern/um/impl.zig create mode 100644 src/kern/wait.zig create mode 100644 src/limine.zig create mode 100644 src/main.zig create mode 100644 src/platform/amd64/entry.zig create mode 100644 src/platform/amd64/impl.zig create mode 100644 src/platform/root.zig create mode 100644 src/platform/um/entry.zig create mode 100644 src/platform/um/impl.zig create mode 100644 src/platform/um/timer.zig create mode 100644 src/root.zig create mode 100644 src/rtl/barrier.zig create mode 100644 src/rtl/list.zig create mode 100644 src/rtl/pairing_heap.zig create mode 100644 src/rtl/root.zig create mode 100644 src/rtl/seqlock.zig diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..355c132 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.zig-cache/ +zig-out/ +deps/ +iso_root*/ +docs/ +*.a +zig-pkg/ +.config* diff --git a/Kconfig b/Kconfig new file mode 100644 index 0000000..cc042e0 --- /dev/null +++ b/Kconfig @@ -0,0 +1,78 @@ +mainmenu "Bach Configuration" + + +choice + prompt "CPU Architecture" + default ARCH_AMD64 + + config ARCH_AMD64 + bool "amd64" + help + Enable this option to select the AMD64 (x86_64) architecture. + + config ARCH_AARCH64 + bool "aarch64" + help + Enable this option to select the AARCH64 architecture. + + config ARCH_UM + bool "User-Mode" + help + Enable this option to select the User-Mode (UM) architecture. + +endchoice + + +choice + prompt "Bootloader support" + depends on (!ARCH_UM) + default BOOT_LIMINE + + config BOOT_LIMINE + depends on (ARCH_AMD64 || ARCH_AARCH64) + bool "Limine" + help + Enable this option to select support for the Limine bootloader. +endchoice + + +menu "Kernel Options" + +choice + prompt "Scheduling Algorithm" + default SCHED_RR + + config SCHED_ULE + bool "ULE" + help + Enable this option to select the ULE-like scheduler. + + config SCHED_RR + bool "Round-Robin" + help + Enable this option to select the Round-Robin scheduler. + +endchoice + + +config SCHED_TIMESLICE + int "Scheduler time slice" + default 10 + help + This option sets the default time slice (in milliseconds) used by the scheduler. + +config BALANCE_INTERVAL + int "Load balancing interval" + depends on (SCHED_ULE) + default 1000 + help + This option sets the rate (in milliseconds) at which thread load balancing is perfomed. + + +config NCPUS + int "Maximal number of CPUs" + default 1024 + help + This option sets the maximum number of CPUs the kernel can support. + +endmenu diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..c5015fa --- /dev/null +++ b/build.zig @@ -0,0 +1,195 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const config = @import("build/config.zig"); +const image = @import("build/image.zig"); +const run = @import("build/run.zig"); + +pub fn build(b: *std.Build) void { + const config_step = b.step("config", "Show the config menu"); + config_step.dependOn(config.configStep(b)); + + // Parse the config + const cfg = config.parseConfig(b) catch |err| { + if (err == error.FileNotFound) { + b.default_step.dependOn( + &b.addFail("No .config found, run `zig build config` first").step, + ); + } else { + b.default_step.dependOn( + &b.addFail("Failed to parse .config").step, + ); + } + return; + }; + + // Generate the zig config file we can import in the code + const config_wf = config.generateConfig(b, cfg) catch { + b.default_step.dependOn( + &b.addFail("Failed to generate config.gen.zig").step, + ); + return; + }; + + // Get the target platform (might be linux or freestanding) + const plat = config.getPlatform(cfg) catch { + b.default_step.dependOn( + &b.addFail("Invalid platform (this should not happen!)").step, + ); + return; + }; + + const optimize = b.standardOptimizeOption(.{ .preferred_optimize_mode = .Debug }); + + const config_module = b.createModule(.{ + .root_source_file = config_wf.getDirectory().path(b, "config.gen.zig"), + }); + + // first pass + const empty_cmd = b.addSystemCommand(&.{ "python3", "build/ksyms.py", "--empty" }); + empty_cmd.addArg("--output"); + const empty_zig = empty_cmd.addOutputFileArg("ksyms_empty.gen.zig"); + const empty_ksyms_module = b.createModule(.{ + .root_source_file = empty_zig, + }); + + const rtl_module = b.createModule(.{ + .root_source_file = b.path("src/rtl/root.zig"), + }); + + rtl_module.addImport("rtl", rtl_module); + + const kernel_nosym = addKernel(b, plat, optimize, config_module, empty_ksyms_module, rtl_module); + + b.step("kernel-nosym", "Build nosym kernel").dependOn(&kernel_nosym.step); + + // generate symbols + const ksyms = b.addSystemCommand(&.{ "python3", "build/ksyms.py" }); + ksyms.addArg("--input"); + ksyms.step.dependOn(&kernel_nosym.step); + ksyms.addFileArg(kernel_nosym.getEmittedBin()); + ksyms.addArg("--output"); + const ksyms_zig = ksyms.addOutputFileArg("ksyms.gen.zig"); + const ksyms_module = b.createModule(.{ + .root_source_file = ksyms_zig, + }); + + // second pass + const kernel = addKernel(b, plat, optimize, config_module, ksyms_module, rtl_module); + + b.default_step.dependOn(&b.addInstallArtifact(kernel, .{}).step); + + const iso_step = b.step("iso", "Build the ISO image"); + const result = image.addIso(b, plat.arch, "myos", kernel, b.dependency("limine", .{})); + iso_step.dependOn(result.step); + + run.addRun(b, kernel, plat); + + const rtl_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("src/rtl/root.zig"), + .target = b.graph.host, + .optimize = optimize, + }), + }); + + const run_tests = b.addRunArtifact(rtl_tests); + const test_step = b.step("test", "Run tests"); + + test_step.dependOn(&run_tests.step); +} + +fn targetQueryForPlatform(plat: config.Platform) std.Target.Query { + var q: std.Target.Query = .{ + .cpu_arch = plat.arch, + .os_tag = plat.os, + .abi = if (plat.os == .freestanding) .none else .gnu, + .cpu_model = .baseline, + }; + + if (plat.os == .freestanding) { + switch (plat.arch) { + .x86_64 => { + const f = std.Target.Cpu.Feature.FeatureSetFns(std.Target.x86.Feature); + q.cpu_features_sub = f.featureSet(&.{ .x87, .mmx, .sse, .sse2 }); + }, + .aarch64 => { + const f = std.Target.Cpu.Feature.FeatureSetFns(std.Target.aarch64.Feature); + q.cpu_features_sub = f.featureSet(&.{ .fp_armv8, .neon }); + }, + .riscv64 => {}, + else => {}, + } + } + + return q; +} + +fn addKernel(b: *std.Build, plat: config.Platform, optimize: std.builtin.OptimizeMode, config_module: *std.Build.Module, ksyms_module: *std.Build.Module, rtl: *std.Build.Module) *std.Build.Step.Compile { + const target = b.resolveTargetQuery(targetQueryForPlatform(plat)); + + const name = b.fmt("kernel-{s}", .{@tagName(plat.arch)}); + + const root_source_file = if (plat.os == .freestanding) "src/main.zig" else "src/platform/um/entry.zig"; + + const kernel = b.addExecutable(.{ + .name = name, + .root_module = b.createModule(.{ + .root_source_file = b.path(root_source_file), + .target = target, + .optimize = optimize, + }), + }); + + const base_module = b.createModule(.{ + .root_source_file = b.path("src/root.zig"), + .imports = &.{ + .{ .name = "config", .module = config_module }, + .{ .name = "ksyms", .module = ksyms_module }, + .{ .name = "rtl", .module = rtl }, + }, + }); + + base_module.addImport("base", base_module); + + kernel.root_module.addImport("config", config_module); + kernel.root_module.addImport("ksyms", ksyms_module); + kernel.root_module.addImport("base", base_module); + kernel.root_module.addImport("rtl", rtl); + + kernel.use_llvm = true; + kernel.use_lld = true; + + if (plat.os == .freestanding) { + kernel.linkage = .static; + kernel.pie = false; + kernel.bundle_compiler_rt = false; + kernel.bundle_ubsan_rt = false; + + kernel.root_module.link_libc = false; + kernel.root_module.pic = false; + kernel.root_module.single_threaded = true; + kernel.root_module.strip = false; + kernel.root_module.stack_check = false; + kernel.root_module.stack_protector = false; + kernel.root_module.unwind_tables = .none; + + kernel.entry = .{ .symbol_name = "kmain" }; + + kernel.root_module.code_model = switch (plat.arch) { + .x86_64 => .kernel, + .riscv64 => .medium, + else => .large, + }; + + switch (plat.arch) { + .x86_64 => kernel.root_module.red_zone = false, + else => {}, + } + + kernel.linker_script = b.path(b.fmt("build/linker-scripts/{s}.lds", .{@tagName(plat.arch)})); + } else { + kernel.root_module.link_libc = true; + } + + return kernel; +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..07fabb9 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,24 @@ +.{ + .fingerprint = 0x9179a89c7cb7baa, + .name = .zag, + .version = "0.1.0", + .minimum_zig_version = "0.15.0", + .dependencies = .{ + .limine = .{ + .url = "git+https://codeberg.org/Limine/Limine#v10.x-binary", + .hash = "N-V-__8AAL1qTADWWlnz4JFHM8Pg-h0E3DlZ-Fr5aMZ-gatM", + }, + + .ovmf = .{ + .url = "https://github.com/osdev0/edk2-ovmf-nightly/releases/download/nightly-20260318T020202Z/edk2-ovmf.tar.gz", + .hash = "N-V-__8AAAAAjAwsopZIlbp4U6NN1Eh573uVdLTwl1mkFUug", + }, + }, + + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + "build", + }, +} diff --git a/build/config.zig b/build/config.zig new file mode 100644 index 0000000..ea25810 --- /dev/null +++ b/build/config.zig @@ -0,0 +1,83 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +pub const Config = std.StringHashMap([]const u8); + +pub const Platform = struct { + arch: std.Target.Cpu.Arch, + os: std.Target.Os.Tag, +}; + +pub fn parseConfig(b: *std.Build) !Config { + const file = try std.Io.Dir.cwd().readFileAlloc(b.graph.io, ".config", b.allocator, std.Io.Limit.unlimited); + + var lines = std.mem.splitScalar(u8, file, '\n'); + + var map = std.StringHashMap([]const u8).init(b.allocator); + + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, " \r\t"); + if (trimmed.len == 0 or std.mem.startsWith(u8, trimmed, "#")) { + continue; + } + + var kv = std.mem.splitScalar(u8, trimmed, '='); + const key = kv.first(); + const value = kv.rest(); + + try map.put(key, value); + } + + return map; +} + +pub fn configStep(b: *std.Build) *std.Build.Step { + return &b.addSystemCommand(&.{ "bash", "build/menuconfig" }).step; +} + +fn isInteger(str: []const u8) bool { + _ = std.fmt.parseInt(i64, str, 10) catch return false; + return true; +} + +pub fn generateConfig(b: *std.Build, config: Config) !*std.Build.Step.WriteFile { + var out = std.Io.Writer.Allocating.init(b.allocator); + + var it = config.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, value, "y")) { + try out.writer.print("pub const {s}: bool = true;\n", .{key}); + } else if (isInteger(value)) { + try out.writer.print("pub const {s}: i64 = {s};\n", .{ key, value }); + } else { + try out.writer.print("pub const {s} = \"{s}\";\n", .{ key, value }); + } + } + + const wf = b.addWriteFiles(); + + _ = wf.add("config.gen.zig", out.written()); + _ = wf.addCopyFile(b.path(".config"), ".config"); + + return wf; +} + +pub fn getPlatform(config: Config) !Platform { + const keys = .{ + .{ "CONFIG_ARCH_AMD64", Platform{ .arch = .x86_64, .os = .freestanding } }, + .{ "CONFIG_ARCH_AARCH64", Platform{ .arch = .aarch64, .os = .freestanding } }, + .{ "CONFIG_ARCH_RISCV64", Platform{ .arch = .riscv64, .os = .freestanding } }, + .{ "CONFIG_ARCH_UM", Platform{ .arch = builtin.cpu.arch, .os = .linux } }, + }; + + inline for (keys) |entry| { + if (config.get(entry[0])) |val| { + if (std.mem.eql(u8, val, "y")) return entry[1]; + } + } + + return error.NoArchConfigured; +} diff --git a/build/image.zig b/build/image.zig new file mode 100644 index 0000000..29e21b1 --- /dev/null +++ b/build/image.zig @@ -0,0 +1,111 @@ +const std = @import("std"); + +const Arch = std.Target.Cpu.Arch; + +fn addLimineTool(b: *std.Build, limine: *std.Build.Dependency) *std.Build.Step.Compile { + const tool = b.addExecutable(.{ .name = "limine", .root_module = b.addModule("limine", .{ + .target = b.graph.host, + .link_libc = true, + }) }); + tool.root_module.addCSourceFile(.{ .file = limine.path("limine.c") }); + return tool; +} + +pub const IsoResult = struct { + step: *std.Build.Step, + path: std.Build.LazyPath, +}; + +pub fn addIso( + b: *std.Build, + arch: Arch, + image_name: []const u8, + kernel: *std.Build.Step.Compile, + limine: *std.Build.Dependency, +) IsoResult { + const limine_tool = addLimineTool(b, limine); + const kernel_bin = kernel.getEmittedBin(); + + const prep_script = + \\set -euo pipefail + \\kernel="$1" + \\iso_root="$2" + \\limine_conf="$3" + \\limine_dir="$4" + \\arch="$5" + \\rm -rf "$iso_root" + \\mkdir -p "$iso_root/boot/limine" "$iso_root/EFI/BOOT" + \\cp "$kernel" "$iso_root/boot/kernel" + \\cp "$limine_conf" "$iso_root/boot/limine/limine.conf" + \\case "$arch" in + \\ x86_64) + \\ cp "$limine_dir/limine-bios.sys" "$limine_dir/limine-bios-cd.bin" "$limine_dir/limine-uefi-cd.bin" "$iso_root/boot/limine/" + \\ cp "$limine_dir/BOOTX64.EFI" "$limine_dir/BOOTIA32.EFI" "$iso_root/EFI/BOOT/" + \\ ;; + \\ aarch64) + \\ cp "$limine_dir/limine-uefi-cd.bin" "$iso_root/boot/limine/" + \\ cp "$limine_dir/BOOTAA64.EFI" "$iso_root/EFI/BOOT/" + \\ ;; + \\ riscv64) + \\ cp "$limine_dir/limine-uefi-cd.bin" "$iso_root/boot/limine/" + \\ cp "$limine_dir/BOOTRISCV64.EFI" "$iso_root/EFI/BOOT/" + \\ ;; + \\ loongarch64) + \\ cp "$limine_dir/limine-uefi-cd.bin" "$iso_root/boot/limine/" + \\ cp "$limine_dir/BOOTLOONGARCH64.EFI" "$iso_root/EFI/BOOT/" + \\ ;; + \\esac + ; + + const iso_root = b.addTempFiles(); + + const prep = b.addSystemCommand(&.{ "bash", "-ceu", prep_script, "--" }); + + prep.addFileArg(kernel_bin); + prep.addDirectoryArg(iso_root.getDirectory()); + prep.addFileArg(b.path("build/limine.conf")); + prep.addDirectoryArg(limine.path(".")); + prep.addArg(@tagName(arch)); + prep.step.dependOn(&iso_root.step); + + const mkiso = b.addSystemCommand(&.{ "xorriso", "-as", "mkisofs", "-R", "-r", "-J" }); + switch (arch) { + .x86_64 => mkiso.addArgs(&.{ + "-b", "boot/limine/limine-bios-cd.bin", + "-no-emul-boot", "-boot-load-size", + "4", "-boot-info-table", + "-hfsplus", "-apm-block-size", + "2048", "--efi-boot", + "boot/limine/limine-uefi-cd.bin", "-efi-boot-part", + "--efi-boot-image", "--protective-msdos-label", + }), + else => mkiso.addArgs(&.{ + "-hfsplus", + "-apm-block-size", + "2048", + "--efi-boot", + "boot/limine/limine-uefi-cd.bin", + "-efi-boot-part", + "--efi-boot-image", + "--protective-msdos-label", + }), + } + mkiso.addDirectoryArg(iso_root.getDirectory()); + mkiso.addArg("-o"); + const iso_out = mkiso.addOutputFileArg(b.fmt("{s}.iso", .{image_name})); + mkiso.step.dependOn(&prep.step); + + if (arch == .x86_64) { + const bios_install = b.addRunArtifact(limine_tool); + bios_install.addArg("bios-install"); + bios_install.addFileArg(iso_out); + bios_install.step.dependOn(&mkiso.step); + } + + const install_iso = b.addInstallFile(iso_out, b.fmt("{s}.iso", .{image_name})); + install_iso.step.dependOn(&mkiso.step); + return .{ + .step = &install_iso.step, + .path = iso_out, + }; +} diff --git a/build/ksyms.py b/build/ksyms.py new file mode 100755 index 0000000..69062c5 --- /dev/null +++ b/build/ksyms.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# Simple kernel symbol dumper +# Usage: ksyms.py --input --output + +import sys +import subprocess +import argparse + +excluded_symbols = ['text_start_addr', 'text_end_addr', 'rodata_start_addr', 'rodata_end_addr', 'data_start_addr', 'data_end_addr'] + +def parse_nm_line(line: str) -> tuple[int, str]: + parts = line.split() + if len(parts) < 3: + return None + + addr_str, type, name = parts[0], parts[1], parts[2] + + # Only get symbols in text + if type.lower() != 't': + return None + + if name in excluded_symbols: + return None + + addr = int(addr_str, 16) + return addr, name + +def run_nm(binary: str) -> list[str]: + + result = subprocess.run(["nm", "-n", binary], capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"nm failed: {result.stderr}") + return result.stdout.splitlines() + +parser = argparse.ArgumentParser(description="Generate kernel symbols from a binary") +parser.add_argument("--input", help="Input kernel binary") +parser.add_argument("--output", help="Output C file") +parser.add_argument("--empty", action="store_true", help="Generate empty ksyms") +args = parser.parse_args() + +if args.empty: + with open(args.output, 'w') as f: + f.write("// Auto-generated by build/ksyms.py - do not edit\n\n") + f.write("pub const KSym = struct {\n addr: u64,\n name: []const u8,\n};\n\n") + f.write("pub const ksyms: []const KSym = &.{};\n") + sys.exit(0) + +lines = run_nm(args.input) + +symbols = [] +for line in lines: + parsed = parse_nm_line(line) + if parsed: + symbols.append(parsed) + + +# Sort by address +symbols.sort(key=lambda x: x[0]) + +with open(args.output, 'w') as f: + f.write("// Auto-generated by build/ksyms.py - do not edit\n\n") + f.write("pub const KSym = struct {\n addr: u64,\n name: []const u8,\n};\n\n") + f.write("pub const ksyms: []const KSym = &.{\n") + for addr, name in symbols: + f.write(f" .{{ .addr = 0x{addr:016x}, .name = \"{name}\" }},\n") + f.write("};\n") diff --git a/build/limine.conf b/build/limine.conf new file mode 100644 index 0000000..05e5683 --- /dev/null +++ b/build/limine.conf @@ -0,0 +1,11 @@ +# Timeout in seconds that Limine will use before automatically booting. +timeout: 3 + +# The entry name that will be displayed in the boot menu. +/Limine Template (Zig) + # We use the Limine boot protocol. + protocol: limine + + # Path to the kernel to boot. boot():/ represents the partition on which limine.conf is located. + path: boot():/boot/kernel + diff --git a/build/linker-scripts/aarch64.lds b/build/linker-scripts/aarch64.lds new file mode 100644 index 0000000..5cc6687 --- /dev/null +++ b/build/linker-scripts/aarch64.lds @@ -0,0 +1,75 @@ +/* Tell the linker that we want an aarch64 ELF64 output file */ +OUTPUT_FORMAT(elf64-littleaarch64) + +/* We want the symbol kmain to be our entry point */ +ENTRY(kmain) + +/* Define the program headers we want so the bootloader gives us the right */ +/* MMU permissions; this also allows us to exert more control over the linking */ +/* process. */ +PHDRS +{ + limine_requests PT_LOAD; + text PT_LOAD; + rodata PT_LOAD; + data PT_LOAD; +} + +SECTIONS +{ + /* We want to be placed in the topmost 2GiB of the address space, for optimisations */ + /* and because that is what the Limine spec mandates. */ + /* Any address in this region will do, but often 0xffffffff80000000 is chosen as */ + /* that is the beginning of the region. */ + . = 0xffffffff80000000; + + /* Define a section to contain the Limine requests and assign it to its own PHDR */ + .limine_requests : { + KEEP(*(.limine_requests_start)) + KEEP(*(.limine_requests)) + KEEP(*(.limine_requests_end)) + } :limine_requests + + /* Move to the next memory page for .text */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .text : { + *(.text .text.*) + } :text + + /* Move to the next memory page for .rodata */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .rodata : { + *(.rodata .rodata.*) + } :rodata + + /* Add a .note.gnu.build-id output section in case a build ID flag is added to the */ + /* linker command. */ + .note.gnu.build-id : { + *(.note.gnu.build-id) + } :rodata + + /* Move to the next memory page for .data */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .data : { + *(.data .data.*) + } :data + + /* NOTE: .bss needs to be the last thing mapped to :data, otherwise lots of */ + /* unnecessary zeros will be written to the binary. */ + /* If you need, for example, .init_array and .fini_array, those should be placed */ + /* above this. */ + .bss : { + *(.bss .bss.*) + *(COMMON) + } :data + + /* Discard .note.* and .eh_frame* since they may cause issues on some hosts. */ + /DISCARD/ : { + *(.eh_frame*) + *(.note .note.*) + } +} + diff --git a/build/linker-scripts/loongarch64.lds b/build/linker-scripts/loongarch64.lds new file mode 100644 index 0000000..a935e85 --- /dev/null +++ b/build/linker-scripts/loongarch64.lds @@ -0,0 +1,75 @@ +/* Tell the linker that we want a loongarch64 ELF64 output file */ +OUTPUT_FORMAT(elf64-loongarch) + +/* We want the symbol kmain to be our entry point */ +ENTRY(kmain) + +/* Define the program headers we want so the bootloader gives us the right */ +/* MMU permissions; this also allows us to exert more control over the linking */ +/* process. */ +PHDRS +{ + limine_requests PT_LOAD; + text PT_LOAD; + rodata PT_LOAD; + data PT_LOAD; +} + +SECTIONS +{ + /* We want to be placed in the topmost 2GiB of the address space, for optimisations */ + /* and because that is what the Limine spec mandates. */ + /* Any address in this region will do, but often 0xffffffff80000000 is chosen as */ + /* that is the beginning of the region. */ + . = 0xffffffff80000000; + + /* Define a section to contain the Limine requests and assign it to its own PHDR */ + .limine_requests : { + KEEP(*(.limine_requests_start)) + KEEP(*(.limine_requests)) + KEEP(*(.limine_requests_end)) + } :limine_requests + + /* Move to the next memory page for .text */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .text : { + *(.text .text.*) + } :text + + /* Move to the next memory page for .rodata */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .rodata : { + *(.rodata .rodata.*) + } :rodata + + /* Add a .note.gnu.build-id output section in case a build ID flag is added to the */ + /* linker command. */ + .note.gnu.build-id : { + *(.note.gnu.build-id) + } :rodata + + /* Move to the next memory page for .data */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .data : { + *(.data .data.*) + } :data + + /* NOTE: .bss needs to be the last thing mapped to :data, otherwise lots of */ + /* unnecessary zeros will be written to the binary. */ + /* If you need, for example, .init_array and .fini_array, those should be placed */ + /* above this. */ + .bss : { + *(.bss .bss.*) + *(COMMON) + } :data + + /* Discard .note.* and .eh_frame* since they may cause issues on some hosts. */ + /DISCARD/ : { + *(.eh_frame*) + *(.note .note.*) + } +} + diff --git a/build/linker-scripts/riscv64.lds b/build/linker-scripts/riscv64.lds new file mode 100644 index 0000000..f5f1cda --- /dev/null +++ b/build/linker-scripts/riscv64.lds @@ -0,0 +1,75 @@ +/* Tell the linker that we want a riscv64 ELF64 output file */ +OUTPUT_FORMAT(elf64-littleriscv) + +/* We want the symbol kmain to be our entry point */ +ENTRY(kmain) + +/* Define the program headers we want so the bootloader gives us the right */ +/* MMU permissions; this also allows us to exert more control over the linking */ +/* process. */ +PHDRS +{ + limine_requests PT_LOAD; + text PT_LOAD; + rodata PT_LOAD; + data PT_LOAD; +} + +SECTIONS +{ + /* We want to be placed in the topmost 2GiB of the address space, for optimisations */ + /* and because that is what the Limine spec mandates. */ + /* Any address in this region will do, but often 0xffffffff80000000 is chosen as */ + /* that is the beginning of the region. */ + . = 0xffffffff80000000; + + /* Define a section to contain the Limine requests and assign it to its own PHDR */ + .limine_requests : { + KEEP(*(.limine_requests_start)) + KEEP(*(.limine_requests)) + KEEP(*(.limine_requests_end)) + } :limine_requests + + /* Move to the next memory page for .text */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .text : { + *(.text .text.*) + } :text + + /* Move to the next memory page for .rodata */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .rodata : { + *(.rodata .rodata.*) + } :rodata + + /* Add a .note.gnu.build-id output section in case a build ID flag is added to the */ + /* linker command. */ + .note.gnu.build-id : { + *(.note.gnu.build-id) + } :rodata + + /* Move to the next memory page for .data */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .data : { + *(.data .data.*) + } :data + + /* NOTE: .bss needs to be the last thing mapped to :data, otherwise lots of */ + /* unnecessary zeros will be written to the binary. */ + /* If you need, for example, .init_array and .fini_array, those should be placed */ + /* above this. */ + .bss : { + *(.bss .bss.*) + *(COMMON) + } :data + + /* Discard .note.* and .eh_frame* since they may cause issues on some hosts. */ + /DISCARD/ : { + *(.eh_frame*) + *(.note .note.*) + } +} + diff --git a/build/linker-scripts/x86_64.lds b/build/linker-scripts/x86_64.lds new file mode 100644 index 0000000..0f0da9d --- /dev/null +++ b/build/linker-scripts/x86_64.lds @@ -0,0 +1,75 @@ +/* Tell the linker that we want an x86_64 ELF64 output file */ +OUTPUT_FORMAT(elf64-x86-64) + +/* We want the symbol kmain to be our entry point */ +ENTRY(kmain) + +/* Define the program headers we want so the bootloader gives us the right */ +/* MMU permissions; this also allows us to exert more control over the linking */ +/* process. */ +PHDRS +{ + limine_requests PT_LOAD; + text PT_LOAD; + rodata PT_LOAD; + data PT_LOAD; +} + +SECTIONS +{ + /* We want to be placed in the topmost 2GiB of the address space, for optimisations */ + /* and because that is what the Limine spec mandates. */ + /* Any address in this region will do, but often 0xffffffff80000000 is chosen as */ + /* that is the beginning of the region. */ + . = 0xffffffff80000000; + + /* Define a section to contain the Limine requests and assign it to its own PHDR */ + .limine_requests : { + KEEP(*(.limine_requests_start)) + KEEP(*(.limine_requests)) + KEEP(*(.limine_requests_end)) + } :limine_requests + + /* Move to the next memory page for .text */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .text : { + *(.text .text.*) + } :text + + /* Move to the next memory page for .rodata */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .rodata : { + *(.rodata .rodata.*) + } :rodata + + /* Add a .note.gnu.build-id output section in case a build ID flag is added to the */ + /* linker command. */ + .note.gnu.build-id : { + *(.note.gnu.build-id) + } :rodata + + /* Move to the next memory page for .data */ + . = ALIGN(CONSTANT(MAXPAGESIZE)); + + .data : { + *(.data .data.*) + } :data + + /* NOTE: .bss needs to be the last thing mapped to :data, otherwise lots of */ + /* unnecessary zeros will be written to the binary. */ + /* If you need, for example, .init_array and .fini_array, those should be placed */ + /* above this. */ + .bss : { + *(.bss .bss.*) + *(COMMON) + } :data + + /* Discard .note.* and .eh_frame* since they may cause issues on some hosts. */ + /DISCARD/ : { + *(.eh_frame*) + *(.note .note.*) + } +} + diff --git a/build/menuconfig b/build/menuconfig new file mode 100755 index 0000000..db7ea8a --- /dev/null +++ b/build/menuconfig @@ -0,0 +1,23 @@ +#!/bin/bash + +frontend="${FRONTEND:-kconfig-nconf}" + +has_kconfig_frontends=true + +if ! command -v kconfig-mconf &> /dev/null +then + has_kconfig_frontends=false +fi + + +if [ "$has_kconfig_frontends" = false ]; then + if ! command -v python3 -m menuconfig -h &> /dev/null + then + echo "No Kconfig frontend found. Please install kconfig-frontends or python3-kconfiglib" + exit 1 + fi + + python3 -m menuconfig +else + $frontend Kconfig +fi diff --git a/build/run.zig b/build/run.zig new file mode 100644 index 0000000..9cf68ab --- /dev/null +++ b/build/run.zig @@ -0,0 +1,69 @@ +const std = @import("std"); +const config = @import("config.zig"); +const image = @import("image.zig"); + +const QemuArgs = struct { + suffix: []u8, + args: [][]u8, +}; + +pub fn addRun(b: *std.Build, kernel: *std.Build.Step.Compile, plat: config.Platform) void { + const run_step = b.step("run", "Run the kernel"); + const uefi = b.option(bool, "uefi", "Boot with UEFI firmware") orelse false; + const debug = b.option(bool, "debug", "Boot in debug mode") orelse false; + const extra_qemu_args = b.option([]const []const u8, "qemu-args", "Extra arguments to pass to QEMU") orelse &.{}; + + if (plat.os == .freestanding) { + const limine = b.dependency("limine", .{}); + const ovmf = b.dependency("ovmf", .{}); + const iso_out = image.addIso(b, plat.arch, "myos", kernel, limine); + + const qemu_bin = switch (plat.arch) { + .x86_64 => "qemu-system-x86_64", + .aarch64 => "qemu-system-aarch64", + .riscv64 => "qemu-system-riscv64", + else => "qemu-system-x86_64", + }; + const ovmf_fd = switch (plat.arch) { + .x86_64 => "ovmf-code-x86_64.fd", + .aarch64 => "ovmf-code-aarch64.fd", + .riscv64 => "ovmf-code-riscv64.fd", + else => "ovmf-code-x86_64.fd", + }; + + const qemu = b.addSystemCommand(&.{qemu_bin}); + + if (plat.arch != .x86_64 or uefi) { + qemu.addArg("-drive"); + qemu.addPrefixedFileArg("if=pflash,format=raw,readonly=on,file=", ovmf.path(ovmf_fd)); + } + + qemu.addArgs(extra_qemu_args); + qemu.addArg("-cdrom"); + qemu.addFileArg(iso_out.path); + + if (debug) { + qemu.addArgs(&.{ + "-serial", "stdio", + "-d", "int", + "-D", "qemu.log", + "-no-reboot", "-no-shutdown", + }); + } + + switch (plat.arch) { + .x86_64 => { + qemu.addArgs(&.{ "-debugcon", "stdio" }); + + if (!debug) qemu.addArgs(&.{ "-enable-kvm", "-cpu", "host" }); + }, + .aarch64 => qemu.addArgs(&.{ "-machine", "virt", "-cpu", "cortex-a57" }), + .riscv64 => qemu.addArgs(&.{ "-machine", "virt" }), + else => {}, + } + + run_step.dependOn(&qemu.step); + } else { + run_step.dependOn(&b.addRunArtifact(kernel).step); + } +} diff --git a/scripts/genconfig.py b/scripts/genconfig.py new file mode 100755 index 0000000..6c35a9f --- /dev/null +++ b/scripts/genconfig.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Generate C header file from .config +# Usage: genconfig.py +import sys +import argparse + +parser = argparse.ArgumentParser(description="Generate C header file from KConfig") +parser.add_argument("output", help="Output C file") +args = parser.parse_args() + +values = {} + +def value_from_str(string): + if string == 'y': + return 1 + if string == 'n': + return 0 + return string + +with open('.config', 'r') as f: + lines = f.read().splitlines() + + + for line in lines: + if line.startswith('#') or len(line) == 0: + continue + vals = line.split('=') + values[vals[0]] = value_from_str(vals[1]) + +with open(args.output, 'w') as f: + f.write('/* This file is auto-generated by scripts/genconfig.py */\n') + f.write('#ifndef GENCONFIG_CONFIG_H_\n') + f.write('#define GENCONFIG_CONFIG_H_\n') + + for key, val in values.items(): + f.write(f'#define {key} {val}\n') + f.write('#endif\n') diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig new file mode 100644 index 0000000..e69de29 diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig new file mode 100644 index 0000000..1d93076 --- /dev/null +++ b/src/kern/cpu.zig @@ -0,0 +1,88 @@ +const std = @import("std"); +const rtl = @import("rtl"); +const b = @import("base"); +const ke = b.ke; +const ki = b.ke.private; + +comptime { + if (!@hasDecl(ki.impl, "Cpu")) @compileError("impl must provide Cpu"); +} + +pub const ThreadStatus = packed struct(u8) { + interactive: bool, + priority: u7, +}; + +pub const PreemptionReason = enum(u8) { + None, + /// A higher priority thread was readied. + HigherPriority, +}; + +/// Per-CPU state +pub const Cpu = struct { + /// Implementation-dependent CPU data. + impl: ki.impl.Cpu, + /// Unique ID of this CPU. + id: u32, + /// Current IPL on this CPU. + ipl: ke.Ipl, + /// Bitmask of pending software interrupts on this CPU. + pending_softints: std.atomic.Value(u8), + /// Currently running thread. + current_thread: ?*ke.Thread, + /// Atomic byte for getting the current thread's status locklessly. + /// 1 bit for thread interactivity, 7 for priority. + current_thread_status: std.atomic.Value(ThreadStatus), + /// Per-CPU idle thread. + idle_thread: ?*ke.Thread, + /// Thread selected for preemption. + next_thread: ?*ke.Thread, + /// The reason why the current thread was preempted. + preemption_reason: PreemptionReason, + /// Flag indicating the start of the scheduling timer. + start_timer: bool, + /// Queue of DPCs on this CPU. + dpc_queue: rtl.List, + /// Lock over this CPU's DPC queue. + dpc_lock: ke.SpinLock, + /// Heap of pending timers on this CPU. + timers_heap: rtl.PairingHeap(.min, ki.timer.cmp_timer), + /// Lock over the timer heap. + timers_lock: ke.SpinLock, + /// Per-CPU scheduler state. + sched_data: ki.sched.PerCpu, + /// DPCs + resched_dpc: ke.Dpc, + timer_dpc: ke.Dpc, + resched_timer: ke.Timer, + + /// Initialize `cpu` for usage with its idle thread being `thread`. + pub fn init(cpu: *Cpu, thread: *ke.Thread) void { + cpu.* = .{ + .id = cpu.id, + .ipl = .Zero, + .pending_softints = .init(0), + .idle_thread = thread, + .current_thread = thread, + .timer_dpc = .init(ki.timer.handle_expiry), + .resched_dpc = .init(ki.sched.clock), + .timers_heap = .init(), + .current_thread_status = .init(.{ .interactive = false, .priority = 0 }), + .next_thread = null, + .dpc_lock = .init(), + .preemption_reason = .None, + .start_timer = false, + .dpc_queue = undefined, + .timers_lock = .init(), + .sched_data = undefined, + .resched_timer = undefined, + .impl = cpu.impl, + }; + + cpu.dpc_queue.init(); + cpu.resched_timer.init(); + + ki.sched.init_cpu(&cpu.sched_data); + } +}; diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig new file mode 100644 index 0000000..a042491 --- /dev/null +++ b/src/kern/dpc.zig @@ -0,0 +1,133 @@ +const std = @import("std"); +const config = @import("config"); +const rtl = @import("rtl"); +const b = @import("base"); +const ke = b.ke; +const ki = ke.private; + +/// Deferred Procedure Call (DPC) structure. +/// Used for scheduling work to be done when IPL is lowered below `ke.Ipl.Dispatch`. +pub const Dpc = struct { + /// Entry into per-CPU DPC queue + link: rtl.List.Entry, + /// Routine to call + func: *const fn (?*anyopaque) void, + /// Argument passed to the routine + arg: ?*anyopaque, + /// Whether or not the DPC is currently inserted + inserted: bool, + + /// Initialize a DPC for `func`. + pub fn init(func: *const fn (?*anyopaque) void) Dpc { + return .{ + .link = undefined, + .func = func, + .arg = undefined, + .inserted = false, + }; + } +}; + +/// Enqueue a DPC. +/// If the DPC is already enqueued, this function is a no-op. +pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { + const ipl = ke.ipl.raise(.High); + const cpu = ke.curcpu(); + + if (!dpc.inserted) { + dpc.arg = arg; + + // Insert the DPC on this CPU's DPC queue + cpu.dpc_lock.acquire_no_ipl(); + + cpu.dpc_queue.insert_tail(&dpc.link); + + // Mark the DPC as pending on this CPU + ki.ipl.set_softint_pending(cpu, .Dispatch); + + dpc.inserted = true; + + cpu.dpc_lock.release_no_ipl(); + } + + ke.ipl.lower(ipl); +} + +fn dispatch_queue(cpu: *ke.Cpu) void { + cpu.ipl = .Dispatch; + + // Mark as handled, this must be done before enabling interrupts or we will race. + ki.ipl.clear_softint_pending(cpu, .Dispatch); + + _ = ki.impl.enable_interrupts(); + + while (true) { + const ipl = cpu.dpc_lock.acquire_at(.High); + + var dpc: *Dpc = undefined; + + if (cpu.dpc_queue.is_empty()) { + // No more DPCs to process. + cpu.dpc_lock.release(ipl); + break; + } + + // Pop the head + var first_elem = cpu.dpc_queue.first(); + + first_elem.remove(); + + dpc = @fieldParentPtr("link", first_elem); + dpc.inserted = false; + + // An interrupt which would enqueue this DPC could occur between loading the argument and calling the routine, + // which is why we capture it here to ensure that we get the intended context. + const arg = dpc.arg; + + cpu.dpc_lock.release(ipl); + + std.debug.assert(cpu.ipl == .Dispatch); + + dpc.func(arg); + } + + if (cpu.start_timer) { + ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc); + } + + if (cpu.preemption_reason == .HigherPriority) { + // Reload the quantum for the new thread. + ke.timer.cancel(&cpu.resched_timer); + + ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc); + } + + cpu.start_timer = false; + cpu.preemption_reason = .None; + + if (cpu.next_thread != null) { + ki.sched.handle_preemption(cpu); + } + + _ = ki.impl.disable_interrupts(); +} + +/// Dispatch the DPC queue on `cpu`. +pub fn dispatch(cpu: *ke.Cpu) void { + // DPC processing is done at Ipl.Dispatch + // We can't call lower/raise here because they might call us again recursively. + var mycpu = cpu; + const old_ipl = cpu.ipl; + const int_state = ki.impl.disable_interrupts(); + + while (ki.ipl.is_softint_pending(mycpu, .Dispatch)) { + dispatch_queue(mycpu); + + // Our CPU might have changed. + mycpu = ke.curcpu(); + } + + cpu.ipl = old_ipl; + + ki.impl.restore_interrupts(int_state); +} diff --git a/src/kern/init.zig b/src/kern/init.zig new file mode 100644 index 0000000..db2d908 --- /dev/null +++ b/src/kern/init.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const b = @import("base"); +const pl = b.pl; +const ke = b.ke; +const ki = ke.private; + +var thread0: ke.Thread = undefined; + +fn idle(_: ?*anyopaque) void { + while (true) { + std.atomic.spinLoopHint(); + } +} + +var stack: [8192]u8 align(16) = undefined; + +fn handler(_: ?*anyopaque) void { + std.log.info("Hi! CPU {}", .{ke.curcpu().id}); + + while (true) {} +} + +fn make_thread(entry: *const fn (?*anyopaque) void, td: *ke.Thread) void { + const _stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf"); + + td.init(@intFromPtr(_stack.ptr), 16384, entry, null); +} + +var threads: [10]ke.Thread = undefined; + +pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { + pl.early_init(); + ki.bootstrap_cpu.init(&thread0); + + thread0.init(@intFromPtr(&stack), 8192, idle, null); + thread0.priority = 0; + thread0.priority_class = .Idle; + + std.log.info("hello, world", .{}); + std.log.info("Zag for {s}, cmdline is {?s}", .{ pl.name, boot_info.cmdline }); + + pl.late_init(); + + _ = ke.ipl.raise(.Dispatch); + + for (0..10) |i| { + make_thread(handler, &threads[i]); + ke.sched.enqueue(&threads[i]); + } + + ke.ipl.lower(.Zero); + + while (true) {} +} diff --git a/src/kern/ipl.zig b/src/kern/ipl.zig new file mode 100644 index 0000000..79584b5 --- /dev/null +++ b/src/kern/ipl.zig @@ -0,0 +1,96 @@ +const std = @import("std"); +const b = @import("base"); +const ke = b.ke; +const ki = ke.private; + +/// Interrupt priority level (IPL) +pub const Ipl = enum(u8) { + /// All interrupts enabled. + Zero = 0, + /// Preemption disabled. + Dispatch = 1, + /// Device interrupts disabled. + Device = 13, + /// All interrupts disabled. + High = 15, + + pub fn get_max_software() Ipl { + return .Dispatch; + } +}; + +/// Raise IPL to `new`. +/// Panics if the current IPL is higher than requested. +pub fn raise(new: Ipl) Ipl { + var cpu = ke.curcpu(); + const old = cpu.ipl; + + if (@intFromEnum(new) < @intFromEnum(old)) { + ke.panic("ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})\n", .{ new, old }); + } + + if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) { + ki.impl.set_hardware_ipl(new); + } + + cpu.ipl = new; + + return old; +} + +/// Lower IPL to `new`. +/// Panics if the current IPL is lower than requested. +pub fn lower(new: Ipl) void { + var cpu = ke.curcpu(); + const old = cpu.ipl; + + if (@intFromEnum(new) > @intFromEnum(old)) { + ke.panic("ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})\n", .{ new, old }); + } + + if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) { + ki.impl.set_hardware_ipl(.Zero); + } + + if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(cpu, .Dispatch)) { + ki.dpc.dispatch(cpu); + } + + cpu.ipl = new; +} + +comptime { + if (!@hasDecl(ki.impl, "set_hardware_ipl")) @compileError("impl must provide set_hardware_ipl()"); +} + +/// Set the hardware IPL to `new` +pub fn set_hardware(new: Ipl) Ipl { + var cpu = ke.curcpu(); + const old = cpu.ipl; + + cpu.ipl = new; + + ki.impl.set_hardware_ipl(new); + + return old; +} + +/// Mark a software interrupt of IPL `ipl` on `cpu` as pending +pub fn set_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void { + std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); + _ = cpu.pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic); +} + +/// Mark a software interrupt of IPL `ipl` on `cpu` as handled +pub fn clear_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void { + std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); + _ = cpu.pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic); +} + +/// Check whether a software interrupt of IPL `ipl` on `cpu` is pending +pub fn is_softint_pending(cpu: *ke.Cpu, ipl: Ipl) bool { + std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); + const shift: u3 = @intCast(@intFromEnum(ipl)); + const bit: u8 = @as(u8, 1) << shift; + return cpu.pending_softints.load(.monotonic) & bit != 0; +} diff --git a/src/kern/log.zig b/src/kern/log.zig new file mode 100644 index 0000000..45f68f7 --- /dev/null +++ b/src/kern/log.zig @@ -0,0 +1,57 @@ +const std = @import("std"); +const b = @import("base"); +const pl = b.pl; +const ke = b.ke; + +pub var debug_lock = ke.SpinLock.init(); + +pub fn print(bytes: []const u8) void { + for (bytes) |c| { + pl.debug_write(c); + } +} + +const LogWriter = struct { + interface: std.Io.Writer, + + pub fn init() LogWriter { + return .{ + .interface = .{ + .vtable = &.{ .drain = drain }, + .buffer = &.{}, + .end = 0, + }, + }; + } + + fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + var total_written: usize = 0; + for (data) |slice| { + print(slice); + total_written += slice.len; + } + + return total_written; + } +}; + +pub fn log( + comptime level: std.log.Level, + comptime scope: @EnumLiteral(), + comptime fmt: []const u8, + args: anytype, +) void { + _ = level; + _ = scope; + + const ipl = debug_lock.acquire(); + + var writer = LogWriter.init(); + + writer.interface.print( + fmt ++ "\r\n", + args, + ) catch unreachable; + + debug_lock.release(ipl); +} diff --git a/src/kern/panic.zig b/src/kern/panic.zig new file mode 100644 index 0000000..00a8120 --- /dev/null +++ b/src/kern/panic.zig @@ -0,0 +1,19 @@ +const std = @import("std"); +const b = @import("base"); +const ke = b.ke; +const ki = ke.private; + +/// Panics! +pub fn panic(comptime fmt: []const u8, args: anytype) noreturn { + ki.log.debug_lock.acquire_no_ipl(); + + var buf: [1024]u8 = undefined; + + const str = std.fmt.bufPrint(&buf, fmt, args) catch unreachable; + + ki.log.print("Panic!\n"); + ki.log.print(str); + + ki.log.debug_lock.release_no_ipl(); + while (true) {} +} diff --git a/src/kern/private.zig b/src/kern/private.zig new file mode 100644 index 0000000..d6a89e9 --- /dev/null +++ b/src/kern/private.zig @@ -0,0 +1,31 @@ +const std = @import("std"); +const config = @import("config"); + +const b = @import("base"); +const ke = b.ke; +const pl = b.pl; + +const init_mod = @import("init.zig"); + +pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64")) + @import("amd64/impl.zig") +else if (@hasDecl(config, "CONFIG_ARCH_UM")) + @import("um/impl.zig") +else + @compileError("unsupported architecture"); + +pub const init = init_mod.init; +pub var bootstrap_cpu: ke.Cpu = undefined; + +// === Exported Modules === +pub const ipl = @import("ipl.zig"); +pub const panic = @import("panic.zig"); +pub const spinlock = @import("spinlock.zig"); +pub const log = @import("log.zig"); +pub const cpu = @import("cpu.zig"); +pub const thread = @import("thread.zig"); +pub const dpc = @import("dpc.zig"); +pub const timecounter = @import("timecounter.zig"); +pub const sched = @import("sched.zig"); +pub const wait = @import("wait.zig"); +pub const timer = @import("timer.zig"); diff --git a/src/kern/root.zig b/src/kern/root.zig new file mode 100644 index 0000000..458cd53 --- /dev/null +++ b/src/kern/root.zig @@ -0,0 +1,68 @@ +const config = @import("config"); + +// === Exported Modules === +pub const private = @import("private.zig"); + +const p = private; + +// === Exported types === +pub const Ipl = p.ipl.Ipl; +pub const SpinLock = p.spinlock.SpinLock; +pub const Thread = p.thread.Thread; +pub const Dpc = p.dpc.Dpc; +pub const TimeCounter = p.timecounter.TimeCounter; +pub const Timer = p.timer.Timer; + +// === Exported Interfaces === +pub const log = p.log.log; +pub const panic = p.panic.panic; + +pub const ipl = struct { + pub const raise = p.ipl.raise; + pub const lower = p.ipl.lower; + pub const set_hardware = p.ipl.set_hardware; +}; + +pub const thread = struct { + pub const init = p.thread.init; +}; + +pub const dpc = struct { + pub const enqueue = p.dpc.enqueue; +}; + +pub const timecounter = struct { + pub const register = p.timecounter.register; + pub const read_time_nano = p.timecounter.read_time_nano; +}; + +pub const sched = struct { + pub const enqueue = p.sched.enqueue; + pub const block = p.sched.block; + pub const unblock = p.sched.unblock; +}; + +pub const timer = struct { + pub const set = p.timer.set; + pub const cancel = p.timer.cancel; +}; + +pub const wait = struct { + pub const wait_one = p.wait.wait_one; + pub const wait_any = p.wait.wait_any; +}; + +pub const Cpu = p.cpu.Cpu; + +comptime { + if (!@hasDecl(p.impl, "curcpu")) @compileError("impl must provide curcpu()"); +} + +/// Returns the current CPU +pub const curcpu: fn () *Cpu = p.impl.curcpu; + +/// Number of CPUs on the system +pub var ncpus: usize = 0; + +/// Array of CPUs +pub var cpus: []*Cpu = undefined; diff --git a/src/kern/sched.zig b/src/kern/sched.zig new file mode 100644 index 0000000..a3d7373 --- /dev/null +++ b/src/kern/sched.zig @@ -0,0 +1,764 @@ +//! Kernel scheduler inspired by FreeBSD's ULE scheduler. +//! +//! ## Overview +//! +//! Threads are divided into three priority classes: +//! +//! - `Realtime`: Always scheduled before batch and idle threads. +//! Picked from `realtime_runq` in strict priority order. +//! Interactive batch threads are promoted into this queue. +//! - `Batch`: Scheduled via a circular calendar queue that rotates +//! on every tick, ensuring all threads get CPU time proportional +//! to their priority. +//! - `Idle`: Only scheduled when both realtime and batch queues are empty. +//! +//! ## Interactivity Scoring +//! +//! Each batch thread is assigned an interactivity score based on the +//! ratio of its recent sleep time to its recent CPU time: +//! +//! - If `sleep_time > run_time`: score = `run_time / (sleep_time / scaling_factor)` +//! — heavily sleeping threads score low (more interactive). +//! - If `run_time > sleep_time`: score = `scaling_factor * 2 - (sleep_time / (run_time / scaling_factor))` +//! — CPU-bound threads score high (less interactive). +//! - Niceness is added to the score, making low-nice threads easier to +//! qualify as interactive. +//! +//! Threads scoring below `interactivity_threshold` are considered +//! interactive and promoted to the realtime queue with a priority derived +//! from their score. This lets I/O-bound threads respond quickly without +//! being assigned a static realtime priority. +//! +//! To prevent history from dominating forever, `sleep_time` and `run_time` +//! are decayed periodically by `sched_clamp_time`: +//! - Values are halved when their sum exceeds 120% of the 5-second cap. +//! - Values are reduced by 20% when the sum exceeds 5 seconds. +//! - Values are reset to (max, 1) when the sum exceeds 10 seconds. +//! +//! ## Calendar Queue +//! +//! The calendar queue is a circular array of `runqueues_n` (64) lists. +//! Two indices drive it: +//! +//! - `runidx`: The queue currently being drained by the scheduler. +//! - `insidx`: The queue where newly inserted threads land. +//! Always one slot ahead of `runidx` so fresh threads are never +//! picked in the same tick they were enqueued. +//! +//! Threads are inserted at `(insidx + prio_high_batch - thread.priority) % runqueues_n`, +//! placing lower-priority threads further ahead in the rotation so +//! they run less frequently than higher-priority ones. +//! +//! On every tick (10ms), `insidx` advances by one, giving all threads +//! the chance to run over time regardless of priority. +//! +//! ## Preemption +//! +//! Timeshared (batch) threads do not preempt each other by default, +//! mirroring FreeBSD's default preemption threshold. Interactive threads +//! may still preempt non-interactive batch threads. Realtime threads +//! always preempt batch and idle threads. +//! +//! Remote preemption (sending an IPI to another CPU) is done lock-free +//! by reading a packed `current_thread_status` field that encodes the +//! running thread's priority and interactivity in a single atomic byte. +//! +//! ## Load Balancing +//! +//! Two mechanisms keep load distributed across CPUs: +//! +//! - **Work stealing**: When a CPU goes idle, it immediately attempts to +//! steal a thread from the most loaded CPU. +//! - **Periodic balancing**: Every ~1 second (randomized with an LCG to +//! avoid phase-locking with periodic threads), the bootstrap CPU moves +//! threads from the most loaded CPU to the least loaded one. +//! +//! ## CPU Selection +//! +//! When enqueuing a thread, the target CPU is chosen in priority order: +//! 1. Last CPU the thread ran on, if it can be preempted or the thread is pinned. +//! 2. Least loaded CPU on which the thread would run immediately (preemptible). +//! 3. Least loaded CPU overall. +//! +//! See `sched_ule.c` in FreeBSD for the original implementation this is based on. +const std = @import("std"); +const rtl = @import("rtl"); +const config = @import("config"); +const b = @import("base"); +const ke = b.ke; +const ki = ke.private; + +comptime { + if (!@hasDecl(ki.impl, "send_resched_ipi")) @compileError("impl must provide send_resched_ipi"); +} + +const runqueues_n = 64; +const interactivity_threshold = 30; +const scaling_factor = 50; +const preempt_threshold = ke.Thread.Priority.realtime; + +pub const RunQueue = struct { + /// Bitmap of non-empty queues. A set bit represents a non-empty queue. + status: u64, + queues: [runqueues_n]rtl.List, +}; + +/// Per-CPU scheduler data. +pub const PerCpu = struct { + /// Current load on this CPU (number of ready threads) + load: std.atomic.Value(usize), + /// Lock over this CPU's queues. + queues_lock: ke.SpinLock, + /// Array of queues from which to pick realtime threads from. + realtime_queue: RunQueue, + /// Calendar queue. + calendar_queue: RunQueue, + /// Queue of idle threads. + idle_queue: rtl.List, + /// The index from which to pick the next thread. + runidx: u8, + /// The index on which the next thread is going to be inserted. + insidx: u8, + /// Used to indicate to this CPU it can steal work + steal_work: bool, +}; + +/// Handle preemption. +/// This is called in DPC dispatch when it notices `next_thread` is set. +pub fn handle_preemption(cpu: *ke.Cpu) void { + const sched_cpu = &cpu.sched_data; + sched_cpu.queues_lock.acquire_no_ipl(); + + const cur = cpu.current_thread.?; + + const next = cpu.next_thread orelse { + sched_cpu.queues_lock.release_no_ipl(); + return; + }; + + cpu.next_thread = null; + + sched_cpu.queues_lock.release_no_ipl(); + cur.lock.acquire_no_ipl(); + + if (cur != cpu.idle_thread and cur.state != .Blocked) { + // Put it back in the queue. + sched_cpu.queues_lock.acquire_no_ipl(); + insert_in_queue(sched_cpu, cur, false); + sched_cpu.queues_lock.release_no_ipl(); + } + + do_switch(cpu, cur, next); + + // cur.lock dropped +} + +/// Called every time slice. +pub fn clock(_: ?*anyopaque) void { + const cpu = ke.curcpu(); + const sched_cpu = &cpu.sched_data; + std.debug.assert(cpu.ipl == .Dispatch); + + const curtd = cpu.current_thread orelse return; + + curtd.lock.acquire_no_ipl(); + defer curtd.lock.release_no_ipl(); + + sched_cpu.queues_lock.acquire_no_ipl(); + defer sched_cpu.queues_lock.release_no_ipl(); + + // Advance the insert index every tick, while keeping a separation of 1 with runidx. + // This ensures fairness. + if (sched_cpu.runidx == sched_cpu.insidx) { + sched_cpu.insidx = (sched_cpu.insidx) % runqueues_n; + } + + if (curtd.priority_class == .Batch) { + // Update interactivity metrics. + curtd.run_time += config.CONFIG_SCHED_TIMESLICE; + + clamp_time(curtd); + recompute_priority(curtd); + } + + if (cpu.next_thread != null) { + // Another thread was already selected for preemption. + return; + } + + // Pick a new thread to run. + const newtd = select_thread(curtd, cpu, false); + + // Ensure the timer gets reloaded. + cpu.start_timer = true; + cpu.next_thread = newtd; +} + +/// Enqueue a thread. +pub fn enqueue(td: *ke.Thread) void { + const ipl = td.lock.acquire(); + const cpu = pick_cpu(td); + enqueue_on_cpu(cpu, td); + td.lock.release(ipl); +} + +/// Block the currently running thread. +pub fn block() void { + const ipl = ke.ipl.raise(.Dispatch); + const td = ke.curcpu().current_thread.?; + + td.lock.acquire_no_ipl(); + + td.state = .Blocked; + td.sleep_start = ke.timecounter.read_time_nano(); + td.runq = null; + + yield(ke.curcpu()); + + // td lock released + ke.ipl.lower(ipl); +} + +/// Unblock a thread. +pub fn unblock(td: *ke.Thread) void { + const ipl = td.lock.acquire(); + + td.sleep_time = (ke.timecounter.read_time_nano() - td.sleep_start) / std.time.ns_per_ms; + + if (td.sleep_time >= config.CONFIG_SCHED_TIMESLICE and .priority_class == .Batch) { + // If we have slept for more than a tick, update interactivity. + clamp_time(td); + recompute_priority(td); + } + + // Enqueue the thread + const cpu = pick_cpu(td); + enqueue_on_cpu(cpu, td); + + td.lock.release(ipl); +} + +/// Initialize a CPU for use by the scheduler. +pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void { + percpu.* = .{ + .insidx = 0, + .runidx = 0, + .calendar_queue = .{ .status = 0, .queues = undefined }, + .realtime_queue = .{ .status = 0, .queues = undefined }, + .queues_lock = .init(), + .idle_queue = undefined, + .steal_work = false, + .load = .init(0), + }; + + percpu.idle_queue.init(); + + for (0..runqueues_n) |i| { + percpu.calendar_queue.queues[i].init(); + } + + for (0..runqueues_n) |i| { + percpu.realtime_queue.queues[i].init(); + } +} + +fn calendar_queue_increment(cpu: *PerCpu) void { + cpu.runidx = (cpu.runidx + 1) % runqueues_n; + + // Ensure insidx is always one ahead of runidx + if (cpu.runidx == cpu.insidx) { + cpu.insidx = (cpu.insidx + 1) % runqueues_n; + } +} + +fn pick_realtime_thread(cpu: *PerCpu, minprio: u8, migrate: bool) ?*ke.Thread { + std.debug.assert(cpu.queues_lock.is_locked()); + + const runq = &cpu.realtime_queue; + + if (runq.status == 0) return null; + + // Search higher priority queues first. + var i: usize = ke.Thread.Priority.max; + + qloop: while (i >= minprio) : (i -= 1) { + const bit = (@as(u64, 1) << @intCast(i)); + + if (runq.status & bit == 0) { + // Queue is empty. + continue; + } + + const curr_queue = &runq.queues[i]; + + std.debug.assert(!curr_queue.is_empty()); + + // Get the first thread of the queue. + var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first()); + + if (migrate) { + // Find the first thread that is not pinned. + while (td.pinned) { + const entry = td.runq_link.next; + + td = @fieldParentPtr("runq_link", entry); + + // Every thread is pinned, go through another queue + if (entry == &curr_queue.head) continue :qloop; + } + } + + // Remove it from the queue. + td.runq_link.remove(); + + if (curr_queue.is_empty()) { + // Queue is now empty, clear its bit. + runq.status &= ~bit; + } + } + + return null; +} + +fn pick_batch_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread { + std.debug.assert(cpu.queues_lock.is_locked()); + const runq = &cpu.calendar_queue; + + if (runq.status == 0) return null; + + // Search all queues, starting from runidx. + var i: usize = cpu.runidx; + var cnt: usize = 0; + + qloop: while (cnt < runqueues_n) : ({ + i = (i + 1) % runqueues_n; + cnt += 1; + }) { + const bit = (@as(u64, 1) << @intCast(i)); + + if (runq.status & bit == 0) { + // Queue is empty. + continue; + } + + const curr_queue = &runq.queues[i]; + + std.debug.assert(!curr_queue.is_empty()); + + // Get the first thread of the queue. + var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first()); + + if (migrate) { + // Find the first thread that is not pinned. + while (td.pinned) { + const entry = td.runq_link.next; + + td = @fieldParentPtr("runq_link", entry); + + // Every thread is pinned, go through another queue + if (entry == &curr_queue.head) continue :qloop; + } + } + + // Remove it from the queue. + td.runq_link.remove(); + + if (curr_queue.is_empty()) { + // Queue is now empty, clear its bit and increment runidx. + runq.status &= ~bit; + + if (!migrate) { + calendar_queue_increment(cpu); + } + } + + return td; + } + + return null; +} + +fn pick_idle_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread { + std.debug.assert(cpu.queues_lock.is_locked()); + if (cpu.idle_queue.is_empty()) return null; + + // Just get the first thread from the queue. + const td: *ke.Thread = @fieldParentPtr("runq_link", cpu.idle_queue.first()); + + if (!migrate and td.pinned) { + // We can safely remove it. + td.runq_link.remove(); + } + + return td; +} + +fn insert_in_list(list: *rtl.List, link: *rtl.List.Entry, head: bool) void { + if (head) list.insert_head(link) else list.insert_tail(link); +} + +// Insert a thread into its respective queue. +fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, head: bool) void { + std.debug.assert(td.lock.is_locked()); + std.debug.assert(cpu.queues_lock.is_locked()); + + if (td.priority_class == .Realtime or td.interactive) { + // Insert in realtime queue. + cpu.realtime_queue.status |= + (@as(u64, 1) << @intCast(td.priority)); + + insert_in_list(&cpu.realtime_queue.queues[td.priority], &td.runq_link, head); + td.runq = &cpu.realtime_queue; + td.runq_idx = @intCast(td.priority); + } else if (td.priority_class == .Batch) { + // Insert in calendar queue. + // The insertion index is determined by insidx and the the priority of the thread, + // higher priority threads will be put closer to insidx, which ensures that they are ran more frequently. + const idx = (cpu.insidx + (ke.Thread.Priority.high_batch - td.priority)) % runqueues_n; + + td.runq_idx = @intCast(idx); + cpu.calendar_queue.status |= (@as(u64, 1) << @intCast(idx)); + + insert_in_list(&cpu.calendar_queue.queues[idx], &td.runq_link, head); + td.runq = &cpu.calendar_queue; + } else { + // Insert in idle queue. + insert_in_list(&cpu.idle_queue, &td.runq_link, head); + td.runq = null; + td.runq_idx = 0; + } + + td.state = .Ready; + _ = cpu.load.fetchAdd(1, .monotonic); +} + +// Remove a thread from a queue. +fn remove_from_queue(cpu: *PerCpu, td: *ke.Thread) void { + std.debug.assert(td.lock.is_locked()); + std.debug.assert(cpu.queues_lock.is_locked()); + + // Decrease load. + cpu.load.fetchSub(1, .monotonic); + + // Remove the thread from its queue. + td.runq_link.remove(); + + if (td.runq) |rq| { + if (rq.queues[td.runq_idx].is_empty()) { + // Clear the bit. + rq.status &= ~(@as(u64, 1) << @intCast(td.runq_idx)); + } + } +} + +// Update recent history for a thread. +fn clamp_time(td: *ke.Thread) void { + const max = 5 * std.time.ms_per_s; // 5 seconds + const sum = td.run_time + td.sleep_time; + + if (sum < max) return; + + if (sum > max * 2) { + // History is way out of range (>10s), hard reset. + // Preserve the dominant side to avoid flipping interactivity classification. + if (td.run_time > td.sleep_time) { + td.run_time = max; + td.sleep_time = 1; + } else { + td.sleep_time = max; + td.run_time = 1; + } + + return; + } + + if (sum > ((max / 5) * 6)) { + // History is moderately out of range (>6s), halve both values. + // Keeps the sleep/run ratio intact while pulling the sum back in range. + td.run_time /= 2; + td.sleep_time /= 2; + return; + } + + // History is slightly out of range (5s-6s), gentle 20% decay. + // Gradually ages out old history without disturbing the ratio. + td.run_time = (td.run_time / 5) * 4; + td.sleep_time = (td.sleep_time / 5) * 4; +} + +// Compute the interactivity score for a thread. +fn interactive_score(td: *ke.Thread) usize { + var div: usize = undefined; + var score: usize = undefined; + + // Calculate the interactivity penalty. + if (td.sleep_time > td.run_time) { + div = @max(1, td.sleep_time / scaling_factor); + score = td.run_time / div; + } else if (td.run_time > td.sleep_time) { + div = @max(1, td.run_time / scaling_factor); + score = (scaling_factor + scaling_factor - (td.sleep_time / div)); + } else if (td.run_time != 0) { + score = scaling_factor; + } else return 0; + + // Add niceness values to the penalty, this makes it easier for threads with + // lower nice values (higher priority) to be considered interactive. + var signed_score: isize = @intCast(score); + signed_score += td.nice; + score = @intCast(signed_score); + + return score; +} + +// Recompute the priority for a thread. +fn recompute_priority(td: *ke.Thread) void { + + // Priority is computed only for time-shared (batch) thread based on + // interactivity and nice. If the thread is determined interactive, it is + // effectively promoted to real-time with a lower priority than actual + // real-time threads. If the thread is determined non-interactive, + // priority is calculated based on recent CPU usage and nice. + const score = interactive_score(td); + + if (score < interactivity_threshold) { + // Choose a priority based on score, the lower the score, + // the higher the priority it will be. + // This is a simple formula I came up with that is probably good enough. + td.priority = @intCast(ke.Thread.Priority.interactive + ((interactivity_threshold - score) / 4)); + td.interactive = true; + + return; + } + + // Thread is not interactive, compute priority from recent CPU usage. + // cpu_pri_off is the fraction of the history window spent running, + // scaled to [0, batch_range). A thread that spent all its time running + // gets cpu_pri_off = batch_range - 1 (lowest priority). + // A thread that barely ran gets cpu_pri_off = 0 (highest batch priority). + // This is a pretty straightforward formula, which /should/ be good enough. + const cpu_range = ke.Thread.Priority.batch_range; + const window = if (td.run_time + td.sleep_time == 0) @as(u64, 1) else td.run_time + td.sleep_time; + const cpu_pri_off: u8 = @intCast((td.run_time * (cpu_range - 1)) / window); + + // nice offset: negative nice increases priority value, positive decreases it. + // Clamped to half the range to avoid pushing priority out of bounds before clamping. + const nice_off = std.math.clamp( + -@as(i32, td.nice), + -@as(i32, cpu_range / 2), + @as(i32, cpu_range / 2), + ); + + const raw = @as(i32, ke.Thread.Priority.low_batch) + + @as(i32, cpu_pri_off) + + nice_off; + + td.priority = @intCast(std.math.clamp( + raw, + ke.Thread.Priority.low_batch, + ke.Thread.Priority.high_batch, + )); + td.interactive = false; +} + +// Return whether or not a thread with given priority and interactivity should get preempted by `td`. +fn should_prio_preempt(td: *ke.Thread, other_prio: u8, other_interactive: bool) bool { + const class = ke.Thread.Priority.class_from_prio(other_prio); + + if (class == .Idle) { + return true; + } + + if (class == .Batch and !other_interactive and td.interactive) { + // Interactive threads always preempt batch non-interactive ones. + return true; + } + + // Preempt if the priority exceeds the preemption threshold (i.e the thread is real-time) + // or if the thread is interactive. + // Interactive threads may still preempt each other based on interactivity. + if ((td.priority >= preempt_threshold or td.interactive) and td.priority > other_prio) { + return true; + } + + return false; +} + +// Return whether or not `td` should preempt `cur`. +// This is called with the current CPU's queue lock held. +fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool { + return should_prio_preempt(td, cur.priority, cur.interactive); +} + +// Return whether or not `td` should preempt the thread running on `cpu`. +// This is done in a lock-free manner. +fn should_preempt_cpu(td: *ke.Thread, cpu: *ke.Cpu) bool { + const status = cpu.current_thread_status.load(.monotonic); + return should_prio_preempt(td, status.priority, status.interactive); +} + +// Enqueue a thread on a CPU. +fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void { + var c = &cpu.sched_data; + + c.queues_lock.acquire_no_ipl(); + + defer c.queues_lock.release_no_ipl(); + + if (cpu.current_thread == null or should_preempt(td, cpu.current_thread.?)) { + // We can preempt the current thread, first check whether or not there + // is already a next thread selected, and if we can preempt it. + + const next = cpu.next_thread; + var can_preempt = true; + + if (next) |n| { + if (should_preempt(td, n)) { + // This thread can preempt the next thread, + // put the next thread back into a queue as the first element. + insert_in_queue(c, n, true); + } else { + // The next thread is higher priority, we can't preempt it. + can_preempt = false; + } + } + + if (can_preempt) { + // Preempt the CPU. + cpu.next_thread = td; + cpu.preemption_reason = .HigherPriority; + + ki.ipl.set_softint_pending(cpu, .Dispatch); + + if (cpu != ke.curcpu()) { + // Send an IPI. + ki.impl.send_resched_ipi(cpu); + } + + // queues_lock dropped + return; + } + } + + // Normal enqueue path. + // Add the thread to its appropriate queue. + insert_in_queue(c, td, false); + + if (cpu.resched_timer.state.load(.monotonic) == .Stopped) { + cpu.start_timer = true; + ki.ipl.set_softint_pending(cpu, .Dispatch); + } + + // queues_lock dropped +} + +// Find a suitable CPU to run the thread. +fn pick_cpu(td: *ke.Thread) *ke.Cpu { + // CPU selection policy in order: + // 1. Pick the last CPU the thread ran on if it would run immediately or if it's pinned. + // 2. Pick the least loaded CPU on which the thread can run immediately. + // 3. Pick the least loaded CPU overall. + + if (td.last_cpu) |cpu| { + if (should_preempt_cpu(td, cpu) or td.pinned) return cpu; + } + + // Check all CPUs, keeping track of the least loaded one overall and the least loaded preemptible one. + var least: ?*ke.Cpu = null; + var least_preempt: ?*ke.Cpu = null; + + for (0..ke.ncpus) |i| { + const cpu = ke.cpus[i]; + + if (least == null or cpu.sched_data.load.load(.monotonic) < least.?.sched_data.load.load(.monotonic)) { + least = cpu; + } + + if (should_preempt_cpu(td, cpu)) { + if (least_preempt == null or cpu.sched_data.load.load(.monotonic) < least_preempt.?.sched_data.load.load(.monotonic)) { + least_preempt = cpu; + } + } + } + + // If we found a preemptible CPU, pick that one. Otherwise pick the least loaded one. + return least_preempt orelse least.?; +} + +// Find a thread to preempt `cur` with. +fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread { + const sched_cpu = &cpu.sched_data; + + // First try to pick from the realtime queues. + if (pick_realtime_thread(sched_cpu, if (cur) |c| c.priority else 0, migrate)) |td| { + _ = sched_cpu.load.fetchSub(1, .monotonic); + return td; + } + + // Nothing lower can preempt this thread. + if (cur != null and (cur.?.priority_class == .Realtime or cur.?.interactive)) { + return null; + } + + // Then try picking a batch thread. + if (pick_batch_thread(sched_cpu, migrate)) |td| { + _ = sched_cpu.load.fetchSub(1, .monotonic); + return td; + } + + if (cur != null and cur.?.priority_class == .Idle) { + // Idle threads don't preempt each other + return null; + } + + // Finally try picking from the idle queue. + if (pick_idle_thread(sched_cpu, migrate)) |td| { + _ = sched_cpu.load.fetchSub(1, .monotonic); + return td; + } + + // Nothing to run. + return null; +} + +// Execute a context switch on `cpu`. +fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void { + next.state = .Running; + cpu.current_thread = next; + next.last_cpu = cpu; + + // Stash thread status. + cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic); + + // Do machine-dependent switch. + cur.context.switch_to(&next.context); +} + +// Yield on `cpu` +fn yield(cpu: *ke.Cpu) void { + const sched_cpu = &cpu.sched_data; + + sched_cpu.queues_lock.acquire_no_ipl(); + + const cur = cpu.current_thread; + var next = cpu.next_thread; + + if (next) { + cpu.next_thread = null; + } else { + // Pick a new thread to run. + next = select_thread(null, cpu, false); + } + + if (next == null) { + // Nothing to run, go idle. + next = cpu.idle_thread; + } + + sched_cpu.queues_lock.release_no_ipl(); + + // Switch into the thread. + do_switch(cpu, cur, next); + + // cur lock dropped +} diff --git a/src/kern/spinlock.zig b/src/kern/spinlock.zig new file mode 100644 index 0000000..1e5eb5e --- /dev/null +++ b/src/kern/spinlock.zig @@ -0,0 +1,60 @@ +const std = @import("std"); +const b = @import("base"); +const ke = b.ke; + +/// Simple Spin lock implementation. +pub const SpinLock = struct { + locked: u8, + + pub fn init() SpinLock { + return .{ + .locked = 0, + }; + } + + /// Acquire the lock at IPL `ipl`. + pub fn acquire_at(self: *SpinLock, ipl: ke.Ipl) ke.Ipl { + const old_ipl = ke.ipl.raise(ipl); + + self.acquire_no_ipl(); + + return old_ipl; + } + + /// Acquire the lock at IPL `.Dispatch`. + pub fn acquire(self: *SpinLock) ke.Ipl { + return self.acquire_at(.Dispatch); + } + + /// Release the lock at IPL `ipl`. + pub fn release(self: *SpinLock, ipl: ke.Ipl) void { + ke.ipl.lower(ipl); + self.release_no_ipl(); + } + + /// Acquire the lock without changing the IPL. + pub fn acquire_no_ipl(self: *SpinLock) void { + while (true) { + if (@cmpxchgWeak(u8, &self.locked, 0, 1, .acquire, .monotonic) == null) + return; + + while (@atomicLoad(u8, &self.locked, .monotonic) != 0) { + std.atomic.spinLoopHint(); + } + } + } + + /// Release the lock without changing the IPL. + pub fn release_no_ipl(self: *SpinLock) void { + @atomicStore(u8, &self.locked, 0, .release); + } + + /// Try to acquire the lock. Return true if lock was acquired. + pub fn try_acquire(self: *SpinLock) bool { + return @cmpxchgStrong(u8, &self.locked, 0, 1, .acquire, .monotonic) == null; + } + + pub fn is_locked(self: *SpinLock) bool { + return @atomicLoad(u8, &self.locked, .monotonic) == 1; + } +}; diff --git a/src/kern/thread.zig b/src/kern/thread.zig new file mode 100644 index 0000000..e70546e --- /dev/null +++ b/src/kern/thread.zig @@ -0,0 +1,128 @@ +const std = @import("std"); +const b = @import("base"); +const rtl = @import("rtl"); + +const ke = b.ke; +const ki = ke.private; + +/// Structure representing a kernel thread. +pub const Thread = struct { + /// Thread priority classes. + /// Realtime: Highest priority, aggressively scheduled. + /// Batch: Normal priority. + /// Idle: For idle and background threads. + pub const Priority = struct { + pub const Class = enum(u8) { + Realtime, + Batch, + Idle, + }; + + pub const idle = 0; + pub const low_batch = 1; + pub const high_batch = 32; + pub const batch_range = high_batch - low_batch + 1; + pub const realtime = 33; + pub const max = 63; + /// In the mid-range of batch threads + pub const default = 16; + /// Interactive threads have a slightly higher priority + pub const interactive = 20; + + pub fn class_from_prio(prio: u8) Class { + if (prio >= realtime and prio <= max) return .Realtime; + if (prio >= low_batch and prio <= high_batch) return .Batch; + if (prio == idle) return .Idle; + + unreachable; + } + }; + + /// Thread state. + pub const State = enum(u8) { + /// The thread is ready to run. + Ready, + /// The thread is currently running. + Running, + /// The thread is currently sleeping. + Blocked, + /// The thread has exited and is waiting to be reaped. + Zombie, + /// The thread has exited. + Terminated, + }; + /// Implementation-dependent context. + context: ki.impl.ThreadContext, + /// Thread lock. + lock: ke.SpinLock, + /// Niceness value. + nice: i8, + /// Priority value of the thread. + priority: u8, + /// Priority class of the thread. + priority_class: Priority.Class, + /// When the thread started sleeping. + sleep_start: u64, + /// Ticks spent voluntarily sleeping recently. + sleep_time: u64, + /// Ticks spent running recently. + run_time: u64, + /// Whether the thread is interactive or not. + interactive: bool, + /// Whether the thread is pinned to this CPU. + /// If it is pinned, then it can't be moved across another CPU. + pinned: bool, + /// Current state of the thread. + state: State, + /// Linkage into a scheduler run queue. + runq_link: rtl.List.Entry, + /// Last CPU this thread ran on. + last_cpu: ?*ke.Cpu, + + /// Run queue this thread is in + runq: ?*ki.sched.RunQueue, + /// Index into the run queue this thread is currently in. + /// Only valid when `runq` is non-null. + runq_idx: u8, + /// Current wait status. + wait_status: std.atomic.Value(ki.wait.Status), + waitblocks: [4]ki.wait.WaitBlock, + + /// Timer used for timeouts. + timer: ke.Timer, + + /// Initialize a thread. + /// - `stack`: Address of the **base** of the stack on which the initial context for the thread is built + /// - `stack_size`: Size of the stack + /// - `entry`: Entry point of the thread + /// - `arg`: Extraneous argument to be passed to `entry` + pub fn init(thread: *Thread, stack: b.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) void { + thread.* = .{ + .context = .init(stack, stack_size, entry, arg), + .lock = ke.SpinLock.init(), + .nice = 0, + .priority = Thread.Priority.default, + .priority_class = Thread.Priority.Class.Batch, + .sleep_start = 0, + .sleep_time = 0, + .run_time = 0, + .interactive = false, + .pinned = false, + .state = .Ready, + .runq_link = .{}, + .last_cpu = null, + .runq = null, + .runq_idx = 0, + .wait_status = .init(.InProgress), + .waitblocks = undefined, + .timer = undefined, + }; + + thread.timer.init(); + } +}; + +comptime { + if (!@hasDecl(ki.impl, "ThreadContext")) @compileError("impl must provide ThreadContext"); + if (!@hasDecl(ki.impl.ThreadContext, "init")) @compileError("ThreadContext must have init()"); +} diff --git a/src/kern/timecounter.zig b/src/kern/timecounter.zig new file mode 100644 index 0000000..15cbf2a --- /dev/null +++ b/src/kern/timecounter.zig @@ -0,0 +1,143 @@ +const std = @import("std"); +const rtl = @import("rtl"); +const b = @import("base"); +const ke = b.ke; +const ki = ke.private; + +/// TimeCounter structure. +/// Represents a time-keeping source upon which the kernel can rely. +pub const TimeCounter = struct { + /// Callback that reads the current count. + read_count: *const fn () u64, + /// Frequency of the timer in Hz. + frequency: u64, + /// Name of the timer. + name: []const u8, + /// Quality of the timer. + /// Used to choose between two TimeCounters. + quality: i16, + /// Mask to apply to the counter value. + mask: u64, + /// Binary scaling exponent for fast ns conversion (0 if unused). + /// See `ticks_to_ns`. + p: u64, + /// Scaled nanoseconds-per-tick multiplier for fast ns conversion (0 if unused). + /// See `ticks_to_ns`. + n: u64, +}; + +var best_tc: ?*TimeCounter = null; + +const TimeState = struct { + // Counter value at the time the current TimeCounter was selected. + // Used as the baseline for elapsed time calculation. + initial_count: u64, + // Accumulated nanosecond offset from previous TimeCounters and overflow events. + // Added to the current counter's contribution in `read_time_nano`. + offset: u64, +}; + +// SeqLock-protected time state. +// Avoids a race on `initial_count` and `offset` with `update_overflow`. +var state = rtl.SeqLock(TimeState).init(.{ + .initial_count = 0, + .offset = 0, +}); + +// Convert a TimeCounter's count to nanoseconds. +// For 64-bit counters (`mask == maxInt(u64)`), uses a fast multiply-shift path +// via precomputed `p` and `n` to avoid 128-bit division at runtime. +// Falls back to a division-based conversion for narrower counters. +fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 { + if (tc.p != 0) { + const val: u128 = @as(u128, @intCast(count)) * tc.n; + return @intCast(val >> @intCast(tc.p)); + } + + return (count * std.time.ns_per_s) / tc.frequency; +} + +/// Register a time-keeping source. +/// The TimeCounter will be used if it is the highest quality one available. +/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided. +pub fn register(tc: *TimeCounter) void { + // Precompute scaling factors for the fast multiply-shift path. + // This is meant to avoid 128-bit division and only use multiplication for 64-bit counters. + if (tc.mask == std.math.maxInt(u64)) { + tc.p = std.math.log2_int_ceil(u64, tc.frequency); + tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency; + } else { + tc.p = 0; + tc.n = 0; + } + + // If the best TimeCounter changed, pass the baton: + // snapshot the old counter's elapsed time into `offset` and + // reset `initial_count` to the new counter's current value. + if (best_tc == null or tc.quality > best_tc.?.quality) { + const old_best = best_tc; + best_tc = tc; + + if (old_best) |old| { + const old_count = old.read_count() & old.mask; + state.raw.offset = ticks_to_ns(old, old_count); + } + + state.raw.initial_count = tc.read_count() & tc.mask; + } +} + +/// Return the time elapsed since boot in nanoseconds. +pub fn read_time_nano() b.Nanoseconds { + const tc = best_tc orelse return 0; + + const s = state.load(); + const curr_count = tc.read_count() & tc.mask; + const elapsed = (curr_count - s.initial_count) & tc.mask; + return s.offset + ticks_to_ns(tc, elapsed); +} + +/// Return the best TimeCounter. +pub fn best() ?*TimeCounter { + return best_tc; +} + +/// Do a busy sleep of `ns` nanoseconds using TimeCounter. +pub fn sleep(ns: b.Nanoseconds) void { + const start = read_time_nano(); + + while (true) { + var cur = read_time_nano(); + + if (cur < start) continue; + + if (cur - start >= ns) { + cur = read_time_nano(); + + // Check again as we may have overflowed and read a bogus value.. + // This is kinda hacky but oh well. + if (cur - start >= ns) { + break; + } + } + + std.atomic.spinLoopHint(); + } +} + +/// Called periodically to handle counter overflows. +pub fn update_overflow() void { + const tc = best_tc orelse return; + const s = state.load(); + const cur_count = tc.read_count() & tc.mask; + + if (s.initial_count > cur_count) { + // Counter has wrapped, use wrapping arithmetic to figure out by how much, + // then fold it into the offset and reset `initial_count`. + const overflowed_by = (cur_count -% s.initial_count) & tc.mask; + state.store(.{ + .initial_count = cur_count, + .offset = s.offset + ticks_to_ns(tc, overflowed_by), + }); + } +} diff --git a/src/kern/timer.zig b/src/kern/timer.zig new file mode 100644 index 0000000..e703b1f --- /dev/null +++ b/src/kern/timer.zig @@ -0,0 +1,175 @@ +//! Timer object implementation. +//! Timer objects are useful when one wants to wait a fixed amount of time for an event to occur. +const std = @import("std"); +const rtl = @import("rtl"); +const b = @import("base"); +const ke = b.ke; +const ki = ke.private; +const pl = b.pl; + +pub const Timer = struct { + pub const State = enum(u8) { + /// The timer is currently being handled. + Running, + /// The timer is currently enqueued. + Pending, + /// The timer was stopped. + Stopped, + }; + + hdr: ki.wait.DispatchHeader, + + /// Timer state. + state: std.atomic.Value(State), + /// When the timer is bound to expire. + deadline: b.Nanoseconds, + /// Attached DPC. + dpc: ?*ke.Dpc, + /// Intrusive pairing heap node. + node: rtl.pairing_heap.Node, + /// CPU this timer is enqueued on. + cpu: ?*ke.Cpu, + + pub fn init(self: *Timer) void { + self.* = .{ + .hdr = undefined, + .state = .init(.Stopped), + .deadline = 0, + .dpc = null, + .node = .{}, + .cpu = null, + }; + + self.hdr.init(.Notification); + } +}; + +/// Start a timer with an expiration time. +/// A DPC that will be enqueued upon expiration can be passed. +pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void { + const ipl = timer.hdr.lock.acquire(); + defer timer.hdr.lock.release(ipl); + + const cpu = ke.curcpu(); + + if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) { + // Could not set the timer. + return; + } + + cpu.timers_lock.acquire_no_ipl(); + defer cpu.timers_lock.release_no_ipl(); + + // Initialize the timer. + timer.deadline = ke.timecounter.read_time_nano() + time; + + timer.cpu = cpu; + timer.dpc = dpc; + timer.hdr.signaled = 0; + + cpu.timers_heap.insert(&timer.node); + + if (cpu.timers_heap.root == &timer.node) { + // This is the earliest timer to expire, arm the hardware timer. + pl.arm_timer(time); + } + + // Locks dropped +} + +/// Cancel a timer. +pub fn cancel(timer: *Timer) void { + const old_state = timer.state.load(.monotonic); + + if (old_state == .Stopped or old_state == .Running) { + // Already stopped or currently executing. + return; + } + + const ipl = timer.hdr.lock.acquire(); + defer timer.hdr.lock.release(ipl); + + if (timer.state.cmpxchgStrong(.Pending, .Stopped, .acquire, .monotonic) != null) { + // Something changed, it probably expired. + return; + } + + if (timer.cpu) |cpu| { + // Dequeue the timer. + cpu.timers_lock.acquire_no_ipl(); + + cpu.timers_heap.remove(&timer.node); + timer.cpu = null; + + cpu.timers_lock.release_no_ipl(); + } + + // Lock dropped +} + +/// Compare two timers. +pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math.Order { + const timer_a: *Timer = @fieldParentPtr("node", a); + const timer_b: *Timer = @fieldParentPtr("node", b_); + + return std.math.order(timer_a.deadline, timer_b.deadline); +} + +/// Called in a DPC when a timer has expired. +pub fn handle_expiry(_: ?*anyopaque) void { + const cpu = ke.curcpu(); + std.debug.assert(cpu.ipl == .Dispatch); + + while (true) { + const curtime = ke.timecounter.read_time_nano(); + + cpu.timers_lock.acquire_no_ipl(); + + // Get the timer that expires the soonest. + const timer_node = cpu.timers_heap.root orelse { + cpu.timers_lock.release_no_ipl(); + return; + }; + + const timer: *Timer = @fieldParentPtr("node", timer_node); + + // If the timer expires more than 1ms in the future, consider it not yet due. + // Sub-millisecond differences are close enough to expire immediately. + if (timer.deadline > curtime and timer.deadline - curtime > std.time.ns_per_ms) { + const next = timer; + cpu.timers_lock.release_no_ipl(); + + // Re-check in case the timer expired while we were here. + const now = ke.timecounter.read_time_nano(); + if (next.deadline <= now or next.deadline - now <= std.time.ns_per_ms) { + continue; // expired in the meantime, loop again + } + + std.debug.assert(next.deadline > now); + pl.arm_timer(next.deadline - now); + return; + } + + _ = cpu.timers_heap.pop(); + cpu.timers_lock.release_no_ipl(); + + if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) { + // Timer must've been canceled. + continue; + } + + timer.hdr.lock.acquire_no_ipl(); + + // Set signaled. + timer.hdr.signaled = 1; + timer.cpu = null; + + // Wake whomever was waiting on the timer. + ki.wait.satisfy_wait(&timer.hdr); + + timer.state.store(.Stopped, .release); + + if (timer.dpc) |dpc| ke.dpc.enqueue(dpc, null); + timer.hdr.lock.release_no_ipl(); + } +} diff --git a/src/kern/um/impl.zig b/src/kern/um/impl.zig new file mode 100644 index 0000000..f9b6fd0 --- /dev/null +++ b/src/kern/um/impl.zig @@ -0,0 +1,106 @@ +const std = @import("std"); +const b = @import("base"); +const c = b.pl.impl.c; +const ke = b.ke; +const ki = ke.private; + +pub const Cpu = struct { pthread: c.pthread_t }; + +/// Return the current CPU. +pub fn curcpu() *ke.Cpu { + return b.pl.impl.my_cpu; +} + +fn set_signals(enabled: bool) bool { + var newmask: c.sigset_t = undefined; + var oldmask: c.sigset_t = undefined; + + _ = c.sigfillset(&newmask); + + const how = if (enabled) c.SIG_BLOCK else c.SIG_UNBLOCK; + + if (c.pthread_sigmask(how, &newmask, &oldmask) != 0) { + ke.panic("pthread_sigmask failed", .{}); + } + + return std.mem.eql(u8, std.mem.asBytes(&newmask), std.mem.asBytes(&oldmask)); +} + +/// Set the hardware IPL. +pub fn set_hardware_ipl(ipl: ke.Ipl) void { + _ = set_signals(@intFromEnum(ipl) > @intFromEnum(ke.Ipl.get_max_software())); +} + +/// Disable interrupts and return the previous state. +pub fn disable_interrupts() bool { + return set_signals(false); +} + +/// Enable interrupts and return the previous state. +pub fn enable_interrupts() bool { + return set_signals(true); +} + +/// Restore an interrupt state. +pub fn restore_interrupts(val: bool) void { + if (val) { + _ = enable_interrupts(); + } else { + _ = disable_interrupts(); + } +} + +pub fn send_resched_ipi(cpu: *ke.Cpu) void { + _ = c.pthread_kill(cpu.impl.pthread, c.SIGUSR1); +} + +pub const ThreadContext = struct { + ucontext: c.ucontext_t, + + fn real_entry(entry_int: c_ulong, arg_int: c_ulong) callconv(.c) void { + const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_int); + const arg: ?*anyopaque = @ptrFromInt(arg_int); + ke.ipl.lower(.Zero); + entry(arg); + } + + pub fn init(stack: b.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) ThreadContext { + var new: ThreadContext = .{ .ucontext = undefined }; + + if (c.getcontext(&new.ucontext) == -1) { + ke.panic("getcontext() failed", .{}); + } + + new.ucontext.uc_stack.ss_sp = @ptrFromInt(stack); + new.ucontext.uc_stack.ss_size = stack_size; + + // No successor + new.ucontext.uc_link = null; + + const entry_int: usize = @intFromPtr(entry); + const arg_int: usize = @intFromPtr(arg); + + c.makecontext( + &new.ucontext, + @ptrCast(&real_entry), + 2, + @as(c_ulong, entry_int), + @as(c_ulong, arg_int), + ); + return new; + } + + pub fn load(self: *ThreadContext) void { + _ = c.setcontext(&self.ucontext); + } + + pub fn switch_to(self: *ThreadContext, new: *ThreadContext) void { + const thread: *ke.Thread = @fieldParentPtr("context", self); + + thread.lock.release_no_ipl(); + + if (c.swapcontext(&self.ucontext, &new.ucontext) == -1) { + ke.panic("swapcontext() failed", .{}); + } + } +}; diff --git a/src/kern/wait.zig b/src/kern/wait.zig new file mode 100644 index 0000000..5a9968b --- /dev/null +++ b/src/kern/wait.zig @@ -0,0 +1,235 @@ +//! Implementation of generic wait mechanisms. +//! This is mostly based on the work by Arun Kishan on Windows 7. +//! See more here: https://youtu.be/OAAiOEQhsK0 +const rtl = @import("rtl"); +const std = @import("std"); +const b = @import("base"); +const ke = b.ke; + +/// Header for waitable objects. +/// This must be added to any structured which is considered waitable. +pub const DispatchHeader = struct { + pub const Kind = enum { + /// When signaled, `signaled` is kept high and all waiters are woken up. + Notification, + /// When signaled, `signaled` is decreased until 0, and a single waiter is woken up. + Synchronization, + }; + + /// List of WaitBlocks. + waitblocks: rtl.List, + /// Kind of object. + kind: Kind, + /// Lock protecting the object. + lock: ke.SpinLock, + /// Signaled count. + signaled: u32, + + /// Initialize a waitable object. + pub fn init(obj: *DispatchHeader, kind: Kind) void { + obj.kind = kind; + obj.lock = ke.SpinLock.init(); + obj.signaled = 0; + obj.waitblocks.init(); + } + + fn consume(self: *DispatchHeader) void { + switch (self.kind) { + .Synchronization => self.signaled -= 1, + .Notification => { + // Object remains signaled. + }, + } + } +}; + +pub const WaitBlock = struct { + pub const Status = enum(u8) { + /// WaitBlock is linked to an object as part of a thread that is waiting. + Active, + /// The wait associated with this WaitBlock has been satisfied (or timed out). + Inactive, + /// A signal was delivered to the wait (this could be before it was committed). + Signaled, + }; + /// List linkage. + link: rtl.List.Entry, + /// Object being waited on. + object: *DispatchHeader, + /// Thread that owns the WaitBlock. + thread: *ke.Thread, + /// Status of the WaitBlock. + status: WaitBlock.Status, +}; + +/// Status of a wait operation. +pub const Status = enum(u8) { + /// Wait is currently being processed. + InProgress, + /// Wait has been committed and is waiting for the object to be signaled. + Committed, + /// Wait has been satisfied (or timed out). + Satisfied, +}; + +/// Wait for the provided object to be signaled. +pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize { + return wait_any(object, timeout, null); +} + +/// Wait for any of the provided objects to be signaled. +/// Returns the index of the object that was signaled, or an error. +/// If `timeout` is not provided, it will wait indefinitely. +/// If `waitblocks` is specified, then the wait will use those waitblocks for the operation. +/// Note that if `timeout` is provided, then one additional waitblock must be allocated. +pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize { + const ipl = ke.ipl.raise(.Dispatch); + defer ke.ipl.lower(ipl); + const curtd = ke.curcpu().current_thread; + const obj_count = objects.len; + const has_timeout = timeout != null; + const blocks = waitblocks orelse blk: { + std.debug.assert(obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout)); + break :blk &curtd.waitblocks; + }; + + const total_count = obj_count + @intFromBool(has_timeout); + const timer_i = obj_count; + const timer = &curtd.timer; + const timer_block = &curtd.waitblocks[3]; + + var satisfier: ?usize = null; + curtd.wait_status.store(.InProgress, .acquire); + + for (0..total_count) |i| { + const is_timer = has_timeout and i == timer_i; + var obj = if (is_timer) &timer.hdr else objects[i]; + var wb = if (is_timer) timer_block else &blocks[i]; + + obj.lock.acquire_no_ipl(); + defer obj.lock.release_no_ipl(); + + if (obj.signaled > 0) { + // Object was already signaled. Try consuming it. + if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) { + obj.consume(); + } + // We have already been satisfied in the meantime, abort. + satisfier = i; + break; + } + + wb.object = obj; + wb.thread = curtd; + wb.status = .Active; + // We are not satisfied yet, so add a waitblock to the object's waitblocks + obj.waitblocks.insert_tail(&wb.link); + } + + // Wait was already satisfied, back out. + if (satisfier != null or (has_timeout and timeout.? == 0)) { + std.debug.assert(curtd.wait_status.load(.monotonic) == .Satisfied); + + // Remove any wait block we might've installed + for (0..satisfier) |i| { + const is_timer = has_timeout and i == timer_i; + var obj = if (is_timer) &timer.hdr else objects[i]; + var wb = if (is_timer) timer_block else &blocks[i]; + + obj.lock.acquire_no_ipl(); + wb.status = .Inactive; + wb.link.remove(); + obj.lock.release_no_ipl(); + } + + return satisfier orelse error.Timeout; + } + + if (has_timeout) { + ke.timer.set(timer, timeout.?, null); + } + + // Now try committing the wait. + // While we're trying to commit the wait, the object locks have been released, and the state could therefore change. + // We need to re-check the state of the wait before actually blocking. + if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acquire, .monotonic) == null) { + // We're good, now actually block. + ke.sched.block(); + } + + // We're back! + // Stop the timer if it was set. + if (has_timeout) { + ke.timer.cancel(timer); + } + + // Find the object that satisfied us. + for (0..total_count) |i| { + const is_timer = has_timeout and i == timer_i; + var obj = if (is_timer) &timer.hdr else objects[i]; + var wb = if (is_timer) timer_block else &blocks[i]; + + obj.lock.acquire_no_ipl(); + if (wb.status == .Active) { + // Waitblock is still active, remove it from the list. + wb.link.remove(); + } + if (wb.status == .Signaled or wb.status == .SignaledInProgress) { + // This waitblock was signaled, it must be the one that satisfied the wait. + std.debug.assert(satisfier == null); + satisfier = i; + } + // Ignore inactive waitblocks + obj.lock.release_no_ipl(); + } + + const final_sat = satisfier orelse return error.Timeout; + return if (has_timeout and final_sat == timer_i) error.Timeout else final_sat; +} + +/// Satisfy a wait on an object. +pub fn satisfy_wait(obj: *DispatchHeader) void { + const all = obj.kind == .Notification; + + // Go through (potentially) all wait blocks and try to satisfy them. + while (!obj.waitblocks.is_empty() and obj.signaled > 0) { + // Get the first waitblock from the list. + var wb: *WaitBlock = @fieldParentPtr("link", obj.waitblocks.first()); + var td = wb.thread; + + // Remove it. + wb.link.remove(); + + // Three cases may occur here: + // 1. The wait was still preparing (status == .InProgress) and we interrupted it. + // 2. The wait was already committed (status == .Committed) and we satisfied it. + // 3. The wait was already satisfied by another object (status == .Satisfied). + + // 1. + if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) { + // We interrupted the wait while it was being prepared. + wb.status = .Signaled; + obj.consume(); + } + + // 2. + else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acquire, .monotonic) == null) { + // We interrupted the wait while it was committed. + // Wake the thread. + wb.status = .Signaled; + obj.consume(); + ke.sched.unblock(td); + } + + // 3. + else if (td.wait_status.load(.acquire) == .Satisfied) { + // Someone else satisfied the wait, deactivate the wait block. + wb.status = .Inactive; + continue; + } + + if (!all) { + break; + } + } +} diff --git a/src/limine.zig b/src/limine.zig new file mode 100644 index 0000000..664f6c6 --- /dev/null +++ b/src/limine.zig @@ -0,0 +1,70 @@ +pub const common_magic: [2]u64 = .{ + 0xc7b1dd30df4c8b88, + 0x0a82e883a194f07b, +}; + +pub const requests_start_marker: [4]u64 = .{ + 0xf6b8f4b39de7d1ae, + 0xfab91a6940fcb9cf, + 0x785c6ed015d3e316, + 0x181e920a7852b9d9, +}; + +pub const requests_end_marker: [2]u64 = .{ + 0xadc0e0531bb10d03, + 0x9572709f31764c62, +}; + +pub fn baseRevision(rev: u64) [3]u64 { + return .{ + 0xf9562b2d5c95a6c8, + 0x6a7b384944536bdc, + rev, + }; +} + +pub fn baseRevisionSupported(base_rev: *const [3]u64) bool { + // Limine will overwrite base_rev[2] with 0 if it supports the requested revision. + return base_rev[2] == 0; +} + +pub const framebuffer_request_id: [4]u64 = .{ + common_magic[0], + common_magic[1], + 0x9d5827dcd881dd75, + 0xa3148604f6fab11b, +}; + +pub const FramebufferRequest = extern struct { + id: [4]u64, + revision: u64, + response: ?*FramebufferResponse, +}; + +pub const FramebufferResponse = extern struct { + revision: u64, + framebuffer_count: u64, + framebuffers: ?[*]*Framebuffer, +}; + +pub const Framebuffer = extern struct { + address: ?*anyopaque, + width: u64, + height: u64, + pitch: u64, + bpp: u16, + memory_model: u8, + red_mask_size: u8, + red_mask_shift: u8, + green_mask_size: u8, + green_mask_shift: u8, + blue_mask_size: u8, + blue_mask_shift: u8, + unused: [7]u8, + edid_size: u64, + edid: ?*anyopaque, + // Response revision 1 fields (ignored by this template). + mode_count: u64, + modes: ?[*]*anyopaque, +}; + diff --git a/src/main.zig b/src/main.zig new file mode 100644 index 0000000..c03ca18 --- /dev/null +++ b/src/main.zig @@ -0,0 +1,57 @@ +const std = @import("std"); +const ksyms = @import("ksyms"); +const pl = @import("base").pl; + +const DebugWriter = struct { + pub const Error = error{}; + pub const Writer = std.io.GenericWriter(*DebugWriter, Error, write); + + fn write(self: *DebugWriter, bytes: []const u8) Error!usize { + _ = self; + for (bytes) |c| pl.debug_write(c); + return bytes.len; + } + + fn writer(self: *DebugWriter) Writer { + return .{ .context = self }; + } +}; + +var debug_writer = DebugWriter{}; + +export fn kmain() callconv(.c) void { + print("hello from kernel!\n", .{}); + + for (ksyms.ksyms) |sym| { + print(" 0x{x:0>16} {s}\n", .{ sym.addr, sym.name }); + } + + pl.first_init(); + + while (true) { + asm volatile ("hlt"); + } +} + +export fn memset(dest: [*]u8, c: u8, n: usize) [*]u8 { + for (dest[0..n]) |*b| b.* = c; + return dest; +} + +export fn memcpy(dest: [*]u8, src: [*]const u8, n: usize) [*]u8 { + for (0..n) |i| dest[i] = src[i]; + return dest; +} + +export fn memmove(dest: [*]u8, src: [*]const u8, n: usize) [*]u8 { + if (@intFromPtr(dest) < @intFromPtr(src)) { + for (0..n) |i| dest[i] = src[i]; + } else { + var i = n; + while (i > 0) { + i -= 1; + dest[i] = src[i]; + } + } + return dest; +} diff --git a/src/platform/amd64/entry.zig b/src/platform/amd64/entry.zig new file mode 100644 index 0000000..534a03b --- /dev/null +++ b/src/platform/amd64/entry.zig @@ -0,0 +1,3 @@ +const b = @import("base"); + +pub fn first_init() linksection(b.init) void {} diff --git a/src/platform/amd64/impl.zig b/src/platform/amd64/impl.zig new file mode 100644 index 0000000..f77b5ac --- /dev/null +++ b/src/platform/amd64/impl.zig @@ -0,0 +1,16 @@ +const entry = @import("entry.zig"); + +pub const first_init = entry.first_init; + +pub fn devices_init() void {} + +pub fn debug_write(c: u8) void { + // Write to debugcon + asm volatile ("outb %[c], %[port]" + : + : [c] "{al}" (c), + [port] "N{dx}" (@as(u16, 0xe9)), + ); +} + +pub fn arm_timer(_: u64) void {} diff --git a/src/platform/root.zig b/src/platform/root.zig new file mode 100644 index 0000000..087da67 --- /dev/null +++ b/src/platform/root.zig @@ -0,0 +1,75 @@ +//! Platform interface: each port must implement these functions and variables. +//! To add a new platform, create a new file and implement all required functions. + +const config = @import("config"); + +pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64")) + @import("amd64/impl.zig") +else if (@hasDecl(config, "CONFIG_ARCH_UM")) + @import("um/impl.zig") +else + @compileError("unsupported architecture"); + +/// Early platform-specific initialization, called before anything else. +pub inline fn early_init() void { + return impl.early_init(); +} + +/// Late platform init. Memory allocator is available at this point. +/// SMP bringup must be done after this returns. +pub inline fn late_init() void { + return impl.late_init(); +} + +/// Write a single character to the debug console. +pub inline fn debug_write(c: u8) void { + return impl.debug_write(c); +} + +/// Arm a one-shot timer to fire in `ns` nanoseconds. +/// The timer must call `ki_timer_expiry()` when it fires. +pub inline fn arm_timer(ns: u64) void { + return impl.arm_timer(ns); +} + +/// Pretty name for the platform +pub const name = impl.name; + +/// A generic struct representing information passed from the bootloader. +pub const BootInfo = struct { + /// Represents the system memory map. + pub const MemMap = struct { + /// A memory map entry. + pub const Entry = struct { + pub const Kind = enum { + Free, + Reserved, + LoaderReclaimable, + AcpiNvs, + AcpiReclaimable, + Kernel, + }; + + kind: Kind, + base: usize, + size: usize, + }; + + entry_count: usize, + entries: [256]Entry, + }; + + pub const KernelAddress = struct { physical_base: usize, virtual_base: usize }; + + /// RSDP on ACPI machines + rsdp: usize, + + /// Kernel command-line arguments + cmdline: ?[]const u8, + + /// Memory map + memory_map: MemMap, + + /// Kernel address + kernel_address: KernelAddress, +}; diff --git a/src/platform/um/entry.zig b/src/platform/um/entry.zig new file mode 100644 index 0000000..53e4a4f --- /dev/null +++ b/src/platform/um/entry.zig @@ -0,0 +1,81 @@ +const b = @import("base"); + +pub const std_options = b.std_options; + +const std = @import("std"); +const ke = b.ke; +const pl = b.pl; + +const default_mem_size = b.gib(1); + +fn parse_mem_size(str: []const u8) !usize { + if (str.len == 0) return default_mem_size; + + const last = str[str.len - 1]; + + const mul: usize = switch (std.ascii.toLower(last)) { + 'k' => b.kib(1), + 'm' => b.mib(1), + 'g' => b.gib(1), + else => 1, + }; + + const digits = if (mul == 1) str else str[0 .. str.len - 1]; + const size = try std.fmt.parseInt(usize, digits, 10); + return size * mul; +} + +pub fn main(init: std.process.Init) !void { + var args = try init.minimal.args.iterateAllocator(init.gpa); + defer args.deinit(); + + const exe = args.next().?; + + var mem_size: usize = default_mem_size; + var cmdline: ?[]const u8 = null; + var ncpus: usize = 1; + + while (args.next()) |arg| { + if (std.mem.eql(u8, arg, "-m") or std.mem.eql(u8, arg, "--mem-size")) { + const val = args.next() orelse fatal("--mem-size requires a value", .{}); + mem_size = parse_mem_size(val) catch fatal("invalid mem-size", .{}); + } else if (std.mem.eql(u8, arg, "-c") or std.mem.eql(u8, arg, "--cmdline")) { + cmdline = args.next() orelse fatal("--cmdline requires a value", .{}); + } else if (std.mem.eql(u8, arg, "-t") or std.mem.eql(u8, arg, "--threads")) { + const val = args.next() orelse fatal("--threads requires a value", .{}); + ncpus = std.fmt.parseInt(usize, val, 10) catch fatal("invalid thread count", .{}); + } else if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) { + var stderr = std.Io.File.stderr().writerStreaming(init.io, &.{}); + var w = &stderr.interface; + + try w.print( + \\Usage: {s} [options] + \\ -m, --mem-size SIZE Set the memory size + \\ -c, --cmdline STR Set the kernel cmdline + \\ -t, --threads COUNT Set the number of threads + \\ + , .{exe}); + std.process.exit(0); + } else { + fatal("unknown argument: {s}", .{arg}); + } + } + + pl.impl.global_state.params = + .{ .ncpus = ncpus, .cmdline = cmdline, .mem_size = mem_size }; + + pl.impl.global_state.boot_info.kernel_address.physical_base = 0; + pl.impl.global_state.boot_info.kernel_address.virtual_base = 0; + + pl.impl.global_state.boot_info.rsdp = 0; + pl.impl.global_state.boot_info.cmdline = cmdline; + + ke.ncpus = ncpus; + + ke.private.init(&pl.impl.global_state.boot_info); +} + +fn fatal(comptime fmt: []const u8, args: anytype) noreturn { + std.debug.print(fmt ++ "\n", args); + std.process.exit(1); +} diff --git a/src/platform/um/impl.zig b/src/platform/um/impl.zig new file mode 100644 index 0000000..a4249b6 --- /dev/null +++ b/src/platform/um/impl.zig @@ -0,0 +1,200 @@ +const std = @import("std"); +const linux = std.os.linux; +const b = @import("base"); +const pl = b.pl; +const ke = b.ke; +const ki = b.ke.private; +const posix = std.posix; + +const timer = @import("timer.zig"); + +pub const c = @cImport({ + @cDefine("_GNU_SOURCE", "1"); + @cInclude("pthread.h"); + @cInclude("sys/mman.h"); + @cInclude("ucontext.h"); + @cInclude("signal.h"); + @cInclude("unistd.h"); + @cInclude("sys/time.h"); + @cInclude("sched.h"); +}); + +pub const name = "User-Mode"; + +pub const GlobalState = struct { + const Params = struct { + mem_size: usize, + cmdline: ?[]const u8, + ncpus: usize, + }; + + params: Params, + boot_info: pl.BootInfo, + phys_memory_memfd: i32, + phys_base: usize, +}; + +pub var global_state: GlobalState = undefined; +pub threadlocal var my_cpu: *ke.Cpu = undefined; + +var cpu_id: u32 = 0; + +pub fn devices_init() void {} + +pub fn debug_write(char: u8) void { + std.debug.print("{c}", .{char}); +} + +pub const arm_timer = timer.arm_timer; + +fn check_function(fn_name: []const u8, ret: usize) void { + const real_ret: i64 = @intCast(ret); + + if (real_ret < 0) { + const errno_val = linux.errno(ret); + + std.debug.print("error: {s}: {s}\n", .{ fn_name, @tagName(errno_val) }); + std.process.exit(1); + } +} + +pub fn early_init() linksection(b.init) void { + // Set current CPU to the bootstrap CPU + my_cpu = &ki.bootstrap_cpu; + + my_cpu.impl.pthread = c.pthread_self(); + + // Allocate our ID + my_cpu.id = 0; + cpu_id += 1; + + const memfd: i32 = @intCast(linux.memfd_create("phys_memory", 0)); + + check_function("ftruncate", linux.ftruncate(memfd, @intCast(global_state.params.mem_size))); + + const phys_base = linux.mmap(null, global_state.params.mem_size, .{ .READ = true, .WRITE = true }, .{ .ANONYMOUS = true, .TYPE = .SHARED }, memfd, 0); + + check_function("mmap", phys_base); + + // Build memory map + global_state.boot_info.memory_map.entry_count = 1; + global_state.boot_info.memory_map.entries[0].base = phys_base; + global_state.boot_info.memory_map.entries[0].size = global_state.params.mem_size; + global_state.boot_info.memory_map.entries[0].kind = .Free; + + global_state.phys_memory_memfd = memfd; + global_state.phys_base = phys_base; + + const pfndb_base = linux.mmap(null, b.tib(1), .{}, .{ .ANONYMOUS = true, .TYPE = .PRIVATE }, -1, 0); + + check_function("mmap", pfndb_base); + + const heap_base = linux.mmap(null, b.gib(8), .{}, .{ .ANONYMOUS = true, .TYPE = .PRIVATE }, -1, 0); + + check_function("mmap", heap_base); + + b.kernel_pfndb_base = pfndb_base; + b.kernel_heap_base = heap_base; + + std.log.info("um: kernel heap lives at {X}", .{heap_base}); +} + +fn sigusr1_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void { + if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) { + ki.dpc.dispatch(ke.curcpu()); + } +} + +fn make_thread(entry: *const fn (?*anyopaque) void, td: *ke.Thread) void { + const stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf"); + + td.init(@intFromPtr(stack.ptr), 16384, entry, null); + + td.priority = 0; + td.priority_class = .Idle; +} + +var cpus_up = std.atomic.Value(usize).init(0); + +fn other_cpu_entry(arg: ?*anyopaque) callconv(.c) ?*anyopaque { + my_cpu = @ptrCast(@alignCast(arg.?)); + + my_cpu.id = @atomicRmw(u32, &cpu_id, .Add, 1, .monotonic); + my_cpu.impl.pthread = c.pthread_self(); + + my_cpu.init(my_cpu.idle_thread.?); + + timer.init_cpu(); + + _ = cpus_up.fetchAdd(1, .monotonic); + + while (true) {} + + return null; +} + +fn idle(_: ?*anyopaque) void {} + +pub fn late_init() linksection(b.init) void { + timer.init(); + timer.init_cpu(); + + var sa: posix.Sigaction = std.mem.zeroes(posix.Sigaction); + sa.handler = .{ + .sigaction = sigusr1_handler, + }; + + sa.flags = posix.SA.SIGINFO; + + sa.mask = posix.sigemptyset(); + + _ = posix.sigaction(posix.SIG.USR1, &sa, null); + + std.log.info("bum: starting {} thread{s}", .{ + ke.ncpus - 1, + if ((ke.ncpus - 1) > 1) "s" else "", + }); + + const other_count = ke.ncpus - 1; + + const allocator = std.heap.page_allocator; + + var other_threads = + allocator.alloc(c.pthread_t, other_count) catch @panic("oom"); + + ke.cpus = allocator.alloc(*ke.Cpu, ke.ncpus) catch @panic("oom"); + ke.cpus[0] = &ki.bootstrap_cpu; + + for (1..ke.ncpus) |i| { + ke.cpus[i] = allocator.create(ke.Cpu) catch @panic("oom"); + + ke.cpus[i].idle_thread = allocator.create(ke.Thread) catch @panic("oom"); + + make_thread(idle, ke.cpus[i].idle_thread.?); + + var attr: c.pthread_attr_t = undefined; + + _ = c.pthread_attr_init(&attr); + + var cpuset: c.cpu_set_t = std.mem.zeroes(c.cpu_set_t); + cpuset.__bits[i / 64] |= @as(c_ulong, 1) << @intCast(i % 64); + _ = c.pthread_attr_setaffinity_np(&attr, @sizeOf(c.cpu_set_t), &cpuset); + + const r = c.pthread_create( + &other_threads[i - 1], + &attr, + other_cpu_entry, + ke.cpus[i], + ); + + if (r != 0) { + @panic("pthread_create() failed"); + } + } + + if (ke.ncpus > 1) { + while (cpus_up.load(.monotonic) < @as(u32, @intCast(ke.ncpus - 1))) { + std.atomic.spinLoopHint(); + } + } +} diff --git a/src/platform/um/timer.zig b/src/platform/um/timer.zig new file mode 100644 index 0000000..96df68f --- /dev/null +++ b/src/platform/um/timer.zig @@ -0,0 +1,73 @@ +const b = @import("base"); +const std = @import("std"); +const posix = std.posix; +const linux = std.os.linux; +const c = b.pl.impl.c; +const ke = b.ke; +const ki = ke.private; + +threadlocal var timerid: c.timer_t = undefined; + +fn timer_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void { + const old_ipl = ke.ipl.set_hardware(.Device); + + ke.dpc.enqueue(&ke.curcpu().timer_dpc, null); + + _ = ke.ipl.set_hardware(old_ipl); + + if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) { + ki.dpc.dispatch(ke.curcpu()); + } +} + +pub fn arm_timer(ns: b.Nanoseconds) void { + const value: c.struct_itimerspec = .{ .it_value = .{ + .tv_sec = @intCast(ns / std.time.ns_per_s), + .tv_nsec = @intCast(ns % std.time.ns_per_s), + }, .it_interval = .{ .tv_sec = 0, .tv_nsec = 0 } }; + + if (c.timer_settime(timerid, 0, &value, null) == -1) { + ke.panic("timer_settime failed", .{}); + } +} + +fn tc_read() u64 { + var ts: std.c.timespec = undefined; + + _ = linux.clock_gettime(linux.CLOCK.MONOTONIC, &ts); + + return @intCast(ts.sec * std.time.ns_per_s + ts.nsec); +} + +var tc: ke.TimeCounter = .{ + .read_count = tc_read, + .frequency = std.time.ns_per_s, + .name = "CLOCK_MONOTONIC", + .quality = 1000, + .mask = std.math.maxInt(u64), + .p = 0, + .n = 0, +}; + +pub fn init() void { + var act: posix.Sigaction = undefined; + + act.handler.sigaction = timer_handler; + act.flags = posix.SA.RESTART | posix.SA.SIGINFO; + + posix.sigaction(posix.SIG.ALRM, &act, null); + + ke.timecounter.register(&tc); +} + +pub fn init_cpu() void { + var sev: c.sigevent = std.mem.zeroes(c.sigevent); + + sev.sigev_notify = c.SIGEV_THREAD_ID; + sev.sigev_signo = c.SIGALRM; + sev._sigev_un._tid = linux.gettid(); + + if (c.timer_create(c.CLOCK_MONOTONIC, &sev, &timerid) == -1) { + ke.panic("timer_create failed", .{}); + } +} diff --git a/src/root.zig b/src/root.zig new file mode 100644 index 0000000..8579a57 --- /dev/null +++ b/src/root.zig @@ -0,0 +1,38 @@ +const std = @import("std"); +pub const ke = @import("kern/root.zig"); +pub const pl = @import("platform/root.zig"); +pub const rtl = @import("rtl"); + +pub const init = ".init.text"; + +pub const VAddr = usize; +pub const PAddr = usize; + +pub const Nanoseconds = u64; + +pub var kernel_heap_base: usize = 0; +pub var kernel_pfndb_base: usize = 0; + +/// Return N kibibytes in bytes. +pub fn kib(comptime N: u32) usize { + return N * 1024; +} + +/// Return N mebibytes in bytes. +pub fn mib(comptime N: u32) usize { + return kib(N) * 1024; +} + +/// Return N gibibytes in bytes. +pub fn gib(comptime N: u32) usize { + return mib(N) * 1024; +} + +/// Return N tibibytes in bytes. +pub fn tib(comptime N: u32) usize { + return gib(N) * 1024; +} + +pub const std_options = std.Options{ + .logFn = ke.log, +}; diff --git a/src/rtl/barrier.zig b/src/rtl/barrier.zig new file mode 100644 index 0000000..573a6db --- /dev/null +++ b/src/rtl/barrier.zig @@ -0,0 +1,57 @@ +//! Atomic barrier implementations on various architectures. +//! This is needed since Zig removed @fence. + +const builtin = @import("builtin"); + +/// Read Memory Barrier (Load-Load) +/// Ensures all loads before the barrier complete before any loads after it. +pub inline fn rmb() void { + switch (builtin.cpu.arch) { + .x86, .x86_64 => { + asm volatile ("" ::: .{ .memory = true }); + }, + .aarch64 => { + asm volatile ("dmb ishld" ::: .{ .memory = true }); + }, + .arm => { + asm volatile ("dmb ish" ::: .{ .memory = true }); + }, + .riscv64, .riscv32 => { + asm volatile ("fence r,r" ::: .{ .memory = true }); + }, + else => @compileError("unsupported architecture"), + } +} + +/// Write Memory Barrier (Store-Store) +/// Ensures all stores before the barrier complete before any stores after it. +pub inline fn wmb() void { + switch (builtin.cpu.arch) { + .x86, .x86_64 => { + asm volatile ("" ::: .{ .memory = true }); + }, + .aarch64, .arm => { + asm volatile ("dmb ish" ::: .{ .memory = true }); + }, + .riscv64, .riscv32 => { + asm volatile ("fence rw,w" ::: .{ .memory = true }); + }, + else => @compileError("unsupported architecture"), + } +} + +/// Full Memory Barrier (Load/Store - Load/Store) +pub inline fn mb() void { + switch (builtin.cpu.arch) { + .x86, .x86_64 => { + asm volatile ("mfence" ::: .{ .memory = true }); + }, + .aarch64, .arm => { + asm volatile ("dmb ish" ::: .{ .memory = true }); + }, + .riscv64, .riscv32 => { + asm volatile ("fence rw,rw" ::: .{ .memory = true }); + }, + else => @compileError("unsupported architecture"), + } +} diff --git a/src/rtl/list.zig b/src/rtl/list.zig new file mode 100644 index 0000000..3cba3eb --- /dev/null +++ b/src/rtl/list.zig @@ -0,0 +1,204 @@ +//! Intrusive circular linked list module. +/// Branchless intrusive circular linked list. +/// Embed `List.Entry` in your struct and use `@fieldParentPtr` to recover it. +pub const List = struct { + head: Entry, + + /// A list entry + pub const Entry = struct { + /// Next element in the list + next: *Entry = undefined, + /// Previous element + prev: *Entry = undefined, + + /// Removes an entry from whatever list it's in + pub fn remove(elem: *Entry) void { + elem.prev.next = elem.next; + elem.next.prev = elem.prev; + } + + /// Inserts elem before `before` + pub fn insert_before(elem: *Entry, before: *Entry) void { + elem.prev = before.prev; + elem.next = before; + before.prev.next = elem; + before.prev = elem; + } + }; + + /// Initializes the list + pub fn init(self: *List) void { + self.head.next = &self.head; + self.head.prev = &self.head; + } + + /// Returns whether the list is empty + pub fn is_empty(self: *List) bool { + return self.head.next == &self.head; + } + + /// Inserts elem at the end of the list + pub fn insert_tail(self: *List, elem: *Entry) void { + self.head.prev.next = elem; + elem.next = &self.head; + elem.prev = self.head.prev; + self.head.prev = elem; + } + + /// Inserts elem at the beginning of the list + pub fn insert_head(self: *List, elem: *Entry) void { + self.head.next.prev = elem; + elem.next = self.head.next; + elem.prev = &self.head; + self.head.next = elem; + } + + /// Returns the first element of the list + pub fn first(self: *List) *Entry { + return self.head.next; + } + + /// Returns the last element of the list + pub fn last(self: *List) *Entry { + return self.head.prev; + } + + pub fn iterator(self: *List) Iterator { + return .{ .head = &self.head, .current = self.head.next }; + } + + pub const Iterator = struct { + head: *Entry, + current: *Entry, + + pub fn next(self: *Iterator) bool { + return self.current != self.head; + } + + pub fn get(self: *Iterator) *Entry { + return self.current; + } + + pub fn advance(self: *Iterator) void { + self.current = self.current.next; + } + }; +}; + +const std = @import("std"); + +test "init creates empty list" { + var list: List = undefined; + list.init(); + try std.testing.expect(list.is_empty()); +} + +test "insert_tail adds elements in order" { + var list: List = undefined; + list.init(); + + const Node = struct { value: u32, link: List.Entry = undefined }; + var a = Node{ .value = 1 }; + var b = Node{ .value = 2 }; + var c = Node{ .value = 3 }; + + list.insert_tail(&a.link); + list.insert_tail(&b.link); + list.insert_tail(&c.link); + + var it = list.iterator(); + var i: u32 = 1; + while (it.next()) : (it.advance()) { + const node: *Node = @fieldParentPtr("link", it.get()); + try std.testing.expectEqual(i, node.value); + i += 1; + } + try std.testing.expectEqual(@as(u32, 4), i); +} + +test "insert_head adds elements in reverse order" { + var list: List = undefined; + list.init(); + + const Node = struct { value: u32, link: List.Entry = undefined }; + var a = Node{ .value = 1 }; + var b = Node{ .value = 2 }; + var c = Node{ .value = 3 }; + + list.insert_head(&a.link); + list.insert_head(&b.link); + list.insert_head(&c.link); + + var it = list.iterator(); + var i: u32 = 3; + while (it.next()) : (it.advance()) { + const node: *Node = @fieldParentPtr("link", it.get()); + try std.testing.expectEqual(i, node.value); + i -= 1; + } +} + +test "remove detaches element" { + var list: List = undefined; + list.init(); + + const Node = struct { value: u32, link: List.Entry = undefined }; + var a = Node{ .value = 1 }; + var b = Node{ .value = 2 }; + var c = Node{ .value = 3 }; + + list.insert_tail(&a.link); + list.insert_tail(&b.link); + list.insert_tail(&c.link); + + b.link.remove(); + + var it = list.iterator(); + const first: *Node = @fieldParentPtr("link", it.get()); + try std.testing.expectEqual(@as(u32, 1), first.value); + it.advance(); + const second: *Node = @fieldParentPtr("link", it.get()); + try std.testing.expectEqual(@as(u32, 3), second.value); + it.advance(); + try std.testing.expect(!it.next()); +} + +test "insert_before inserts in correct position" { + var list: List = undefined; + list.init(); + + const Node = struct { value: u32, link: List.Entry = undefined }; + var a = Node{ .value = 1 }; + var b = Node{ .value = 3 }; + var mid = Node{ .value = 2 }; + + list.insert_tail(&a.link); + list.insert_tail(&b.link); + + mid.link.insert_before(&b.link); + + var it = list.iterator(); + var i: u32 = 1; + while (it.next()) : (it.advance()) { + const node: *Node = @fieldParentPtr("link", it.get()); + try std.testing.expectEqual(i, node.value); + i += 1; + } +} + +test "first and last return correct elements" { + var list: List = undefined; + list.init(); + + const Node = struct { value: u32, link: List.Entry = undefined }; + var a = Node{ .value = 1 }; + var b = Node{ .value = 2 }; + + list.insert_tail(&a.link); + list.insert_tail(&b.link); + + const first: *Node = @fieldParentPtr("link", list.first()); + const last: *Node = @fieldParentPtr("link", list.last()); + try std.testing.expectEqual(@as(u32, 1), first.value); + try std.testing.expectEqual(@as(u32, 2), last.value); +} diff --git a/src/rtl/pairing_heap.zig b/src/rtl/pairing_heap.zig new file mode 100644 index 0000000..f416556 --- /dev/null +++ b/src/rtl/pairing_heap.zig @@ -0,0 +1,331 @@ +//! Intrusive Pairing Heap implementation. +const std = @import("std"); + +pub const Node = struct { + child: ?*Node = null, + next: ?*Node = null, + prev: ?*Node = null, +}; + +pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *Node) std.math.Order) type { + return struct { + const Self = @This(); + + /// Number of elements in the heap + size: usize = 0, + + /// Topmost element of the heap + root: ?*Node = null, + + fn meld(a: *Node, b: *Node) *Node { + const a_wins = switch (order) { + .min => cmp(a, b) != .gt, // a <= b + .max => cmp(a, b) == .gt, // a > b + }; + + if (a_wins) { + if (a.child) |c| { + c.prev = b; + } + + b.next = a.child; + a.child = b; + b.prev = a; + return a; + } + + if (b.child) |c| { + c.prev = a; + } + + a.prev = b; + a.next = b.child; + b.child = a; + return b; + } + + fn merge_pairs(node: *Node) *Node { + // First pass: pair up siblings left to right + var list: ?*Node = node; + var pairs: ?*Node = null; + + while (list) |a| { + const b = a.next orelse { + a.next = pairs; + a.prev = null; + pairs = a; + break; + }; + list = b.next; + + a.next = null; + a.prev = null; + b.next = null; + b.prev = null; + + const merged = meld(a, b); + merged.next = pairs; + merged.prev = null; + pairs = merged; + } + + // Second pass: fold pairs right to left + var result = pairs.?; + var rest = result.next; + result.next = null; + while (rest) |p| { + rest = p.next; + p.next = null; + result = meld(result, p); + } + return result; + } + + pub fn init() Self { + return .{ + .size = 0, + .root = null, + }; + } + + /// Insert `node` into the heap. + pub fn insert(self: *Self, node: *Node) void { + node.* = .{ + .child = null, + .next = null, + .prev = null, + }; + + self.size += 1; + + if (self.root == null) { + self.root = node; + return; + } + + self.root = meld(self.root.?, node); + } + + /// Return and remove the topmost element. + pub fn pop(self: *Self) ?*Node { + const root = self.root orelse return null; + const child = root.child; + + root.child = null; + self.size -= 1; + + if (child) |c| { + c.prev = null; + self.root = merge_pairs(c); + } else { + self.root = null; + } + + return root; + } + + /// Remove `node` from the heap. + pub fn remove(self: *Self, node: *Node) void { + if (self.root == node) { + _ = self.pop(); + return; + } + + std.debug.assert(self.root != null); + + self.size -= 1; + + // Unlink the node + if (node.next) |n| { + n.prev = node.prev; + } + + if (node.prev) |p| { + if (p.child == node) { + // node was leftmost child + p.child = node.next; + } else { + // node was a sibling + p.next = node.next; + } + } + + node.prev = null; + node.next = null; + + // Merge node's children back into the heap + if (node.child) |c| { + c.prev = null; + self.root = meld(self.root.?, merge_pairs(c)); + } + + node.child = null; + } + }; +} +const TestNode = struct { + value: u32, + node: Node = .{}, +}; + +fn my_cmp(a: *Node, b: *Node) std.math.Order { + const ta: *TestNode = @fieldParentPtr("node", a); + const tb: *TestNode = @fieldParentPtr("node", b); + return std.math.order(ta.value, tb.value); +} + +const MinHeap = PairingHeap(.min, my_cmp); +const MaxHeap = PairingHeap(.max, my_cmp); + +test "empty heap" { + var heap = MinHeap.init(); + try std.testing.expect(heap.root == null); + try std.testing.expectEqual(0, heap.size); + try std.testing.expect(heap.pop() == null); +} + +test "single element insert and pop" { + var heap = MinHeap.init(); + var a = TestNode{ .value = 42 }; + + heap.insert(&a.node); + try std.testing.expectEqual(1, heap.size); + + const top: *TestNode = @fieldParentPtr("node", heap.root.?); + try std.testing.expectEqual(42, top.value); + + _ = heap.pop(); + try std.testing.expect(heap.root == null); + try std.testing.expectEqual(0, heap.size); + try std.testing.expect(heap.pop() == null); +} + +test "min heap ordering" { + var heap = MinHeap.init(); + var a = TestNode{ .value = 3 }; + var b = TestNode{ .value = 1 }; + var c = TestNode{ .value = 2 }; + + heap.insert(&a.node); + heap.insert(&b.node); + heap.insert(&c.node); + try std.testing.expectEqual(3, heap.size); + + const first: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(1, first.value); + + const second: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(2, second.value); + + const third: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(3, third.value); + + try std.testing.expect(heap.pop() == null); +} + +test "max heap ordering" { + var heap = MaxHeap.init(); + var a = TestNode{ .value = 3 }; + var b = TestNode{ .value = 1 }; + var c = TestNode{ .value = 2 }; + + heap.insert(&a.node); + heap.insert(&b.node); + heap.insert(&c.node); + + const first: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(3, first.value); + + const second: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(2, second.value); + + const third: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(1, third.value); +} + +test "remove middle node" { + var heap = MinHeap.init(); + var a = TestNode{ .value = 1 }; + var b = TestNode{ .value = 2 }; + var c = TestNode{ .value = 3 }; + + heap.insert(&a.node); + heap.insert(&b.node); + heap.insert(&c.node); + + heap.remove(&b.node); + try std.testing.expectEqual(2, heap.size); + + const first: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(1, first.value); + + const second: *TestNode = @fieldParentPtr("node", heap.pop().?); + try std.testing.expectEqual(3, second.value); + + try std.testing.expect(heap.pop() == null); +} + +test "remove root" { + var heap = MinHeap.init(); + var a = TestNode{ .value = 1 }; + var b = TestNode{ .value = 2 }; + var c = TestNode{ .value = 3 }; + + heap.insert(&a.node); + heap.insert(&b.node); + heap.insert(&c.node); + + heap.remove(&a.node); + try std.testing.expectEqual(2, heap.size); + + const top: *TestNode = @fieldParentPtr("node", heap.root.?); + try std.testing.expectEqual(2, top.value); +} + +test "remove last element" { + var heap = MinHeap.init(); + var a = TestNode{ .value = 1 }; + + heap.insert(&a.node); + heap.remove(&a.node); + try std.testing.expectEqual(0, heap.size); + try std.testing.expect(heap.root == null); +} + +test "duplicate values" { + var heap = MinHeap.init(); + var a = TestNode{ .value = 1 }; + var b = TestNode{ .value = 1 }; + var c = TestNode{ .value = 1 }; + + heap.insert(&a.node); + heap.insert(&b.node); + heap.insert(&c.node); + try std.testing.expectEqual(3, heap.size); + + _ = heap.pop(); + _ = heap.pop(); + _ = heap.pop(); + try std.testing.expectEqual(0, heap.size); + try std.testing.expect(heap.pop() == null); +} + +test "large sequence" { + var heap = MinHeap.init(); + var nodes: [16]TestNode = undefined; + + for (&nodes, 0..) |*n, i| { + n.* = .{ .value = @intCast(16 - i) }; + heap.insert(&n.node); + } + + try std.testing.expectEqual(16, heap.size); + + var prev: u32 = 0; + while (heap.pop()) |n| { + const t: *TestNode = @fieldParentPtr("node", n); + try std.testing.expect(t.value >= prev); + prev = t.value; + } + try std.testing.expectEqual(0, heap.size); +} diff --git a/src/rtl/root.zig b/src/rtl/root.zig new file mode 100644 index 0000000..342aa3f --- /dev/null +++ b/src/rtl/root.zig @@ -0,0 +1,11 @@ +const std = @import("std"); + +pub const List = @import("list.zig").List; +pub const SeqLock = @import("seqlock.zig").SeqLock; +pub const barrier = @import("barrier.zig"); +pub const pairing_heap = @import("pairing_heap.zig"); +pub const PairingHeap = pairing_heap.PairingHeap; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/src/rtl/seqlock.zig b/src/rtl/seqlock.zig new file mode 100644 index 0000000..d3afc61 --- /dev/null +++ b/src/rtl/seqlock.zig @@ -0,0 +1,52 @@ +const std = @import("std"); +const rtl = @import("rtl"); + +pub fn SeqLock(comptime T: type) type { + return struct { + const Self = @This(); + + raw: T, + sequence: std.atomic.Value(usize), + + pub fn init(val: T) Self { + return .{ .raw = val, .sequence = .init(0) }; + } + + pub fn load(self: *Self) T { + var ret: T = undefined; + + while (true) { + const seq = self.sequence.load(.acquire); + if (seq & 1 != 0) { + std.atomic.spinLoopHint(); + continue; + } + + ret = self.raw; + + // Ensure the sequence read happens *after* the data is fully loaded. + rtl.barrier.rmb(); + + if (self.sequence.load(.monotonic) == seq) + break; + + std.atomic.spinLoopHint(); + } + + return ret; + } + + pub fn store(self: *Self, data: T) void { + const seq = self.sequence.load(.monotonic); + + self.sequence.store(seq + 1, .monotonic); + + // Ensure the data write happens *after* the sequence is incremented. + rtl.fence.wmb(); + + self.raw = data; + + self.sequence.store(seq + 2, .release); + } + }; +}