diff --git a/src/loader/amd64/impl.zig b/src/loader/amd64/impl.zig index 61291d6..26b7445 100644 --- a/src/loader/amd64/impl.zig +++ b/src/loader/amd64/impl.zig @@ -97,6 +97,10 @@ pub inline fn activate(root_pa: usize) void { amd64.write_cr(3, root_pa); } +pub inline fn flush(va: usize) void { + amd64.invlpg(va); +} + pub fn is_leaf_level_enabled(level: usize) bool { if (!levels[level].leaf) return false; if (level == 2) return amd64.cpu_features.gib_pages; @@ -106,3 +110,7 @@ pub fn is_leaf_level_enabled(level: usize) bool { pub fn debug_write(c: u8) void { amd64.outb(0xe9, c); } + +pub fn init() void { + amd64.detect_cpu_features(); +} diff --git a/src/loader/info.zig b/src/loader/info.zig index f2159b6..eacdfaf 100644 --- a/src/loader/info.zig +++ b/src/loader/info.zig @@ -6,6 +6,7 @@ pub const MemMap = struct { Free, Reserved, LoaderReclaimable, + LoaderData, AcpiNvs, AcpiReclaimable, }; @@ -16,6 +17,7 @@ pub const MemMap = struct { }; entry_count: usize, + loader_memory_used: usize, entries: [128]Entry, }; @@ -27,6 +29,9 @@ pub const Framebuffer = struct { bpp: u8, }; +// Kept in sync with the kernel +pub const page_struct_size = 16; + pub const KernelAddress = struct { physical_base: usize, virtual_base: usize }; /// RSDP on ACPI machines diff --git a/src/loader/limine/impl.zig b/src/loader/limine/impl.zig index dc054f2..54fc66f 100644 --- a/src/loader/limine/impl.zig +++ b/src/loader/limine/impl.zig @@ -65,8 +65,6 @@ pub fn get_image_layout() r.ImageLayout { export fn loader_entry() callconv(.c) void { const mmap = memmap_request.response.?; - std.log.info("hi", .{}); - for (0..mmap.entry_count) |i| { const entry = mmap.entries.?[i]; diff --git a/src/loader/main.zig b/src/loader/main.zig index 6ea4e40..35f1e14 100644 --- a/src/loader/main.zig +++ b/src/loader/main.zig @@ -1,6 +1,11 @@ const r = @import("root"); const std = @import("std"); +const Entry = r.BootInfo.MemMap.Entry; +const stack_size = 1024 * 16; // should be enough + +extern fn jump_to_kernel(loader_info: usize, stack: usize, entry: usize) void; + fn load_kernel(kernel: *anyopaque) usize { const elf: *std.elf.Elf64.Ehdr = @ptrCast(@alignCast(kernel)); const ident = elf.ident; @@ -22,26 +27,49 @@ fn load_kernel(kernel: *anyopaque) usize { for (0..elf.phnum) |_| { switch (phdr.type) { .LOAD => { - r.mem.pagemap.map_range_allocating(phdr.vaddr, std.mem.alignForward(usize, phdr.memsz, r.page_size), .{ - .execute = phdr.flags.X, - // Gotta have RW access for loading it. - // XXX remap the pages and set protections properly... - .read = true, - .write = true, - .global = true, - }); + const as_pages = std.mem.alignForward(usize, phdr.memsz, r.page_size); + const npages = as_pages / r.page_size; + var filesz_rem = phdr.filesz; - const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset); - const source = off[0..phdr.filesz]; - const va: [*]u8 = @ptrFromInt(phdr.vaddr); - const dest = va[0..phdr.filesz]; + for (0..npages) |i| { + const pa = r.mem.alloc_page(); + const addr = phdr.vaddr + i * r.page_size; - // Copy the file data. - @memcpy(dest, source); + r.mem.pagemap.map_page(addr, pa, .{ + // Gotta have RW access for loading it. + .read = true, + .write = true, + }); - // Fill the rest of memsz with 0. - const zero_dest = va[phdr.filesz..phdr.memsz]; - @memset(zero_dest, 0); + const file_remaining = @min(filesz_rem, r.page_size); + + const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset + i * r.page_size); + const source = off[0..file_remaining]; + const va: [*]u8 = @ptrFromInt(addr); + const dest = va[0..file_remaining]; + + filesz_rem -= file_remaining; + + // Copy the file data. + @memcpy(dest, source); + + if (filesz_rem == 0) { + const zeroes = phdr.memsz - phdr.filesz; + + // Fill the rest of memsz with 0. + @memset(va[0..zeroes], 0); + } + + // Now update the page permissions to their true value. + r.mem.pagemap.map_page(addr, pa, .{ + .execute = phdr.flags.X, + .read = phdr.flags.R, + .write = phdr.flags.W, + .global = true, + }); + + r.arch.flush(addr); + } }, else => {}, } @@ -52,13 +80,21 @@ fn load_kernel(kernel: *anyopaque) usize { return elf.entry; } -extern fn jump_to_kernel(loader_info: usize, stack: usize, entry: usize) void; +fn cmp_entries(ctx: void, a: Entry, b: Entry) bool { + _ = ctx; + return a.base < b.base; +} pub fn loader_main(kernel: *anyopaque) void { + r.arch.init(); r.mem.init(); const entry = load_kernel(kernel); - const stack = r.mem.alloc(1024 * 16); + const stack = r.mem.alloc(stack_size); - std.log.info("jumping to {x} with stack {x}", .{ entry, stack }); - jump_to_kernel(@intFromPtr(&r.loader_info), stack + (1024 * 16), entry); + const memmap = &r.loader_info.memory_map; + + // Finally, sort the memory map once we're done allocating everything. + std.mem.sort(Entry, memmap.entries[0..memmap.entry_count], {}, cmp_entries); + + jump_to_kernel(@intFromPtr(&r.loader_info), stack + stack_size, entry); } diff --git a/src/loader/mem.zig b/src/loader/mem.zig index f875823..cb21568 100644 --- a/src/loader/mem.zig +++ b/src/loader/mem.zig @@ -1,11 +1,13 @@ const r = @import("root"); const config = @import("config"); const pmap = @import("pmap.zig"); +const std = @import("std"); var memory_map: *r.BootInfo.MemMap = &r.loader_info.memory_map; var alloc_idx: usize = 0; var bump_base: usize = 0; var map_loaded: bool = false; +var last_data_idx: ?usize = null; const MemoryLayout = struct { /// Kernel direct map (if applicable) @@ -40,6 +42,19 @@ pub fn p2v(pa: usize) usize { return r.impl.p2v(pa); } +fn record_loader_data(addr: usize) void { + if (last_data_idx) |i| { + const e = &memory_map.entries[i]; + if (e.base + e.size == addr) { + e.size += r.page_size; + return; + } + } + + last_data_idx = memory_map.entry_count; + add_entry(addr, r.page_size, .LoaderData); +} + pub fn alloc_page() usize { while (alloc_idx < memory_map.entry_count) { const entry = &memory_map.entries[alloc_idx]; @@ -53,6 +68,8 @@ pub fn alloc_page() usize { entry.base += r.page_size; entry.size -= r.page_size; + memory_map.loader_memory_used += r.page_size; + record_loader_data(addr); const a: [*]u64 = @ptrFromInt(p2v(addr)); @@ -205,6 +222,41 @@ fn map_self() void { } } +fn map_pfndb() void { + const size = r.BootInfo.page_struct_size; + + // Now map the PFN database. + for (0..memory_map.entry_count) |i| { + const entry = memory_map.entries[i]; + + if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) { + continue; + } + + const npages = std.math.divCeil(usize, entry.size, r.page_size) catch unreachable; + + // 1. Calculate the exact virtual address range needed for this region's page structs. + const start_pfn: usize = entry.base / r.page_size; + const exact_start = memory_layout.pfndb + (start_pfn * size); + const exact_end = exact_start + (npages * size); + + // 2. Ensure the addresses are aligned on page boundaries. + const map_start = std.mem.alignBackward(usize, exact_start, r.page_size); + const map_end = std.mem.alignForward(usize, exact_end, r.page_size); + + // 3. Map the virtual pages. + pagemap.map_range_allocating( + map_start, + map_end - map_start, + .{ + .read = true, + .write = true, + .global = true, + }, + ); + } +} + pub fn init() void { const vbits = r.arch.virtual_bits; @@ -240,6 +292,7 @@ pub fn init() void { } map_self(); + map_pfndb(); map_loaded = true; r.arch.activate(pagemap.root_pa); diff --git a/src/loader/pmap.zig b/src/loader/pmap.zig index 83bd521..1603224 100644 --- a/src/loader/pmap.zig +++ b/src/loader/pmap.zig @@ -1,3 +1,6 @@ +//! Generic page map manipulation code. +//! Part of the code is duplicated from the kernel :( + const r = @import("root"); const mem = r.mem; @@ -167,11 +170,6 @@ pub const PMap = struct { } } - pub fn walk_down_resolve(self: *Cursor, target_level: usize) bool { - self.walk_down(target_level, false, false) catch return false; - return true; - } - /// Choose the largest page size that satisfies the alignment and /// size constraints of the current VA, PA, and remaining byte count. /// Tries large pages first; falls back to 4K. diff --git a/src/mm/amd64/impl.zig b/src/mm/amd64/impl.zig index e4355f7..e6edfb7 100644 --- a/src/mm/amd64/impl.zig +++ b/src/mm/amd64/impl.zig @@ -14,8 +14,8 @@ pub fn virt_to_phys(vaddr: r.VAddr) usize { } pub const hhdm_base = 0xffff800000000000; -pub const kernel_heap_base = 0xffffe00000000000; -pub const pfndb_base = 0xffffff0000000000; +pub const kernel_heap_base = 0xffffc00000000000; +pub const pfndb_base = 0xffffd00000000000; pub const levels = [_]mi.PMapLevel{ .{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K diff --git a/src/mm/init.zig b/src/mm/init.zig index 384e8ce..6ffb9cf 100644 --- a/src/mm/init.zig +++ b/src/mm/init.zig @@ -6,8 +6,6 @@ const config = @import("config"); pub fn init(boot_info: *r.BootInfo) linksection(r.init) void { mi.phys.init(boot_info); - mi.kernel_space.pmap.activate(); - mi.phys.init_pfndb(); mi.zone.early_init(); mi.vmem.init(); mi.heap.init(); diff --git a/src/mm/phys.zig b/src/mm/phys.zig index ec9e2cd..eebb758 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -18,8 +18,6 @@ pub var low_memory_threshold: usize = 0; /// Threshold at which reclamation stops. pub var high_memory_threshold: usize = 0; -var bootstrapped = false; -var early_alloc_entry_idx: usize = 0; var memory_map: *r.BootInfo.MemMap = undefined; var early_allocs: usize = 0; var list_lock: ke.SpinLock = .init(); @@ -31,27 +29,6 @@ var free_pages: usize = 0; pub var usable_memory: std.atomic.Value(usize) = .init(0); -pub fn early_alloc() usize { - while (early_alloc_entry_idx < memory_map.entry_count) { - const entry = &memory_map.entries[early_alloc_entry_idx]; - - if (entry.type != .Free or entry.size < mm.page_size) { - early_alloc_entry_idx += 1; - continue; - } - - const addr = entry.base; - - entry.base += mm.page_size; - entry.size -= mm.page_size; - early_allocs += mm.page_size; - - return addr; - } - - @panic("mm/phys: Out of memory for early allocation"); -} - // Wait for pages to be available. // List lock is held. fn wait_for_pages(old_ipl: ke.Ipl) void { @@ -167,7 +144,7 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void { free_list.init(); free_page_event.init(.Notification); - var total_usable_memory: usize = 0; + var total_usable_memory: usize = boot_info.memory_map.loader_memory_used; log.info("physical memory map:", .{}); for (0..boot_info.memory_map.entry_count) |i| { @@ -175,50 +152,18 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void { log.info("[{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) }); - if (entry.type == .Free) { + if (entry.type == .Free or entry.type == .LoaderReclaimable) { total_usable_memory += entry.size; } } const pfndb_size_required = @sizeOf(mm.Page) * (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable); - log.info("using {} KiB for pfndb ({} bytes per page)", .{ pfndb_size_required / 1024, @sizeOf(mm.Page) }); - // Create the kernel pagemap from the early allocator. + // Get the kernel pagemap from the loader, + // we are already in it so no need to activate it. mi.impl.init_kernel(); - // Now map the PFN database. - for (0..boot_info.memory_map.entry_count) |i| { - const entry = boot_info.memory_map.entries[i]; - - if (entry.type != .Free or entry.size < mm.page_size) { - continue; - } - - const npages = std.math.divCeil(usize, entry.size, mm.page_size) catch unreachable; - - // 1. Calculate the exact virtual address range needed for this region's page structs. - const start_pfn: usize = mm.page_to_pfn(entry.base); - const exact_start = mi.impl.pfndb_base + (start_pfn * @sizeOf(mm.Page)); - const exact_end = exact_start + (npages * @sizeOf(mm.Page)); - - // 2. Ensure the addresses are aligned on page boundaries. - const map_start = std.mem.alignBackward(usize, exact_start, mm.page_size); - const map_end = std.mem.alignForward(usize, exact_end, mm.page_size); - - // 3. Map the virtual pages. - mi.kernel_space.pmap.map_range_allocating( - map_start, - map_end - map_start, - .{ - .read = true, - .write = true, - .global = true, - }, - .Boot, - ); - } - // Calculate the minimum amount of free memory we accept, // this formula is from Linux and does not grow linearly; it would be wasteful // to take a fixed percentage of memory from a large server. @@ -232,13 +177,12 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void { low_memory_threshold = min_memory_threshold + gap; high_memory_threshold = low_memory_threshold + gap; -} -pub fn init_pfndb() void { + // Initialize the free list from the PFNDB linkage. for (0..memory_map.entry_count) |i| { const entry = memory_map.entries[i]; - if (entry.type != .Free or entry.size < mm.page_size) { + if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) { continue; } @@ -253,6 +197,4 @@ pub fn init_pfndb() void { _ = usable_memory.fetchAdd(mm.page_size, .monotonic); } } - - bootstrapped = true; } diff --git a/src/mm/pmap.zig b/src/mm/pmap.zig index 9c527e4..6222070 100644 --- a/src/mm/pmap.zig +++ b/src/mm/pmap.zig @@ -1,28 +1,4 @@ -//! Generic radix-tree page map implementation. -//! -//! `RadixPmap` is a comptime-parameterized page table walker that implements -//! virtual-to-physical address mapping over a multi-level radix tree (e.g. -//! x86-64's four-level PT). The caller supplies an -//! `Impl` type that describes the tree's level structure and provides -//! platform-specific PTE constructors. -//! -//! The `Impl` type passed to `RadixPmap` must provide: -//! -//! - `levels: []const Level`: ordered from the leaf up to the root, -//! each entry carrying the VA index shift and whether that level can -//! hold a leaf (large-page) mapping. -//! - `is_leaf_level_enabled(level: usize) bool`:runtime gate that -//! controls whether large pages are used at a given level (e.g. to -//! disable 1 GiB pages on hardware that lacks them). -//! - `Pte`: the PTE type. Must be zero-initializable and expose a -//! `present: bool` field and an `address() PAddr` method. -//! - `make_table_pte(pa: PAddr) Pte`: construct a non-leaf PTE pointing -//! to a child page table at `pa`. -//! - `make_leaf_pte(pa: PAddr, flags: MapFlags, level: usize) Pte`: -//! construct a leaf PTE mapping `pa` with the given flags at the -//! chosen level. -//! - `activate(root_pa: PAddr) void`: install the root page table into -//! the CPU (e.g. write CR3 or satp). +//! Generic page map manipulation code. const r = @import("root"); const rtl = @import("rtl"); @@ -37,11 +13,6 @@ const MapItem = struct { flags: mm.MapFlags, }; -const Allocator = enum { - Boot, - Normal, -}; - /// A mapping source that maps a single contiguous virtual-to-physical region. /// All calls to `next` return the same physical base address and size, so this /// is intended to be consumed exactly once by `map_from`. @@ -60,22 +31,6 @@ const ContiguousSource = struct { } }; -/// A mapping source that allocates physical pages on demand. Each call to `next` -/// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped. -const AllocatingSource = struct { - flags: mm.MapFlags, - allocator: Allocator, - - pub fn next(self: *const AllocatingSource, _: r.VAddr) MapItem { - const pa = if (self.allocator == .Boot) mi.phys.early_alloc() else mi.phys.alloc(); - return .{ - .pa = pa, - .len = mm.page_size, - .flags = self.flags, - }; - } -}; - fn level_shift(level: usize) u6 { return mi.impl.levels[level].shift; } @@ -121,19 +76,19 @@ pub const PMap = struct { mi.impl.activate(self.root_pa); } - pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype, comptime allocator: Allocator) void { + pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void { var c = self.cursor(va); var remain = size; while (remain > 0) { const item = source.next(c.va); - c.map_range(item.pa, item.len, item.flags, allocator); + c.map_range(item.pa, item.len, item.flags); remain -|= item.len; } } /// Map a contiguous virtual address range to a contiguous physical address range. - pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void { + pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags) void { const src = ContiguousSource{ .flags = flags, .size = size, @@ -141,17 +96,7 @@ pub const PMap = struct { .base_pa = pa, }; - self.map_from(va, size, src, allocator); - } - - /// Map a virtual address range to physical addresses allocated on demand by the boot allocator. - pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void { - const src = AllocatingSource{ - .allocator = allocator, - .flags = flags, - }; - - self.map_from(va, size, src, allocator); + self.map_from(va, size, src); } pub fn query(self: *Self, va: r.VAddr) ?r.PAddr { @@ -164,7 +109,7 @@ pub const PMap = struct { if (!pte.present) return null; - if (level == 0 or (leaf_level_enabled(level) and pte.huge_page)) { + if (level == 0) { return pte.address() + (va & (page_size(level) - 1)); } @@ -232,7 +177,7 @@ pub const PMap = struct { /// Reset upward in `advance` when the VA crosses a level boundary. top_level: usize, - fn walk_down(self: *Cursor, target_level: usize, allocate: bool, comptime allocator: Allocator) error{PageNotMapped}!void { + fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void { var current_level = self.top_level; while (current_level > target_level) { @@ -244,7 +189,7 @@ pub const PMap = struct { return error.PageNotMapped; } - const new_table_pa = if (allocator == .Boot) mi.phys.early_alloc() else mi.phys.alloc(); + const new_table_pa = mi.phys.alloc(); const table_ptr: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(new_table_pa)); @memset(table_ptr[0..entries_per_table], std.mem.zeroes(mi.impl.Pte)); @@ -261,41 +206,18 @@ pub const PMap = struct { } pub fn walk_down_resolve(self: *Cursor, target_level: usize) bool { - self.walk_down(target_level, false, .Normal) catch return false; + self.walk_down(target_level, false) catch return false; return true; } - /// Choose the largest page size that satisfies the alignment and - /// size constraints of the current VA, PA, and remaining byte count. - /// Tries large pages first; falls back to 4K. - fn choose_target_level(self: *Cursor, pa: usize, remain: usize) usize { - var level: usize = num_levels; - while (level > 0) { - level -= 1; - if (!leaf_level_enabled(level)) continue; - - const pg_size = page_size(level); - const align_mask = pg_size - 1; - - if (remain >= pg_size and - (self.va & align_mask) == 0 and - (pa & align_mask) == 0) - { - return level; - } - } - - @panic("no leaf level can map current alignment/size"); - } - - pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void { + pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags) void { var remain = size; var current_pa = pa; while (remain > 0) { - const target_level = self.choose_target_level(current_pa, remain); + const target_level = 0; - self.walk_down(target_level, true, allocator) catch @panic("walk_down failed during map_range"); + self.walk_down(target_level, true) catch @panic("walk_down failed during map_range"); const table = self.tables[target_level]; table[index_for_level(self.va, target_level)] = diff --git a/src/mm/root.zig b/src/mm/root.zig index 3bad0fa..31147ce 100644 --- a/src/mm/root.zig +++ b/src/mm/root.zig @@ -2,6 +2,7 @@ pub const private = @import("private.zig"); const std = @import("std"); const rtl = @import("rtl"); +const r = @import("root"); pub const page_size = 4096; @@ -30,6 +31,17 @@ pub const Page = extern union { }, }; +comptime { + if (@sizeOf(Page) != r.BootInfo.page_struct_size) { + @compileError( + std.fmt.comptimePrint( + "'Page' size mismatch between loader and kernel: kernel={} loader={}", + .{ @sizeOf(Page), r.BootInfo.page_struct_size }, + ), + ); + } +} + pub const MapFlags = packed struct { read: bool = true, write: bool = false, diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index 2a5b332..ab7ca92 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -55,7 +55,7 @@ pub fn init() linksection(r.init) void { .read = true, .write = true, .execute = true, - }, .Normal); + }); // From Linux, on modern CPUs we can skip the long delay after INIT. const skip_delay = switch (amd64.cpu_features.vendor) {