loader: map pfndb and cleanup

src/loader/amd64/impl.zig

@@ -97,6 +97,10 @@ pub inline fn activate(root_pa: usize) void {
     amd64.write_cr(3, root_pa);
 }
 
+pub inline fn flush(va: usize) void {
+    amd64.invlpg(va);
+}
+
 pub fn is_leaf_level_enabled(level: usize) bool {
     if (!levels[level].leaf) return false;
     if (level == 2) return amd64.cpu_features.gib_pages;
@@ -106,3 +110,7 @@ pub fn is_leaf_level_enabled(level: usize) bool {
 pub fn debug_write(c: u8) void {
     amd64.outb(0xe9, c);
 }
+
+pub fn init() void {
+    amd64.detect_cpu_features();
+}

src/loader/info.zig

@@ -6,6 +6,7 @@ pub const MemMap = struct {
             Free,
             Reserved,
             LoaderReclaimable,
+            LoaderData,
             AcpiNvs,
             AcpiReclaimable,
         };
@@ -16,6 +17,7 @@ pub const MemMap = struct {
     };
 
     entry_count: usize,
+    loader_memory_used: usize,
     entries: [128]Entry,
 };
 
@@ -27,6 +29,9 @@ pub const Framebuffer = struct {
     bpp: u8,
 };
 
+// Kept in sync with the kernel
+pub const page_struct_size = 16;
+
 pub const KernelAddress = struct { physical_base: usize, virtual_base: usize };
 
 /// RSDP on ACPI machines

src/loader/limine/impl.zig

@@ -65,8 +65,6 @@ pub fn get_image_layout() r.ImageLayout {
 export fn loader_entry() callconv(.c) void {
     const mmap = memmap_request.response.?;
 
-    std.log.info("hi", .{});
-
     for (0..mmap.entry_count) |i| {
         const entry = mmap.entries.?[i];
 

src/loader/main.zig

@@ -1,6 +1,11 @@
 const r = @import("root");
 const std = @import("std");
 
+const Entry = r.BootInfo.MemMap.Entry;
+const stack_size = 1024 * 16; // should be enough
+
+extern fn jump_to_kernel(loader_info: usize, stack: usize, entry: usize) void;
+
 fn load_kernel(kernel: *anyopaque) usize {
     const elf: *std.elf.Elf64.Ehdr = @ptrCast(@alignCast(kernel));
     const ident = elf.ident;
@@ -22,26 +27,49 @@ fn load_kernel(kernel: *anyopaque) usize {
     for (0..elf.phnum) |_| {
         switch (phdr.type) {
             .LOAD => {
-                r.mem.pagemap.map_range_allocating(phdr.vaddr, std.mem.alignForward(usize, phdr.memsz, r.page_size), .{
+                const as_pages = std.mem.alignForward(usize, phdr.memsz, r.page_size);
-                    .execute = phdr.flags.X,
+                const npages = as_pages / r.page_size;
-                    // Gotta have RW access for loading it.
+                var filesz_rem = phdr.filesz;
-                    // XXX remap the pages and set protections properly...
-                    .read = true,
-                    .write = true,
-                    .global = true,
-                });
 
-                const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset);
+                for (0..npages) |i| {
-                const source = off[0..phdr.filesz];
+                    const pa = r.mem.alloc_page();
-                const va: [*]u8 = @ptrFromInt(phdr.vaddr);
+                    const addr = phdr.vaddr + i * r.page_size;
-                const dest = va[0..phdr.filesz];
 
-                // Copy the file data.
+                    r.mem.pagemap.map_page(addr, pa, .{
-                @memcpy(dest, source);
+                        // Gotta have RW access for loading it.
+                        .read = true,
+                        .write = true,
+                    });
 
-                // Fill the rest of memsz with 0.
+                    const file_remaining = @min(filesz_rem, r.page_size);
-                const zero_dest = va[phdr.filesz..phdr.memsz];
+
-                @memset(zero_dest, 0);
+                    const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset + i * r.page_size);
+                    const source = off[0..file_remaining];
+                    const va: [*]u8 = @ptrFromInt(addr);
+                    const dest = va[0..file_remaining];
+
+                    filesz_rem -= file_remaining;
+
+                    // Copy the file data.
+                    @memcpy(dest, source);
+
+                    if (filesz_rem == 0) {
+                        const zeroes = phdr.memsz - phdr.filesz;
+
+                        // Fill the rest of memsz with 0.
+                        @memset(va[0..zeroes], 0);
+                    }
+
+                    // Now update the page permissions to their true value.
+                    r.mem.pagemap.map_page(addr, pa, .{
+                        .execute = phdr.flags.X,
+                        .read = phdr.flags.R,
+                        .write = phdr.flags.W,
+                        .global = true,
+                    });
+
+                    r.arch.flush(addr);
+                }
             },
             else => {},
         }
@@ -52,13 +80,21 @@ fn load_kernel(kernel: *anyopaque) usize {
     return elf.entry;
 }
 
-extern fn jump_to_kernel(loader_info: usize, stack: usize, entry: usize) void;
+fn cmp_entries(ctx: void, a: Entry, b: Entry) bool {
+    _ = ctx;
+    return a.base < b.base;
+}
 
 pub fn loader_main(kernel: *anyopaque) void {
+    r.arch.init();
     r.mem.init();
     const entry = load_kernel(kernel);
-    const stack = r.mem.alloc(1024 * 16);
+    const stack = r.mem.alloc(stack_size);
 
-    std.log.info("jumping to {x} with stack {x}", .{ entry, stack });
+    const memmap = &r.loader_info.memory_map;
-    jump_to_kernel(@intFromPtr(&r.loader_info), stack + (1024 * 16), entry);
+
+    // Finally, sort the memory map once we're done allocating everything.
+    std.mem.sort(Entry, memmap.entries[0..memmap.entry_count], {}, cmp_entries);
+
+    jump_to_kernel(@intFromPtr(&r.loader_info), stack + stack_size, entry);
 }

src/loader/mem.zig

@@ -1,11 +1,13 @@
 const r = @import("root");
 const config = @import("config");
 const pmap = @import("pmap.zig");
+const std = @import("std");
 
 var memory_map: *r.BootInfo.MemMap = &r.loader_info.memory_map;
 var alloc_idx: usize = 0;
 var bump_base: usize = 0;
 var map_loaded: bool = false;
+var last_data_idx: ?usize = null;
 
 const MemoryLayout = struct {
     /// Kernel direct map (if applicable)
@@ -40,6 +42,19 @@ pub fn p2v(pa: usize) usize {
     return r.impl.p2v(pa);
 }
 
+fn record_loader_data(addr: usize) void {
+    if (last_data_idx) |i| {
+        const e = &memory_map.entries[i];
+        if (e.base + e.size == addr) {
+            e.size += r.page_size;
+            return;
+        }
+    }
+
+    last_data_idx = memory_map.entry_count;
+    add_entry(addr, r.page_size, .LoaderData);
+}
+
 pub fn alloc_page() usize {
     while (alloc_idx < memory_map.entry_count) {
         const entry = &memory_map.entries[alloc_idx];
@@ -53,6 +68,8 @@ pub fn alloc_page() usize {
 
         entry.base += r.page_size;
         entry.size -= r.page_size;
+        memory_map.loader_memory_used += r.page_size;
+        record_loader_data(addr);
 
         const a: [*]u64 = @ptrFromInt(p2v(addr));
 
@@ -205,6 +222,41 @@ fn map_self() void {
     }
 }
 
+fn map_pfndb() void {
+    const size = r.BootInfo.page_struct_size;
+
+    // Now map the PFN database.
+    for (0..memory_map.entry_count) |i| {
+        const entry = memory_map.entries[i];
+
+        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) {
+            continue;
+        }
+
+        const npages = std.math.divCeil(usize, entry.size, r.page_size) catch unreachable;
+
+        // 1. Calculate the exact virtual address range needed for this region's page structs.
+        const start_pfn: usize = entry.base / r.page_size;
+        const exact_start = memory_layout.pfndb + (start_pfn * size);
+        const exact_end = exact_start + (npages * size);
+
+        // 2. Ensure the addresses are aligned on page boundaries.
+        const map_start = std.mem.alignBackward(usize, exact_start, r.page_size);
+        const map_end = std.mem.alignForward(usize, exact_end, r.page_size);
+
+        // 3. Map the virtual pages.
+        pagemap.map_range_allocating(
+            map_start,
+            map_end - map_start,
+            .{
+                .read = true,
+                .write = true,
+                .global = true,
+            },
+        );
+    }
+}
+
 pub fn init() void {
     const vbits = r.arch.virtual_bits;
 
@@ -240,6 +292,7 @@ pub fn init() void {
     }
 
     map_self();
+    map_pfndb();
 
     map_loaded = true;
     r.arch.activate(pagemap.root_pa);

src/loader/pmap.zig

@@ -1,3 +1,6 @@
+//! Generic page map manipulation code.
+//! Part of the code is duplicated from the kernel :(
+
 const r = @import("root");
 const mem = r.mem;
 
@@ -167,11 +170,6 @@ pub const PMap = struct {
             }
         }
 
-        pub fn walk_down_resolve(self: *Cursor, target_level: usize) bool {
-            self.walk_down(target_level, false, false) catch return false;
-            return true;
-        }
-
         /// Choose the largest page size that satisfies the alignment and
         /// size constraints of the current VA, PA, and remaining byte count.
         /// Tries large pages first; falls back to 4K.

src/mm/amd64/impl.zig

@@ -14,8 +14,8 @@ pub fn virt_to_phys(vaddr: r.VAddr) usize {
 }
 
 pub const hhdm_base = 0xffff800000000000;
-pub const kernel_heap_base = 0xffffe00000000000;
+pub const kernel_heap_base = 0xffffc00000000000;
-pub const pfndb_base = 0xffffff0000000000;
+pub const pfndb_base = 0xffffd00000000000;
 
 pub const levels = [_]mi.PMapLevel{
     .{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K

src/mm/init.zig

@@ -6,8 +6,6 @@ const config = @import("config");
 
 pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
     mi.phys.init(boot_info);
-    mi.kernel_space.pmap.activate();
-    mi.phys.init_pfndb();
     mi.zone.early_init();
     mi.vmem.init();
     mi.heap.init();

src/mm/phys.zig

@@ -18,8 +18,6 @@ pub var low_memory_threshold: usize = 0;
 /// Threshold at which reclamation stops.
 pub var high_memory_threshold: usize = 0;
 
-var bootstrapped = false;
-var early_alloc_entry_idx: usize = 0;
 var memory_map: *r.BootInfo.MemMap = undefined;
 var early_allocs: usize = 0;
 var list_lock: ke.SpinLock = .init();
@@ -31,27 +29,6 @@ var free_pages: usize = 0;
 
 pub var usable_memory: std.atomic.Value(usize) = .init(0);
 
-pub fn early_alloc() usize {
-    while (early_alloc_entry_idx < memory_map.entry_count) {
-        const entry = &memory_map.entries[early_alloc_entry_idx];
-
-        if (entry.type != .Free or entry.size < mm.page_size) {
-            early_alloc_entry_idx += 1;
-            continue;
-        }
-
-        const addr = entry.base;
-
-        entry.base += mm.page_size;
-        entry.size -= mm.page_size;
-        early_allocs += mm.page_size;
-
-        return addr;
-    }
-
-    @panic("mm/phys: Out of memory for early allocation");
-}
-
 // Wait for pages to be available.
 // List lock is held.
 fn wait_for_pages(old_ipl: ke.Ipl) void {
@@ -167,7 +144,7 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
     free_list.init();
     free_page_event.init(.Notification);
 
-    var total_usable_memory: usize = 0;
+    var total_usable_memory: usize = boot_info.memory_map.loader_memory_used;
     log.info("physical memory map:", .{});
 
     for (0..boot_info.memory_map.entry_count) |i| {
@@ -175,50 +152,18 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
 
         log.info("[{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) });
 
-        if (entry.type == .Free) {
+        if (entry.type == .Free or entry.type == .LoaderReclaimable) {
             total_usable_memory += entry.size;
         }
     }
 
     const pfndb_size_required = @sizeOf(mm.Page) * (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable);
-
     log.info("using {} KiB for pfndb ({} bytes per page)", .{ pfndb_size_required / 1024, @sizeOf(mm.Page) });
 
-    // Create the kernel pagemap from the early allocator.
+    // Get the kernel pagemap from the loader,
+    // we are already in it so no need to activate it.
     mi.impl.init_kernel();
 
-    // Now map the PFN database.
-    for (0..boot_info.memory_map.entry_count) |i| {
-        const entry = boot_info.memory_map.entries[i];
-
-        if (entry.type != .Free or entry.size < mm.page_size) {
-            continue;
-        }
-
-        const npages = std.math.divCeil(usize, entry.size, mm.page_size) catch unreachable;
-
-        // 1. Calculate the exact virtual address range needed for this region's page structs.
-        const start_pfn: usize = mm.page_to_pfn(entry.base);
-        const exact_start = mi.impl.pfndb_base + (start_pfn * @sizeOf(mm.Page));
-        const exact_end = exact_start + (npages * @sizeOf(mm.Page));
-
-        // 2. Ensure the addresses are aligned on page boundaries.
-        const map_start = std.mem.alignBackward(usize, exact_start, mm.page_size);
-        const map_end = std.mem.alignForward(usize, exact_end, mm.page_size);
-
-        // 3. Map the virtual pages.
-        mi.kernel_space.pmap.map_range_allocating(
-            map_start,
-            map_end - map_start,
-            .{
-                .read = true,
-                .write = true,
-                .global = true,
-            },
-            .Boot,
-        );
-    }
-
     // Calculate the minimum amount of free memory we accept,
     // this formula is from Linux and does not grow linearly; it would be wasteful
     // to take a fixed percentage of memory from a large server.
@@ -232,13 +177,12 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
 
     low_memory_threshold = min_memory_threshold + gap;
     high_memory_threshold = low_memory_threshold + gap;
-}
 
-pub fn init_pfndb() void {
+    // Initialize the free list from the PFNDB linkage.
     for (0..memory_map.entry_count) |i| {
         const entry = memory_map.entries[i];
 
-        if (entry.type != .Free or entry.size < mm.page_size) {
+        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) {
             continue;
         }
 
@@ -253,6 +197,4 @@ pub fn init_pfndb() void {
             _ = usable_memory.fetchAdd(mm.page_size, .monotonic);
         }
     }
-
-    bootstrapped = true;
 }

src/mm/pmap.zig

@@ -1,28 +1,4 @@
-//! Generic radix-tree page map implementation.
+//! Generic page map manipulation code.
-//!
-//! `RadixPmap` is a comptime-parameterized page table walker that implements
-//! virtual-to-physical address mapping over a multi-level radix tree (e.g.
-//! x86-64's four-level PT). The caller supplies an
-//! `Impl` type that describes the tree's level structure and provides
-//! platform-specific PTE constructors.
-//!
-//! The `Impl` type passed to `RadixPmap` must provide:
-//!
-//!   - `levels: []const Level`: ordered from the leaf up to the root,
-//!     each entry carrying the VA index shift and whether that level can
-//!     hold a leaf (large-page) mapping.
-//!   - `is_leaf_level_enabled(level: usize) bool`:runtime gate that
-//!     controls whether large pages are used at a given level (e.g. to
-//!     disable 1 GiB pages on hardware that lacks them).
-//!   - `Pte`: the PTE type. Must be zero-initializable and expose a
-//!     `present: bool` field and an `address() PAddr` method.
-//!   - `make_table_pte(pa: PAddr) Pte`: construct a non-leaf PTE pointing
-//!     to a child page table at `pa`.
-//!   - `make_leaf_pte(pa: PAddr, flags: MapFlags, level: usize) Pte`:
-//!     construct a leaf PTE mapping `pa` with the given flags at the
-//!     chosen level.
-//!   - `activate(root_pa: PAddr) void`: install the root page table into
-//!     the CPU (e.g. write CR3 or satp).
 
 const r = @import("root");
 const rtl = @import("rtl");
@@ -37,11 +13,6 @@ const MapItem = struct {
     flags: mm.MapFlags,
 };
 
-const Allocator = enum {
-    Boot,
-    Normal,
-};
-
 /// A mapping source that maps a single contiguous virtual-to-physical region.
 /// All calls to `next` return the same physical base address and size, so this
 /// is intended to be consumed exactly once by `map_from`.
@@ -60,22 +31,6 @@ const ContiguousSource = struct {
     }
 };
 
-/// A mapping source that allocates physical pages on demand. Each call to `next`
-/// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped.
-const AllocatingSource = struct {
-    flags: mm.MapFlags,
-    allocator: Allocator,
-
-    pub fn next(self: *const AllocatingSource, _: r.VAddr) MapItem {
-        const pa = if (self.allocator == .Boot) mi.phys.early_alloc() else mi.phys.alloc();
-        return .{
-            .pa = pa,
-            .len = mm.page_size,
-            .flags = self.flags,
-        };
-    }
-};
-
 fn level_shift(level: usize) u6 {
     return mi.impl.levels[level].shift;
 }
@@ -121,19 +76,19 @@ pub const PMap = struct {
         mi.impl.activate(self.root_pa);
     }
 
-    pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype, comptime allocator: Allocator) void {
+    pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void {
         var c = self.cursor(va);
         var remain = size;
 
         while (remain > 0) {
             const item = source.next(c.va);
-            c.map_range(item.pa, item.len, item.flags, allocator);
+            c.map_range(item.pa, item.len, item.flags);
             remain -|= item.len;
         }
     }
 
     /// Map a contiguous virtual address range to a contiguous physical address range.
-    pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void {
+    pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags) void {
         const src = ContiguousSource{
             .flags = flags,
             .size = size,
@@ -141,17 +96,7 @@ pub const PMap = struct {
             .base_pa = pa,
         };
 
-        self.map_from(va, size, src, allocator);
+        self.map_from(va, size, src);
-    }
-
-    /// Map a virtual address range to physical addresses allocated on demand by the boot allocator.
-    pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void {
-        const src = AllocatingSource{
-            .allocator = allocator,
-            .flags = flags,
-        };
-
-        self.map_from(va, size, src, allocator);
     }
 
     pub fn query(self: *Self, va: r.VAddr) ?r.PAddr {
@@ -164,7 +109,7 @@ pub const PMap = struct {
 
             if (!pte.present) return null;
 
-            if (level == 0 or (leaf_level_enabled(level) and pte.huge_page)) {
+            if (level == 0) {
                 return pte.address() + (va & (page_size(level) - 1));
             }
 
@@ -232,7 +177,7 @@ pub const PMap = struct {
         /// Reset upward in `advance` when the VA crosses a level boundary.
         top_level: usize,
 
-        fn walk_down(self: *Cursor, target_level: usize, allocate: bool, comptime allocator: Allocator) error{PageNotMapped}!void {
+        fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void {
             var current_level = self.top_level;
 
             while (current_level > target_level) {
@@ -244,7 +189,7 @@ pub const PMap = struct {
                         return error.PageNotMapped;
                     }
 
-                    const new_table_pa = if (allocator == .Boot) mi.phys.early_alloc() else mi.phys.alloc();
+                    const new_table_pa = mi.phys.alloc();
 
                     const table_ptr: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(new_table_pa));
                     @memset(table_ptr[0..entries_per_table], std.mem.zeroes(mi.impl.Pte));
@@ -261,41 +206,18 @@ pub const PMap = struct {
         }
 
         pub fn walk_down_resolve(self: *Cursor, target_level: usize) bool {
-            self.walk_down(target_level, false, .Normal) catch return false;
+            self.walk_down(target_level, false) catch return false;
             return true;
         }
 
-        /// Choose the largest page size that satisfies the alignment and
+        pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags) void {
-        /// size constraints of the current VA, PA, and remaining byte count.
-        /// Tries large pages first; falls back to 4K.
-        fn choose_target_level(self: *Cursor, pa: usize, remain: usize) usize {
-            var level: usize = num_levels;
-            while (level > 0) {
-                level -= 1;
-                if (!leaf_level_enabled(level)) continue;
-
-                const pg_size = page_size(level);
-                const align_mask = pg_size - 1;
-
-                if (remain >= pg_size and
-                    (self.va & align_mask) == 0 and
-                    (pa & align_mask) == 0)
-                {
-                    return level;
-                }
-            }
-
-            @panic("no leaf level can map current alignment/size");
-        }
-
-        pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void {
             var remain = size;
             var current_pa = pa;
 
             while (remain > 0) {
-                const target_level = self.choose_target_level(current_pa, remain);
+                const target_level = 0;
 
-                self.walk_down(target_level, true, allocator) catch @panic("walk_down failed during map_range");
+                self.walk_down(target_level, true) catch @panic("walk_down failed during map_range");
 
                 const table = self.tables[target_level];
                 table[index_for_level(self.va, target_level)] =

src/mm/root.zig

@@ -2,6 +2,7 @@ pub const private = @import("private.zig");
 
 const std = @import("std");
 const rtl = @import("rtl");
+const r = @import("root");
 
 pub const page_size = 4096;
 
@@ -30,6 +31,17 @@ pub const Page = extern union {
     },
 };
 
+comptime {
+    if (@sizeOf(Page) != r.BootInfo.page_struct_size) {
+        @compileError(
+            std.fmt.comptimePrint(
+                "'Page' size mismatch between loader and kernel: kernel={} loader={}",
+                .{ @sizeOf(Page), r.BootInfo.page_struct_size },
+            ),
+        );
+    }
+}
+
 pub const MapFlags = packed struct {
     read: bool = true,
     write: bool = false,

src/platform/pc/smp.zig

@@ -55,7 +55,7 @@ pub fn init() linksection(r.init) void {
         .read = true,
         .write = true,
         .execute = true,
-    }, .Normal);
+    });
 
     // From Linux, on modern CPUs we can skip the long delay after INIT.
     const skip_delay = switch (amd64.cpu_features.vendor) {