mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-10-27 13:58:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

loader: map pfndb and cleanup

Browse Source
This commit is contained in:
Loïc Grégoire
2026-07-18 14:06:36 -04:00
parent 3c72101228
commit 96d2bee857
12 changed files with 159 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/loader/amd64/impl.zig
+8
View File
@@ -97,6 +97,10 @@ pub inline fn activate(root_pa: usize) void {
amd64.write_cr(3, root_pa);
}
pub inline fn flush(va: usize) void {
amd64.invlpg(va);
}
pub fn is_leaf_level_enabled(level: usize) bool {
if (!levels[level].leaf) return false;
if (level == 2) return amd64.cpu_features.gib_pages;
@@ -106,3 +110,7 @@ pub fn is_leaf_level_enabled(level: usize) bool {
pub fn debug_write(c: u8) void {
amd64.outb(0xe9, c);
}
pub fn init() void {
amd64.detect_cpu_features();
}
src/loader/info.zig
+5
View File
@@ -6,6 +6,7 @@ pub const MemMap = struct {
Free,
Reserved,
LoaderReclaimable,
LoaderData,
AcpiNvs,
AcpiReclaimable,
};
@@ -16,6 +17,7 @@ pub const MemMap = struct {
};
entry_count: usize,
loader_memory_used: usize,
entries: [128]Entry,
};
@@ -27,6 +29,9 @@ pub const Framebuffer = struct {
bpp: u8,
};
// Kept in sync with the kernel
pub const page_struct_size = 16;
pub const KernelAddress = struct { physical_base: usize, virtual_base: usize };
/// RSDP on ACPI machines
src/loader/limine/impl.zig
-2
View File
@@ -65,8 +65,6 @@ pub fn get_image_layout() r.ImageLayout {
export fn loader_entry() callconv(.c) void {
const mmap = memmap_request.response.?;
std.log.info("hi", .{});
for (0..mmap.entry_count) |i| {
const entry = mmap.entries.?[i];
src/loader/main.zig
+50 -14
View File
@@ -1,6 +1,11 @@
const r = @import("root");
const std = @import("std");
const Entry = r.BootInfo.MemMap.Entry;
const stack_size = 1024 * 16; // should be enough
extern fn jump_to_kernel(loader_info: usize, stack: usize, entry: usize) void;
fn load_kernel(kernel: *anyopaque) usize {
const elf: *std.elf.Elf64.Ehdr = @ptrCast(@alignCast(kernel));
const ident = elf.ident;
@@ -22,26 +27,49 @@ fn load_kernel(kernel: *anyopaque) usize {
for (0..elf.phnum) |_| {
switch (phdr.type) {
.LOAD => {
r.mem.pagemap.map_range_allocating(phdr.vaddr, std.mem.alignForward(usize, phdr.memsz, r.page_size), .{
.execute = phdr.flags.X,
const as_pages = std.mem.alignForward(usize, phdr.memsz, r.page_size);
const npages = as_pages / r.page_size;
var filesz_rem = phdr.filesz;
for (0..npages) |i| {
const pa = r.mem.alloc_page();
const addr = phdr.vaddr + i * r.page_size;
r.mem.pagemap.map_page(addr, pa, .{
// Gotta have RW access for loading it.
// XXX remap the pages and set protections properly...
.read = true,
.write = true,
.global = true,
});
const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset);
const source = off[0..phdr.filesz];
const va: [*]u8 = @ptrFromInt(phdr.vaddr);
const dest = va[0..phdr.filesz];
const file_remaining = @min(filesz_rem, r.page_size);
const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset + i * r.page_size);
const source = off[0..file_remaining];
const va: [*]u8 = @ptrFromInt(addr);
const dest = va[0..file_remaining];
filesz_rem -= file_remaining;
// Copy the file data.
@memcpy(dest, source);
if (filesz_rem == 0) {
const zeroes = phdr.memsz - phdr.filesz;
// Fill the rest of memsz with 0.
const zero_dest = va[phdr.filesz..phdr.memsz];
@memset(zero_dest, 0);
@memset(va[0..zeroes], 0);
}
// Now update the page permissions to their true value.
r.mem.pagemap.map_page(addr, pa, .{
.execute = phdr.flags.X,
.read = phdr.flags.R,
.write = phdr.flags.W,
.global = true,
});
r.arch.flush(addr);
}
},
else => {},
}
@@ -52,13 +80,21 @@ fn load_kernel(kernel: *anyopaque) usize {
return elf.entry;
}
extern fn jump_to_kernel(loader_info: usize, stack: usize, entry: usize) void;
fn cmp_entries(ctx: void, a: Entry, b: Entry) bool {
_ = ctx;
return a.base < b.base;
}
pub fn loader_main(kernel: *anyopaque) void {
r.arch.init();
r.mem.init();
const entry = load_kernel(kernel);
const stack = r.mem.alloc(1024 * 16);
const stack = r.mem.alloc(stack_size);
std.log.info("jumping to {x} with stack {x}", .{ entry, stack });
jump_to_kernel(@intFromPtr(&r.loader_info), stack + (1024 * 16), entry);
const memmap = &r.loader_info.memory_map;
// Finally, sort the memory map once we're done allocating everything.
std.mem.sort(Entry, memmap.entries[0..memmap.entry_count], {}, cmp_entries);
jump_to_kernel(@intFromPtr(&r.loader_info), stack + stack_size, entry);
}
src/loader/mem.zig
+53
View File
@@ -1,11 +1,13 @@
const r = @import("root");
const config = @import("config");
const pmap = @import("pmap.zig");
const std = @import("std");
var memory_map: *r.BootInfo.MemMap = &r.loader_info.memory_map;
var alloc_idx: usize = 0;
var bump_base: usize = 0;
var map_loaded: bool = false;
var last_data_idx: ?usize = null;
const MemoryLayout = struct {
/// Kernel direct map (if applicable)
@@ -40,6 +42,19 @@ pub fn p2v(pa: usize) usize {
return r.impl.p2v(pa);
}
fn record_loader_data(addr: usize) void {
if (last_data_idx) |i| {
const e = &memory_map.entries[i];
if (e.base + e.size == addr) {
e.size += r.page_size;
return;
}
}
last_data_idx = memory_map.entry_count;
add_entry(addr, r.page_size, .LoaderData);
}
pub fn alloc_page() usize {
while (alloc_idx < memory_map.entry_count) {
const entry = &memory_map.entries[alloc_idx];
@@ -53,6 +68,8 @@ pub fn alloc_page() usize {
entry.base += r.page_size;
entry.size -= r.page_size;
memory_map.loader_memory_used += r.page_size;
record_loader_data(addr);
const a: [*]u64 = @ptrFromInt(p2v(addr));
@@ -205,6 +222,41 @@ fn map_self() void {
}
}
fn map_pfndb() void {
const size = r.BootInfo.page_struct_size;
// Now map the PFN database.
for (0..memory_map.entry_count) |i| {
const entry = memory_map.entries[i];
if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) {
continue;
}
const npages = std.math.divCeil(usize, entry.size, r.page_size) catch unreachable;
// 1. Calculate the exact virtual address range needed for this region's page structs.
const start_pfn: usize = entry.base / r.page_size;
const exact_start = memory_layout.pfndb + (start_pfn * size);
const exact_end = exact_start + (npages * size);
// 2. Ensure the addresses are aligned on page boundaries.
const map_start = std.mem.alignBackward(usize, exact_start, r.page_size);
const map_end = std.mem.alignForward(usize, exact_end, r.page_size);
// 3. Map the virtual pages.
pagemap.map_range_allocating(
map_start,
map_end - map_start,
.{
.read = true,
.write = true,
.global = true,
},
);
}
}
pub fn init() void {
const vbits = r.arch.virtual_bits;
@@ -240,6 +292,7 @@ pub fn init() void {
}
map_self();
map_pfndb();
map_loaded = true;
r.arch.activate(pagemap.root_pa);
src/loader/pmap.zig
+3 -5
View File
@@ -1,3 +1,6 @@
//! Generic page map manipulation code.
//! Part of the code is duplicated from the kernel :(
const r = @import("root");
const mem = r.mem;
@@ -167,11 +170,6 @@ pub const PMap = struct {
}
}
pub fn walk_down_resolve(self: *Cursor, target_level: usize) bool {
self.walk_down(target_level, false, false) catch return false;
return true;
}
/// Choose the largest page size that satisfies the alignment and
/// size constraints of the current VA, PA, and remaining byte count.
/// Tries large pages first; falls back to 4K.
src/mm/amd64/impl.zig
+2 -2
View File
@@ -14,8 +14,8 @@ pub fn virt_to_phys(vaddr: r.VAddr) usize {
}
pub const hhdm_base = 0xffff800000000000;
pub const kernel_heap_base = 0xffffe00000000000;
pub const pfndb_base = 0xffffff0000000000;
pub const kernel_heap_base = 0xffffc00000000000;
pub const pfndb_base = 0xffffd00000000000;
pub const levels = [_]mi.PMapLevel{
.{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K
src/mm/init.zig
-2
View File
@@ -6,8 +6,6 @@ const config = @import("config");
pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
mi.phys.init(boot_info);
mi.kernel_space.pmap.activate();
mi.phys.init_pfndb();
mi.zone.early_init();
mi.vmem.init();
mi.heap.init();
src/mm/phys.zig
+6 -64
View File
@@ -18,8 +18,6 @@ pub var low_memory_threshold: usize = 0;
/// Threshold at which reclamation stops.
pub var high_memory_threshold: usize = 0;
var bootstrapped = false;
var early_alloc_entry_idx: usize = 0;
var memory_map: *r.BootInfo.MemMap = undefined;
var early_allocs: usize = 0;
var list_lock: ke.SpinLock = .init();
@@ -31,27 +29,6 @@ var free_pages: usize = 0;
pub var usable_memory: std.atomic.Value(usize) = .init(0);
pub fn early_alloc() usize {
while (early_alloc_entry_idx < memory_map.entry_count) {
const entry = &memory_map.entries[early_alloc_entry_idx];
if (entry.type != .Free or entry.size < mm.page_size) {
early_alloc_entry_idx += 1;
continue;
}
const addr = entry.base;
entry.base += mm.page_size;
entry.size -= mm.page_size;
early_allocs += mm.page_size;
return addr;
}
@panic("mm/phys: Out of memory for early allocation");
}
// Wait for pages to be available.
// List lock is held.
fn wait_for_pages(old_ipl: ke.Ipl) void {
@@ -167,7 +144,7 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
free_list.init();
free_page_event.init(.Notification);
var total_usable_memory: usize = 0;
var total_usable_memory: usize = boot_info.memory_map.loader_memory_used;
log.info("physical memory map:", .{});
for (0..boot_info.memory_map.entry_count) |i| {
@@ -175,50 +152,18 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
log.info("[{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) });
if (entry.type == .Free) {
if (entry.type == .Free or entry.type == .LoaderReclaimable) {
total_usable_memory += entry.size;
}
}
const pfndb_size_required = @sizeOf(mm.Page) * (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable);
log.info("using {} KiB for pfndb ({} bytes per page)", .{ pfndb_size_required / 1024, @sizeOf(mm.Page) });
// Create the kernel pagemap from the early allocator.
// Get the kernel pagemap from the loader,
// we are already in it so no need to activate it.
mi.impl.init_kernel();
// Now map the PFN database.
for (0..boot_info.memory_map.entry_count) |i| {
const entry = boot_info.memory_map.entries[i];
if (entry.type != .Free or entry.size < mm.page_size) {
continue;
}
const npages = std.math.divCeil(usize, entry.size, mm.page_size) catch unreachable;
// 1. Calculate the exact virtual address range needed for this region's page structs.
const start_pfn: usize = mm.page_to_pfn(entry.base);
const exact_start = mi.impl.pfndb_base + (start_pfn * @sizeOf(mm.Page));
const exact_end = exact_start + (npages * @sizeOf(mm.Page));
// 2. Ensure the addresses are aligned on page boundaries.
const map_start = std.mem.alignBackward(usize, exact_start, mm.page_size);
const map_end = std.mem.alignForward(usize, exact_end, mm.page_size);
// 3. Map the virtual pages.
mi.kernel_space.pmap.map_range_allocating(
map_start,
map_end - map_start,
.{
.read = true,
.write = true,
.global = true,
},
.Boot,
);
}
// Calculate the minimum amount of free memory we accept,
// this formula is from Linux and does not grow linearly; it would be wasteful
// to take a fixed percentage of memory from a large server.
@@ -232,13 +177,12 @@ pub fn init(boot_info: *r.BootInfo) linksection(r.init) void {
low_memory_threshold = min_memory_threshold + gap;
high_memory_threshold = low_memory_threshold + gap;
}
pub fn init_pfndb() void {
// Initialize the free list from the PFNDB linkage.
for (0..memory_map.entry_count) |i| {
const entry = memory_map.entries[i];
if (entry.type != .Free or entry.size < mm.page_size) {
if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) {
continue;
}
@@ -253,6 +197,4 @@ pub fn init_pfndb() void {
_ = usable_memory.fetchAdd(mm.page_size, .monotonic);
}
}
bootstrapped = true;
}
src/mm/pmap.zig
+12 -90
View File
@@ -1,28 +1,4 @@
//! Generic radix-tree page map implementation.
//!
//! `RadixPmap` is a comptime-parameterized page table walker that implements
//! virtual-to-physical address mapping over a multi-level radix tree (e.g.
//! x86-64's four-level PT). The caller supplies an
//! `Impl` type that describes the tree's level structure and provides
//! platform-specific PTE constructors.
//!
//! The `Impl` type passed to `RadixPmap` must provide:
//!
//! - `levels: []const Level`: ordered from the leaf up to the root,
//! each entry carrying the VA index shift and whether that level can
//! hold a leaf (large-page) mapping.
//! - `is_leaf_level_enabled(level: usize) bool`:runtime gate that
//! controls whether large pages are used at a given level (e.g. to
//! disable 1 GiB pages on hardware that lacks them).
//! - `Pte`: the PTE type. Must be zero-initializable and expose a
//! `present: bool` field and an `address() PAddr` method.
//! - `make_table_pte(pa: PAddr) Pte`: construct a non-leaf PTE pointing
//! to a child page table at `pa`.
//! - `make_leaf_pte(pa: PAddr, flags: MapFlags, level: usize) Pte`:
//! construct a leaf PTE mapping `pa` with the given flags at the
//! chosen level.
//! - `activate(root_pa: PAddr) void`: install the root page table into
//! the CPU (e.g. write CR3 or satp).
//! Generic page map manipulation code.
const r = @import("root");
const rtl = @import("rtl");
@@ -37,11 +13,6 @@ const MapItem = struct {
flags: mm.MapFlags,
};
const Allocator = enum {
Boot,
Normal,
};
/// A mapping source that maps a single contiguous virtual-to-physical region.
/// All calls to `next` return the same physical base address and size, so this
/// is intended to be consumed exactly once by `map_from`.
@@ -60,22 +31,6 @@ const ContiguousSource = struct {
}
};
/// A mapping source that allocates physical pages on demand. Each call to `next`
/// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped.
const AllocatingSource = struct {
flags: mm.MapFlags,
allocator: Allocator,
pub fn next(self: *const AllocatingSource, _: r.VAddr) MapItem {
const pa = if (self.allocator == .Boot) mi.phys.early_alloc() else mi.phys.alloc();
return .{
.pa = pa,
.len = mm.page_size,
.flags = self.flags,
};
}
};
fn level_shift(level: usize) u6 {
return mi.impl.levels[level].shift;
}
@@ -121,19 +76,19 @@ pub const PMap = struct {
mi.impl.activate(self.root_pa);
}
pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype, comptime allocator: Allocator) void {
pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void {
var c = self.cursor(va);
var remain = size;
while (remain > 0) {
const item = source.next(c.va);
c.map_range(item.pa, item.len, item.flags, allocator);
c.map_range(item.pa, item.len, item.flags);
remain -|= item.len;
}
}
/// Map a contiguous virtual address range to a contiguous physical address range.
pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void {
pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags) void {
const src = ContiguousSource{
.flags = flags,
.size = size,
@@ -141,17 +96,7 @@ pub const PMap = struct {
.base_pa = pa,
};
self.map_from(va, size, src, allocator);
}
/// Map a virtual address range to physical addresses allocated on demand by the boot allocator.
pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void {
const src = AllocatingSource{
.allocator = allocator,
.flags = flags,
};
self.map_from(va, size, src, allocator);
self.map_from(va, size, src);
}
pub fn query(self: *Self, va: r.VAddr) ?r.PAddr {
@@ -164,7 +109,7 @@ pub const PMap = struct {
if (!pte.present) return null;
if (level == 0 or (leaf_level_enabled(level) and pte.huge_page)) {
if (level == 0) {
return pte.address() + (va & (page_size(level) - 1));
}
@@ -232,7 +177,7 @@ pub const PMap = struct {
/// Reset upward in `advance` when the VA crosses a level boundary.
top_level: usize,
fn walk_down(self: *Cursor, target_level: usize, allocate: bool, comptime allocator: Allocator) error{PageNotMapped}!void {
fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void {
var current_level = self.top_level;
while (current_level > target_level) {
@@ -244,7 +189,7 @@ pub const PMap = struct {
return error.PageNotMapped;
}
const new_table_pa = if (allocator == .Boot) mi.phys.early_alloc() else mi.phys.alloc();
const new_table_pa = mi.phys.alloc();
const table_ptr: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(new_table_pa));
@memset(table_ptr[0..entries_per_table], std.mem.zeroes(mi.impl.Pte));
@@ -261,41 +206,18 @@ pub const PMap = struct {
}
pub fn walk_down_resolve(self: *Cursor, target_level: usize) bool {
self.walk_down(target_level, false, .Normal) catch return false;
self.walk_down(target_level, false) catch return false;
return true;
}
/// Choose the largest page size that satisfies the alignment and
/// size constraints of the current VA, PA, and remaining byte count.
/// Tries large pages first; falls back to 4K.
fn choose_target_level(self: *Cursor, pa: usize, remain: usize) usize {
var level: usize = num_levels;
while (level > 0) {
level -= 1;
if (!leaf_level_enabled(level)) continue;
const pg_size = page_size(level);
const align_mask = pg_size - 1;
if (remain >= pg_size and
(self.va & align_mask) == 0 and
(pa & align_mask) == 0)
{
return level;
}
}
@panic("no leaf level can map current alignment/size");
}
pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags, comptime allocator: Allocator) void {
pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags) void {
var remain = size;
var current_pa = pa;
while (remain > 0) {
const target_level = self.choose_target_level(current_pa, remain);
const target_level = 0;
self.walk_down(target_level, true, allocator) catch @panic("walk_down failed during map_range");
self.walk_down(target_level, true) catch @panic("walk_down failed during map_range");
const table = self.tables[target_level];
table[index_for_level(self.va, target_level)] =
src/mm/root.zig
+12
View File
@@ -2,6 +2,7 @@ pub const private = @import("private.zig");
const std = @import("std");
const rtl = @import("rtl");
const r = @import("root");
pub const page_size = 4096;
@@ -30,6 +31,17 @@ pub const Page = extern union {
},
};
comptime {
if (@sizeOf(Page) != r.BootInfo.page_struct_size) {
@compileError(
std.fmt.comptimePrint(
"'Page' size mismatch between loader and kernel: kernel={} loader={}",
.{ @sizeOf(Page), r.BootInfo.page_struct_size },
),
);
}
}
pub const MapFlags = packed struct {
read: bool = true,
write: bool = false,
src/platform/pc/smp.zig
+1 -1
View File
@@ -55,7 +55,7 @@ pub fn init() linksection(r.init) void {
.read = true,
.write = true,
.execute = true,
}, .Normal);
});
// From Linux, on modern CPUs we can skip the long delay after INIT.
const skip_delay = switch (amd64.cpu_features.vendor) {