feat(mm): initial mm work

build/linker-scripts/x86_64.lds

@@ -33,17 +33,25 @@ SECTIONS
     /* Move to the next memory page for .text */
     . = ALIGN(CONSTANT(MAXPAGESIZE));
 
+    text_start_addr = .;
+
     .text : {
         *(.text .text.*)
     } :text
 
+
     /* Move to the next memory page for .rodata */
     . = ALIGN(CONSTANT(MAXPAGESIZE));
 
+    text_end_addr = .;
+
+    rodata_start_addr = .;
+
     .rodata : {
         *(.rodata .rodata.*)
     } :rodata
 
+
     .percpu_init_array : {
         __init_array_percpu_start = .;
         KEEP(*(.percpu_init_array))
@@ -56,9 +64,12 @@ SECTIONS
         *(.note.gnu.build-id)
     } :rodata
 
+
     /* Move to the next memory page for .data */
     . = ALIGN(CONSTANT(MAXPAGESIZE));
+    rodata_end_addr = .;
 
+    data_start_addr = .;
     .data : {
         *(.data .data.*)
     } :data
@@ -69,6 +80,7 @@ SECTIONS
         __percpu_end = .;
     } :data
 
+
     /* NOTE: .bss needs to be the last thing mapped to :data, otherwise lots of */
     /* unnecessary zeros will be written to the binary. */
     /* If you need, for example, .init_array and .fini_array, those should be placed */
@@ -78,10 +90,14 @@ SECTIONS
         *(COMMON)
     } :data
 
+
+    . = ALIGN(CONSTANT(MAXPAGESIZE));
+
+    data_end_addr = .;
+
     /* Discard .note.* and .eh_frame* since they may cause issues on some hosts. */
     /DISCARD/ : {
         *(.eh_frame*)
         *(.note .note.*)
     }
 }
-

build/run.zig

@@ -52,7 +52,7 @@ pub fn addRun(b: *std.Build, kernel: *std.Build.Step.Compile, plat: config.Platf
 
         switch (plat.arch) {
             .x86_64 => {
-                qemu.addArgs(&.{ "-serial", "stdio" });
+                qemu.addArgs(&.{ "-serial", "stdio", "-m", "1G", "-smp", "4" });
 
                 if (!debug) qemu.addArgs(&.{ "-enable-kvm", "-cpu", "host" });
             },

src/ex/init.zig

@@ -2,9 +2,10 @@ const std = @import("std");
 const b = @import("base");
 const pl = b.pl;
 const ke = b.ke;
+const mm = b.mm;
 
 pub fn init(boot_info: *pl.BootInfo) void {
-    _ = boot_info;
+    mm.init(boot_info);
 
     pl.late_init();
     ke.sched.late_init();

src/mm/amd64/impl.zig

@@ -0,0 +1,106 @@
+const amd64 = @import("arch");
+const b = @import("base");
+const mm = b.mm;
+const mi = mm.private;
+
+pub const hhdm_minimum_max_address = b.gib(4);
+
+pub fn phys_to_virt(addr: b.PAddr) b.VAddr {
+    return addr + 0xffff800000000000;
+}
+
+pub fn virt_to_phys(vaddr: b.VAddr) usize {
+    return vaddr - 0xffff800000000000;
+}
+
+pub const pfndb_base = 0xffffea0000000000;
+
+pub const PagingImpl = struct {
+    const Self = @This();
+
+    pub const levels = [_]mi.radix_pmap.Level{
+        .{ .shift = 12, .leaf = true }, // 4K
+        .{ .shift = 21, .leaf = true }, // 2M
+        .{ .shift = 30, .leaf = true }, // 1G
+        .{ .shift = 39, .leaf = false }, // root
+    };
+
+    pub const Pte = packed struct(u64) {
+        present: bool,
+        writable: bool,
+        user: bool,
+        write_through: bool,
+        cache_disable: bool,
+        accessed: bool,
+        dirty: bool,
+        huge_page: bool,
+        global: bool,
+        available: u3,
+        addr: u40,
+        ignored_2: u11,
+        nx: bool,
+
+        pub fn address(self: Pte) b.PAddr {
+            return @as(b.PAddr, self.addr) << 12;
+        }
+    };
+
+    pub inline fn make_table_pte(pa: b.PAddr) Pte {
+        return Pte{
+            .present = true,
+            .writable = true,
+            .user = true,
+            .write_through = false,
+            .cache_disable = false,
+            .accessed = false,
+            .dirty = false,
+            .huge_page = false,
+            .global = false,
+            .available = 0,
+            .addr = @truncate(pa >> 12),
+            .ignored_2 = 0,
+            .nx = false,
+        };
+    }
+
+    pub inline fn make_leaf_pte(pa: b.PAddr, flags: mm.MapFlags, level: usize) Pte {
+        return Pte{
+            .present = true,
+            .writable = flags.write,
+            .user = flags.user,
+            .write_through = flags.write_through,
+            .cache_disable = flags.cache_disable,
+            .accessed = false,
+            .dirty = false,
+            .huge_page = (level > 0),
+            .global = flags.global,
+            .available = 0,
+            .addr = @truncate(pa >> 12),
+            .ignored_2 = 0,
+            .nx = !flags.execute,
+        };
+    }
+
+    pub inline fn activate(root_pa: b.PAddr) void {
+        amd64.write_cr(3, root_pa);
+    }
+
+    pub fn is_leaf_level_enabled(level: usize) bool {
+        if (!levels[level].leaf) return false;
+        if (level == 2) return amd64.cpu_features.gib_pages;
+        return true;
+    }
+};
+
+pub const PMap = mi.radix_pmap.RadixPmap(PagingImpl);
+
+pub fn init_kernel() void {
+    const higher_half_start = phys_to_virt(0);
+    const higher_half_size: usize = 256 * (1 << 39);
+
+    mi.kernel_pmap.impl.root_pa = mi.phys.alloc_page();
+    const table_ptr: [*]u64 = @ptrFromInt(mm.p2v(mi.kernel_pmap.impl.root_pa));
+    @memset(table_ptr[0..512], 0);
+
+    mi.kernel_pmap.impl.preallocate(higher_half_start, higher_half_size, 2);
+}

src/mm/init.zig

@@ -0,0 +1,86 @@
+const b = @import("base");
+const pl = b.pl;
+const mm = b.mm;
+const mi = mm.private;
+
+extern var text_start_addr: u8;
+extern var text_end_addr: u8;
+extern var rodata_start_addr: u8;
+extern var rodata_end_addr: u8;
+extern var data_start_addr: u8;
+extern var data_end_addr: u8;
+
+fn map_kernel(boot_info: *pl.BootInfo) void {
+    const kaddr = boot_info.kernel_address;
+
+    const text_start = @intFromPtr(&text_start_addr);
+    const text_end = @intFromPtr(&text_end_addr);
+    const rodata_start = @intFromPtr(&rodata_start_addr);
+    const rodata_end = @intFromPtr(&rodata_end_addr);
+    const data_start = @intFromPtr(&data_start_addr);
+    const data_end = @intFromPtr(&data_end_addr);
+
+    const text_size = text_end - text_start;
+    const rodata_size = rodata_end - rodata_start;
+    const data_size = data_end - data_start;
+
+    mi.kernel_pmap.map_contiguous_range(text_start, kaddr.physical_base + (text_start - kaddr.virtual_base), text_size, .{
+        .read = true,
+        .execute = true,
+        .global = true,
+    });
+
+    mi.kernel_pmap.map_contiguous_range(rodata_start, kaddr.physical_base + (rodata_start - kaddr.virtual_base), rodata_size, .{
+        .read = true,
+        .global = true,
+    });
+
+    mi.kernel_pmap.map_contiguous_range(data_start, kaddr.physical_base + (data_start - kaddr.virtual_base), data_size, .{
+        .read = true,
+        .write = true,
+        .global = true,
+    });
+
+    mi.kernel_pmap.map_contiguous_range(mm.p2v(0), 0, mi.impl.hhdm_minimum_max_address, .{
+        .read = true,
+        .write = true,
+        .global = true,
+    });
+
+    // Now go through every usable entry and map to the HHDM every part that isnt covered by [0, hhdm_minimum_max_address).
+    // We can't blindly map until the maximum usable physical address because on some CPUs this might cause MCEs.
+    // See https://github.com/torvalds/linux/commit/66520ebc2df3fe52eb4792f8101fac573b766baf
+    for (0..boot_info.memory_map.entry_count) |i| {
+        const entry = boot_info.memory_map.entries[i];
+
+        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) {
+            continue;
+        }
+
+        if (entry.base + entry.size <= mi.impl.hhdm_minimum_max_address) {
+            continue;
+        }
+
+        var entry_start = entry.base;
+        var entry_size = entry.size;
+
+        if (entry.base < mi.impl.hhdm_minimum_max_address) {
+            const adjust = mi.impl.hhdm_minimum_max_address - entry.base;
+            entry_start += adjust;
+            entry_size -= adjust;
+        }
+
+        mi.kernel_pmap.map_contiguous_range(mm.p2v(entry_start), entry_start, entry_size, .{
+            .read = true,
+            .write = true,
+            .global = true,
+        });
+    }
+}
+
+pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
+    mi.phys.init(boot_info);
+    map_kernel(boot_info);
+    mi.kernel_pmap.activate();
+    mi.phys.init_pfndb();
+}

src/mm/phys.zig

@@ -0,0 +1,112 @@
+const b = @import("base");
+const std = @import("std");
+const rtl = @import("rtl");
+const pl = b.pl;
+const mm = b.mm;
+const mi = mm.private;
+
+var early_alloc_entry_idx: usize = 0;
+var memory_map: *pl.BootInfo.MemMap = undefined;
+var early_allocs: usize = 0;
+
+pub var bootstrapped = false;
+
+pub fn early_alloc() usize {
+    while (early_alloc_entry_idx < memory_map.entry_count) {
+        const entry = &memory_map.entries[early_alloc_entry_idx];
+
+        if (entry.type != .Free or entry.size < mm.page_size) {
+            early_alloc_entry_idx += 1;
+            continue;
+        }
+
+        const addr = entry.base;
+
+        entry.base += mm.page_size;
+        entry.size -= mm.page_size;
+        early_allocs += mm.page_size;
+
+        return addr;
+    }
+
+    @panic("mm/phys: Out of memory for early allocation");
+}
+
+pub fn alloc_page() usize {
+    if (!bootstrapped) {
+        return early_alloc();
+    }
+    @panic("not implemented");
+}
+
+pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
+    memory_map = &boot_info.memory_map;
+
+    var total_usable_memory: usize = 0;
+    std.log.info("mm/phys: physical memory map:", .{});
+
+    for (0..boot_info.memory_map.entry_count) |i| {
+        const entry = boot_info.memory_map.entries[i];
+
+        std.log.info("mm/phys: [{x:0>16} - {x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) });
+
+        if (entry.type == .Free) {
+            total_usable_memory += entry.size;
+        }
+    }
+
+    const pfndb_size_required = @sizeOf(mm.Page) * (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable);
+
+    std.log.info("mm: using {} KiB for pfndb", .{pfndb_size_required / 1024});
+
+    // Create the kernel pagemap from the early allocator.
+    mi.impl.init_kernel();
+
+    // Now map the PFN database.
+    for (0..boot_info.memory_map.entry_count) |i| {
+        const entry = boot_info.memory_map.entries[i];
+
+        if (entry.type != .Free or entry.size < mm.page_size) {
+            continue;
+        }
+
+        const npages = entry.size / mm.page_size;
+
+        // 1. Calculate the exact virtual address range needed for this region's page structs.
+        const start_pfn: usize = mm.page_to_pfn(entry.base);
+        const exact_start = mi.impl.pfndb_base + (start_pfn * @sizeOf(mm.Page));
+        const exact_end = exact_start + (npages * @sizeOf(mm.Page));
+
+        // 2. Ensure the addresses are aligned on page boundaries.
+        const map_start = std.mem.alignBackward(usize, exact_start, mm.page_size);
+        const map_end = std.mem.alignForward(usize, exact_end, mm.page_size);
+
+        // 3. Map the virtual pages.
+        mi.kernel_pmap.map_range_allocating(map_start, map_end - map_start, .{
+            .read = true,
+            .write = true,
+            .global = true,
+        });
+    }
+}
+
+pub fn init_pfndb() void {
+    const pfndb: [*]mm.Page = @ptrFromInt(mi.impl.pfndb_base);
+
+    for (0..memory_map.entry_count) |i| {
+        const entry = memory_map.entries[i];
+
+        if (entry.type != .Free or entry.size < mm.page_size) {
+            continue;
+        }
+
+        const start_pfn: usize = mm.page_to_pfn(entry.base);
+        const npages = entry.size / mm.page_size;
+
+        for (0..npages) |j| {
+            const pfn = start_pfn + j;
+            const page = &pfndb[pfn];
+            page.batch_count = 0;
+        }
+    }
+}

src/mm/pmap.zig

@@ -0,0 +1,115 @@
+//! Platform-agnostic page map interface.
+//!
+//! This file defines the `PMap` wrapper and the two built-in mapping sources
+//! (`ContiguousSource`, `AllocatingSource`) used to drive `PMap.map_from`.
+//!
+//! `PMap` itself is a thin wrapper over `mi.impl.PMap`, which is expected to
+//! be a `RadixPmap` instantiation (or any type that satisfies the same
+//! interface). To add a new platform, provide a concrete `impl.PMap` that
+//! exposes:
+//!
+//!   - `map_from(va: VAddr, size: usize, source: anytype) void`
+//!   - `activate() void`
+//!
+//! Mapping sources are ordinary structs with a single method:
+//!
+//!   - `next(self: *const Source, va: VAddr) MapItem`
+//!
+//! `next` is called once per mapping step and returns the physical address,
+//! length, and flags for that step. The two sources here cover the common
+//! cases (contiguous PA range and on-demand allocation); custom sources can
+//! be passed directly to `map_from` for other needs such as file-backed or
+//! scatter-gather mappings.
+const b = @import("base");
+const mm = b.mm;
+const mi = mm.private;
+
+/// Describes a contiguous region of physical memory to be mapped at a given virtual address.
+pub const MapItem = struct {
+    pa: b.PAddr,
+    len: usize,
+    flags: mm.MapFlags,
+};
+
+/// A mapping source that maps a single contiguous virtual-to-physical region.
+/// All calls to `next` return the same physical base address and size, so this
+/// is intended to be consumed exactly once by `map_from`.
+const ContiguousSource = struct {
+    flags: mm.MapFlags,
+    size: usize,
+    base_va: usize,
+    base_pa: usize,
+
+    pub fn next(self: *const ContiguousSource, _: b.VAddr) MapItem {
+        return .{
+            .pa = self.base_pa,
+            .len = self.size,
+            .flags = self.flags,
+        };
+    }
+};
+
+/// A mapping source that allocates physical pages on demand. Each call to `next`
+/// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped.
+const AllocatingSource = struct {
+    flags: mm.MapFlags,
+
+    pub fn next(self: *const AllocatingSource, _: b.VAddr) MapItem {
+        const pa = mi.phys.alloc_page();
+        return .{
+            .pa = pa,
+            .len = mm.page_size,
+            .flags = self.flags,
+        };
+    }
+};
+
+/// Generic wrapper over the platform-specific pmap implementation.
+/// Provides some convenient methods for common mapping patterns.
+pub const PMap = struct {
+    const Self = @This();
+
+    impl: mi.impl.PMap,
+
+    /// Map a virtual address range to physical addresses provided by the source.
+    pub fn map_from(self: *Self, va_start: b.VAddr, size: usize, source: anytype) void {
+        const Source = @TypeOf(source);
+
+        comptime {
+            if (!@hasDecl(Source, "next")) @compileError("source must provide next(self: *Source, va: b.VAddr) MapItem");
+        }
+
+        self.impl.map_from(va_start, size, source);
+    }
+
+    /// Map a contiguous virtual address range to a contiguous physical address range.
+    pub fn map_contiguous_range(self: *Self, va: b.VAddr, pa: b.PAddr, size: usize, flags: mm.MapFlags) void {
+        const src = ContiguousSource{
+            .flags = flags,
+            .size = size,
+            .base_va = va,
+            .base_pa = pa,
+        };
+
+        self.map_from(va, size, src);
+    }
+
+    /// Map a virtual address range to physical addresses allocated on demand.
+    pub fn map_range_allocating(self: *Self, va: b.VAddr, size: usize, flags: mm.MapFlags) void {
+        const src = AllocatingSource{
+            .flags = flags,
+        };
+
+        self.map_from(va, size, src);
+    }
+
+    /// Map a single page. Convenience wrapper around map_contiguous_range.
+    pub fn map_page(self: *Self, va: b.VAddr, pa: b.PAddr, flags: mm.MapFlags) void {
+        self.map_contiguous_range(va, pa, mm.page_size, flags);
+    }
+
+    /// Activate this pagemap.
+    pub fn activate(self: *Self) void {
+        self.impl.activate();
+    }
+};

src/mm/private.zig

@@ -0,0 +1,13 @@
+const config = @import("config");
+pub const phys = @import("phys.zig");
+pub const radix_pmap = @import("radix_pmap.zig");
+pub const pmap = @import("pmap.zig");
+pub const init = @import("init.zig");
+pub const PMap = pmap.PMap;
+
+pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64"))
+    @import("amd64/impl.zig")
+else
+    @compileError("unsupported architecture");
+
+pub var kernel_pmap: PMap = undefined;

src/mm/radix_pmap.zig

@@ -0,0 +1,220 @@
+//! Generic radix-tree page map implementation.
+//!
+//! `RadixPmap` is a comptime-parameterized page table walker that implements
+//! virtual-to-physical address mapping over a multi-level radix tree (e.g.
+//! x86-64's four-level PT). The caller supplies an
+//! `Impl` type that describes the tree's level structure and provides
+//! platform-specific PTE constructors.
+//!
+//! The `Impl` type passed to `RadixPmap` must provide:
+//!
+//!   - `levels: []const Level`: ordered from the leaf up to the root,
+//!     each entry carrying the VA index shift and whether that level can
+//!     hold a leaf (large-page) mapping.
+//!   - `is_leaf_level_enabled(level: usize) bool`:runtime gate that
+//!     controls whether large pages are used at a given level (e.g. to
+//!     disable 1 GiB pages on hardware that lacks them).
+//!   - `Pte`: the PTE type. Must be zero-initializable and expose a
+//!     `present: bool` field and an `address() PAddr` method.
+//!   - `make_table_pte(pa: PAddr) Pte`: construct a non-leaf PTE pointing
+//!     to a child page table at `pa`.
+//!   - `make_leaf_pte(pa: PAddr, flags: MapFlags, level: usize) Pte`:
+//!     construct a leaf PTE mapping `pa` with the given flags at the
+//!     chosen level.
+//!   - `activate(root_pa: PAddr) void`: install the root page table into
+//!     the CPU (e.g. write CR3 or satp).
+
+const b = @import("base");
+const mm = b.mm;
+const mi = mm.private;
+const std = @import("std");
+
+pub const Level = struct {
+    shift: u6,
+    leaf: bool,
+};
+
+pub fn RadixPmap(comptime Impl: type) type {
+    return struct {
+        const Self = @This();
+
+        const num_levels = Impl.levels.len;
+        const entries_per_table = 512;
+        const index_mask = entries_per_table - 1;
+
+        root_pa: b.PAddr,
+
+        fn level_shift(level: usize) u6 {
+            return Impl.levels[level].shift;
+        }
+
+        fn page_size(level: usize) usize {
+            return @as(usize, 1) << level_shift(level);
+        }
+
+        fn index_for_level(va: usize, level: usize) usize {
+            return (va >> level_shift(level)) & index_mask;
+        }
+
+        fn is_leaf_level(level: usize) bool {
+            return Impl.levels[level].leaf;
+        }
+
+        fn leaf_level_enabled(level: usize) bool {
+            return is_leaf_level(level) and Impl.is_leaf_level_enabled(level);
+        }
+
+        fn cursor(self: *Self, start_va: usize) Cursor {
+            var tables: [num_levels][*]Impl.Pte = undefined;
+            tables[num_levels - 1] = @ptrFromInt(mm.p2v(self.root_pa));
+            return .{
+                .pmap = self,
+                .va = start_va,
+                .tables = tables,
+                .top_level = num_levels - 1,
+            };
+        }
+
+        pub fn activate(self: *Self) void {
+            Impl.activate(self.root_pa);
+        }
+
+        pub fn map_from(self: *Self, va: b.VAddr, size: usize, source: anytype) void {
+            var c = self.cursor(va);
+            var remain = size;
+
+            while (remain > 0) {
+                const item = source.next(c.va);
+                c.map_range(item.pa, item.len, item.flags);
+                remain -|= item.len;
+            }
+        }
+
+        /// Ensure all intermediate page tables down to `target_level` exist for
+        /// the given virtual address range, allocating them as needed. Useful for
+        /// pre-populating a shared region (e.g. the kernel half of a user pmap)
+        /// before any leaf mappings are installed.
+        pub fn preallocate(self: *Self, va: b.VAddr, size: usize, target_level: usize) void {
+            var c = self.cursor(va);
+            var remain = size;
+
+            // Step by one table at `target_level + 1` at a time so each
+            // `walk_down` call sees a fresh index at that level.
+            const step = page_size(target_level + 1);
+
+            while (remain > 0) {
+                c.walk_down(target_level, true) catch @panic("walk_down failed during preallocate");
+                const advance_amount = @min(remain, step);
+                c.advance(advance_amount);
+                remain -= advance_amount;
+            }
+        }
+
+        pub const Cursor = struct {
+            pmap: *Self,
+            va: b.VAddr,
+            /// Cached pointers to each level's page table, filled as the cursor
+            /// descends. Only indices in `[0, top_level]` are valid.
+            tables: [num_levels][*]Impl.Pte,
+            /// Index of the highest level for which `tables` currently holds a
+            /// valid pointer. Descends toward 0 (the leaf) as `walk_down` runs.
+            /// Reset upward in `advance` when the VA crosses a level boundary.
+            top_level: usize,
+
+            fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void {
+                var current_level = self.top_level;
+
+                while (current_level > target_level) {
+                    const idx = index_for_level(self.va, current_level);
+                    var pte = self.tables[current_level][idx];
+
+                    if (!pte.present) {
+                        if (!allocate) {
+                            return error.PageNotMapped;
+                        }
+
+                        const new_table_pa = mi.phys.alloc_page();
+                        const table_ptr: [*]Impl.Pte = @ptrFromInt(mm.p2v(new_table_pa));
+                        @memset(table_ptr[0..entries_per_table], std.mem.zeroes(Impl.Pte));
+
+                        pte = Impl.make_table_pte(new_table_pa);
+                        self.tables[current_level][idx] = pte;
+                    }
+
+                    const child_va = mm.p2v(pte.address());
+                    self.tables[current_level - 1] = @ptrFromInt(child_va);
+                    self.top_level -= 1;
+                    current_level -= 1;
+                }
+            }
+
+            fn walk_down_resolve(self: *Cursor, target_level: usize) bool {
+                self.walk_down(target_level, false) catch return false;
+                return true;
+            }
+
+            /// Choose the largest page size that satisfies the alignment and
+            /// size constraints of the current VA, PA, and remaining byte count.
+            /// Tries large pages first; falls back to 4K.
+            fn choose_target_level(self: *Cursor, pa: usize, remain: usize) usize {
+                var level: usize = num_levels;
+                while (level > 0) {
+                    level -= 1;
+                    if (!leaf_level_enabled(level)) continue;
+
+                    const pg_size = page_size(level);
+                    const align_mask = pg_size - 1;
+
+                    if (remain >= pg_size and
+                        (self.va & align_mask) == 0 and
+                        (pa & align_mask) == 0)
+                    {
+                        return level;
+                    }
+                }
+
+                @panic("no leaf level can map current alignment/size");
+            }
+
+            pub fn map_range(self: *Cursor, pa: b.PAddr, size: usize, flags: mm.MapFlags) void {
+                var remain = size;
+                var current_pa = pa;
+
+                while (remain > 0) {
+                    const target_level = self.choose_target_level(current_pa, remain);
+
+                    self.walk_down(target_level, true) catch @panic("walk_down failed during map_range");
+
+                    const table = self.tables[target_level];
+                    table[index_for_level(self.va, target_level)] =
+                        Impl.make_leaf_pte(current_pa, flags, target_level);
+
+                    const step = page_size(target_level);
+                    current_pa += step;
+                    remain -= step;
+
+                    self.advance(step);
+                }
+            }
+
+            /// Advance the cursor's virtual address by `offset` bytes and
+            /// invalidate any cached table pointers that no longer apply.
+            /// The cursor tracks the highest level whose index is still valid
+            /// for the new VA, `walk_down` will re-descend from there on the
+            /// next mapping operation.
+            pub fn advance(self: *Cursor, offset: usize) void {
+                const old_va = self.va;
+                self.va +%= offset;
+
+                var level: usize = 0;
+                while (level < num_levels - 1) : (level += 1) {
+                    const shift = level_shift(level + 1);
+                    if ((old_va >> shift) != (self.va >> shift)) {
+                        self.top_level = level + 1;
+                        break;
+                    }
+                }
+            }
+        };
+    };
+}

src/mm/root.zig

@@ -0,0 +1,36 @@
+pub const private = @import("private.zig");
+
+const b = @import("base");
+const std = @import("std");
+
+pub const page_size = 4096;
+
+pub const init = private.init.init;
+
+// 16 TB of maximum physical memory.
+// This should be fine on consumer hardware for at least a decade :^)
+pub const Pfn = u32;
+
+pub const Page = struct {
+    // Fields used when the page is free.
+    next_pfn: Pfn,
+    batch_next: Pfn,
+    batch_count: u8,
+};
+
+pub const MapFlags = packed struct {
+    read: bool = true,
+    write: bool = false,
+    user: bool = false,
+    execute: bool = false,
+    global: bool = false,
+    write_through: bool = false,
+    cache_disable: bool = false,
+};
+
+pub fn page_to_pfn(addr: usize) Pfn {
+    return @intCast(addr >> 12);
+}
+
+pub const p2v = private.impl.phys_to_virt;
+pub const v2p = private.impl.virt_to_phys;

src/root.zig

@@ -5,6 +5,7 @@ pub const pl = @import("platform/root.zig");
 pub const rtl = @import("rtl");
 pub const ksyms = @import("ksyms");
 pub const arch = @import("arch");
+pub const mm = @import("mm/root.zig");
 
 pub const init = ".text.init";
 pub const percpu_init = ".percpu_init_array";