From 85e6a8fec9501248e6a8c3513e421600bc02e932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Mon, 29 Jun 2026 22:33:43 -0400 Subject: [PATCH] mm: make physical allocation blocking (WIP) --- src/ex/workqueue.zig | 4 -- src/mm/heap.zig | 19 ++++--- src/mm/phys.zig | 115 ++++++++++++++++++++++++++++++++++------ src/mm/pmap.zig | 6 ++- src/mm/private.zig | 2 +- src/mm/radix_pmap.zig | 4 +- src/mm/root.zig | 5 ++ src/mm/tlb.zig | 5 +- src/mm/zone.zig | 62 ++++++++++------------ src/platform/pc/smp.zig | 2 +- src/proc/thread.zig | 8 +-- 11 files changed, 160 insertions(+), 72 deletions(-) diff --git a/src/ex/workqueue.zig b/src/ex/workqueue.zig index e9b2296..7ba42c0 100644 --- a/src/ex/workqueue.zig +++ b/src/ex/workqueue.zig @@ -146,10 +146,6 @@ pub fn enqueue(item: *WorkItem) void { /// Initialize the workqueue subsystem. pub fn init() !void { - // Ensure readying the threads doesn't preempt us. - const ipl = ke.ipl.raise(.Dispatch); - defer ke.ipl.lower(ipl); - for (0..ke.ncpus) |cpu| { const c = percpu.remote(@intCast(cpu)); diff --git a/src/mm/heap.zig b/src/mm/heap.zig index 9baac94..4d13eca 100644 --- a/src/mm/heap.zig +++ b/src/mm/heap.zig @@ -7,23 +7,22 @@ pub fn init() void { mi.kernel_space.arena.init("kernel heap", mi.impl.kernel_heap_base, r.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena"); } -pub fn alloc(size: usize) mm.Error!*anyopaque { - const ipl = mi.kernel_space.lock.acquire(); - defer mi.kernel_space.lock.release(ipl); +pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { + _ = policy; + + mi.kernel_space.lock.acquire(); const addr = try mi.kernel_space.arena.alloc(size, .{}); - mi.kernel_space.pmap.map_range_allocating(addr, size, .{ - .read = true, - .write = true, - .global = false, - }); + mi.kernel_space.pmap.map_range_allocating(addr, size, .{ .read = true, .write = true }, .DontWaitForMemory); + + mi.kernel_space.lock.release(); return @ptrFromInt(addr); } pub fn free(va: r.VAddr, size: usize) void { - const ipl = mi.kernel_space.lock.acquire(); + mi.kernel_space.lock.acquire(); mi.tlb.reclaim_range(&mi.kernel_space, va, size); - mi.kernel_space.lock.release(ipl); + mi.kernel_space.lock.release(); } diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 600fdc9..bb57d00 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -8,13 +8,19 @@ const ke = r.ke; const log = std.log.scoped(.@"mm/phys"); +const free_pages_limit = 128; + var bootstrapped = false; var early_alloc_entry_idx: usize = 0; var memory_map: *pl.BootInfo.MemMap = undefined; var early_allocs: usize = 0; -var list_lock: ke.SpinLock = undefined; +var list_lock: ke.SpinLock = .init(); var free_list: rtl.List = undefined; +var free_page_event: ke.Event = undefined; + +var free_pages: usize = 0; + pub var usable_memory: std.atomic.Value(usize) = .init(0); fn early_alloc() usize { @@ -38,32 +44,86 @@ fn early_alloc() usize { @panic("mm/phys: Out of memory for early allocation"); } +// Wait for pages to be available. +// List lock is held. +fn wait_for_pages(old_ipl: ke.Ipl) void { + var ipl = old_ipl; + + std.debug.assert(@intFromEnum(ipl) < @intFromEnum(ke.Ipl.Dispatch)); + + if (free_pages >= free_pages_limit) { + return; + } + + var cnt = free_pages; + + while (cnt < free_pages_limit) { + free_page_event.reset(); + + list_lock.release(ipl); + + _ = ke.wait.wait_one(&free_page_event.hdr, null) catch unreachable; + + ipl = list_lock.acquire(); + cnt = free_pages; + } +} + /// Allocate a page of physical memory. +/// This may block if no memory is available. pub fn alloc() r.PAddr { if (!bootstrapped) { @branchHint(.unlikely); return early_alloc(); } + const ipl = list_lock.acquire(); + defer list_lock.release(ipl); + + wait_for_pages(ipl); + + std.debug.assert(!free_list.is_empty()); + + const head = free_list.first(); + head.remove(); + + _ = usable_memory.fetchSub(mm.page_size, .monotonic); + free_pages -= 1; + + const elem: *mm.PageFree = @fieldParentPtr("link", head); + const page: *mm.Page = @ptrCast(elem); + const phys_addr = mm.pfn_to_page(mm.struct_page_to_pfn(page)); + + return phys_addr; +} + +pub fn alloc_opts(opts: struct { policy: mm.WaitPolicy }) ?r.PAddr { + if (!bootstrapped) { + @branchHint(.unlikely); + return early_alloc(); + } const ipl = list_lock.acquire(); + defer list_lock.release(ipl); - if (free_list.is_empty()) { - list_lock.release(ipl); - // FIXME: wait for free memory - @panic("mm/phys: Out of memory for physical allocation"); + if (free_list.is_empty() and opts.policy == .DontWaitForMemory) { + return null; + } else if (opts.policy == .WaitForMemory) { + wait_for_pages(ipl); } + std.debug.assert(free_pages > 0); + std.debug.assert(!free_list.is_empty()); + const head = free_list.first(); head.remove(); _ = usable_memory.fetchSub(mm.page_size, .monotonic); + free_pages -= 1; const elem: *mm.PageFree = @fieldParentPtr("link", head); const page: *mm.Page = @ptrCast(elem); const phys_addr = mm.pfn_to_page(mm.struct_page_to_pfn(page)); - list_lock.release(ipl); - return phys_addr; } @@ -74,6 +134,10 @@ pub fn free(addr: r.PAddr) void { const ipl = list_lock.acquire(); _ = usable_memory.fetchAdd(mm.page_size, .monotonic); free_list.insert_head(&page.free.link); + free_pages += 1; + if (free_pages >= free_pages_limit) { + free_page_event.signal(); + } list_lock.release(ipl); } @@ -81,17 +145,28 @@ pub fn free(addr: r.PAddr) void { pub fn free_batch(head: *mm.Page, tail: *mm.Page, count: usize) void { const ipl = list_lock.acquire(); - tail.free.link.next = free_list.first(); - free_list.insert_head(&head.free.link); + const first = free_list.first(); + + head.free.link.prev = &free_list.head; + tail.free.link.next = first; + first.prev = &tail.free.link; + free_list.head.next = &head.free.link; _ = usable_memory.fetchAdd(count * mm.page_size, .monotonic); + free_pages += count; + + if (free_pages >= free_pages_limit) { + free_page_event.signal(); + } + list_lock.release(ipl); } pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void { memory_map = &boot_info.memory_map; free_list.init(); + free_page_event.init(.Notification); var total_usable_memory: usize = 0; log.info("physical memory map:", .{}); @@ -133,11 +208,16 @@ pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void { const map_end = std.mem.alignForward(usize, exact_end, mm.page_size); // 3. Map the virtual pages. - mi.kernel_space.pmap.map_range_allocating(map_start, map_end - map_start, .{ - .read = true, - .write = true, - .global = true, - }); + mi.kernel_space.pmap.map_range_allocating( + map_start, + map_end - map_start, + .{ + .read = true, + .write = true, + .global = true, + }, + .DontWaitForMemory, + ); } } @@ -152,7 +232,12 @@ pub fn init_pfndb() void { const npages = entry.size / mm.page_size; for (0..npages) |j| { - free(entry.base + (j * mm.page_size)); + const page: *mm.Page = mm.pfn_to_struct_page(mm.page_to_pfn(entry.base + (j * mm.page_size))); + + free_list.insert_head(&page.free.link); + free_pages += 1; + + _ = usable_memory.fetchAdd(mm.page_size, .monotonic); } } diff --git a/src/mm/pmap.zig b/src/mm/pmap.zig index d19613d..e94e137 100644 --- a/src/mm/pmap.zig +++ b/src/mm/pmap.zig @@ -53,9 +53,10 @@ const ContiguousSource = struct { /// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped. const AllocatingSource = struct { flags: mm.MapFlags, + policy: mm.WaitPolicy, pub fn next(self: *const AllocatingSource, _: r.VAddr) MapItem { - const pa = mi.phys.alloc(); + const pa = mi.phys.alloc_opts(.{ .policy = self.policy }) orelse @panic("Out of memory"); return .{ .pa = pa, .len = mm.page_size, @@ -95,9 +96,10 @@ pub const PMap = struct { } /// Map a virtual address range to physical addresses allocated on demand. - pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags) void { + pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags, policy: mm.WaitPolicy) void { const src = AllocatingSource{ .flags = flags, + .policy = policy, }; self.map_from(va, size, src); diff --git a/src/mm/private.zig b/src/mm/private.zig index 20aa297..6914c30 100644 --- a/src/mm/private.zig +++ b/src/mm/private.zig @@ -14,7 +14,7 @@ const ke = @import("root").ke; pub const Space = struct { pmap: PMap, arena: vmem.Arena, - lock: ke.QSpinLock, + lock: ke.Mutex, }; pub const PfnList = extern struct { diff --git a/src/mm/radix_pmap.zig b/src/mm/radix_pmap.zig index 212f0cd..3dcc22e 100644 --- a/src/mm/radix_pmap.zig +++ b/src/mm/radix_pmap.zig @@ -148,12 +148,14 @@ pub fn RadixPmap(comptime Impl: type) type { c.advance(mm.page_size); } + if (list.is_empty()) return null; + const head_free: *mm.PageFree = @fieldParentPtr("link", list.first()); const tail_free: *mm.PageFree = @fieldParentPtr("link", list.last()); const head: *mm.Page = @ptrCast(head_free); const tail: *mm.Page = @ptrCast(tail_free); - return if (list.is_empty()) null else .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) }; + return .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) }; } /// Ensure all intermediate page tables down to `target_level` exist for diff --git a/src/mm/root.zig b/src/mm/root.zig index e8e4e5e..3bad0fa 100644 --- a/src/mm/root.zig +++ b/src/mm/root.zig @@ -40,6 +40,11 @@ pub const MapFlags = packed struct { cache_disable: bool = false, }; +pub const WaitPolicy = enum { + WaitForMemory, + DontWaitForMemory, +}; + pub const Error = error{ OutOfMemory, InvalidAddress, InvalidSize }; pub inline fn page_to_pfn(addr: usize) Pfn { diff --git a/src/mm/tlb.zig b/src/mm/tlb.zig index a1c4b46..11be2a1 100644 --- a/src/mm/tlb.zig +++ b/src/mm/tlb.zig @@ -20,10 +20,11 @@ fn reclaim_state(state: *ke.ShootdownState) void { const npages = state.npages; // Reclaim the virtual address space. - const ipl = space.lock.acquire(); + space.lock.acquire(); space.arena.free(base, @as(usize, npages) * mm.page_size) catch unreachable; - space.lock.release(ipl); + + space.lock.release(); _ = async_shootdowns.fetchAdd(1, .monotonic); diff --git a/src/mm/zone.zig b/src/mm/zone.zig index adda49b..b0a32f1 100644 --- a/src/mm/zone.zig +++ b/src/mm/zone.zig @@ -258,7 +258,7 @@ pub const Zone = struct { /// Current color for slab coloring. color: usize, /// Zone lock. - lock: ke.QSpinLock, + lock: ke.Mutex, /// List of full slabs in this zone. full_slabs: rtl.List, /// List of partial slabs in this zone. @@ -344,9 +344,7 @@ pub const Zone = struct { } /// Allocate an object from the zone. - pub fn alloc(self: *Self) mm.Error!*anyopaque { - const ipl = ke.ipl.raise(.Dispatch); - defer ke.ipl.lower(ipl); + pub fn alloc(self: *Self, opts: struct { policy: mm.WaitPolicy = .WaitForMemory }) mm.Error!*anyopaque { var buf: *anyopaque = undefined; // Try grabbing an object from the magazine layer. @@ -354,8 +352,8 @@ pub const Zone = struct { @branchHint(.likely); const cpu = &self.cpus[ke.cpu.current()]; - cpu.lock.acquire_no_ipl(); - defer cpu.lock.release_no_ipl(); + const ipl = cpu.lock.acquire(); + defer cpu.lock.release(ipl); while (true) { if (cpu.alloc_rounds > 0) { @@ -407,13 +405,17 @@ pub const Zone = struct { } // Fall back to the slab layer. - self.lock.acquire_no_ipl(); - defer self.lock.release_no_ipl(); + self.lock.acquire(); + defer self.lock.release(); var slab: *Slab = undefined; if (self.partial_slabs.is_empty()) { - slab = try self.slab_create(); + self.lock.release(); + + slab = try self.slab_create(opts.policy); + + self.lock.acquire(); self.partial_slabs.insert_tail(&slab.link); } else { slab = @fieldParentPtr("link", self.partial_slabs.first()); @@ -446,16 +448,14 @@ pub const Zone = struct { /// Free an object back to the zone. pub fn free(self: *Self, obj: *anyopaque) void { - const ipl = ke.ipl.raise(.Dispatch); - defer ke.ipl.lower(ipl); - if (magazines_initialized and self.use_magazines) { @branchHint(.likely); - const cpu = &self.cpus[ke.cpu.current()]; + const ipl = ke.ipl.raise(.Dispatch); + var cpu = &self.cpus[ke.cpu.current()]; cpu.lock.acquire_no_ipl(); - defer cpu.lock.release_no_ipl(); + defer cpu.lock.release(ipl); while (true) { if (cpu.free != null and cpu.free_rounds < cpu.magazine_size) { @@ -495,11 +495,7 @@ pub const Zone = struct { } // No empty magazine in the depot, try to allocate a new one. - cpu.lock.release_no_ipl(); - - const new_mag: ?*Magazine = @ptrCast(@alignCast(self.magtype.?.zone.alloc() catch null)); - - cpu.lock.acquire_no_ipl(); + const new_mag: ?*Magazine = @ptrCast(@alignCast(self.magtype.?.zone.alloc(.{ .policy = .DontWaitForMemory }) catch null)); if (new_mag) |m| { self.free_to_depot(m, &self.empty_mags); @@ -518,8 +514,8 @@ pub const Zone = struct { fill_with_poison(obj, self.obj_size, free_poison); } - self.lock.acquire_no_ipl(); - defer self.lock.release_no_ipl(); + self.lock.acquire(); + defer self.lock.release(); // Find the slab for the buffer. const slab: *Slab = if (self.obj_size > small_slab_size) blk: { @@ -551,8 +547,8 @@ pub const Zone = struct { } } - fn alloc_page() !*anyopaque { - const alloc_ret = mi.phys.alloc(); + fn alloc_page(policy: mm.WaitPolicy) !*anyopaque { + const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse return error.OutOfMemory; return @ptrFromInt(mm.p2v(alloc_ret)); } @@ -562,8 +558,8 @@ pub const Zone = struct { mi.phys.free(phys); } - fn slab_create_small(self: *Self) mm.Error!*Slab { - const buf = try alloc_page(); + fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { + const buf = try alloc_page(policy); // We employ a simple coloring scheme; every time a slab is created, // the color is shifted by the alignment. This is done until we reach the @@ -604,7 +600,7 @@ pub const Zone = struct { return slab; } - fn slab_create_large(self: *Self) mm.Error!*Slab { + fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { self.color += self.alignment; if (self.color > self.max_color) { @@ -612,7 +608,7 @@ pub const Zone = struct { } const capacity = (self.slab_size - self.color) / self.chunk_size; - const buf = try mi.heap.alloc(self.slab_size); + const buf = try mi.heap.alloc(self.slab_size, policy); var ret: *Slab = @ptrCast(@alignCast(try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity)))); @@ -648,11 +644,11 @@ pub const Zone = struct { return ret; } - fn slab_create(self: *Self) mm.Error!*Slab { + fn slab_create(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { if (self.obj_size <= small_slab_size) { - return self.slab_create_small(); + return self.slab_create_small(policy); } else { - return self.slab_create_large(); + return self.slab_create_large(policy); } } @@ -723,7 +719,7 @@ pub fn TypedZone(comptime T: type) type { } pub fn create(self: *Self) mm.Error!*T { - const ret = try self.zone.alloc(); + const ret = try self.zone.alloc(.{}); return @ptrCast(@alignCast(ret)); } @@ -794,12 +790,12 @@ fn gpa_alloc( if (len > 2048) { if (ptr_align.toByteUnits() > mm.page_size) return null; const pages = std.mem.alignForward(usize, len, mm.page_size); - const ptr = mi.heap.alloc(pages) catch return null; + const ptr = mi.heap.alloc(pages, .WaitForMemory) catch return null; return @ptrCast(ptr); } const zone = zone_for(len, ptr_align.toByteUnits()) orelse return null; - const obj = zone.alloc() catch return null; + const obj = zone.alloc(.{}) catch return null; return @ptrCast(obj); } diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index 18a03bf..fb6237a 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -107,7 +107,7 @@ pub fn init() linksection(r.init) void { // copy the offset into the AP's self_offset variable @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id]; - const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32)) catch @panic("Failed to allocate AP stack")) + r.kib(32); + const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32), .DontWaitForMemory) catch @panic("Failed to allocate AP stack")) + r.kib(32); start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15); diff --git a/src/proc/thread.zig b/src/proc/thread.zig index 605a27a..32ec05f 100644 --- a/src/proc/thread.zig +++ b/src/proc/thread.zig @@ -11,6 +11,7 @@ const mm = r.mm; const kernel_thread_stack_size = r.kib(16); var reaper_item: ex.WorkItem = undefined; +var thread_zone: mm.zone.TypedZone(Thread) = undefined; /// Higher-level wrapper over a thread. pub const Thread = struct { @@ -28,7 +29,7 @@ fn reap_thread(obj: *anyopaque, _: ?*anyopaque) void { psp.turnstile_zone.destroy(td.kern.turnstile); mm.heap.free(td.kern.stack, kernel_thread_stack_size); - mm.zone.gpa.destroy(td); + thread_zone.destroy(td); } fn reaper_fn(_: ?*anyopaque) void { @@ -40,6 +41,7 @@ fn activation(_: *rtl.HandoffList) void { } pub fn init() void { + thread_zone.init("thread", .{}); ke.thread.reaper_list.* = .init(activation); reaper_item.init(.High, reaper_fn, null); } @@ -49,8 +51,8 @@ pub fn init() void { /// - `entry`: entry point. /// - `arg`: argument passed to `entry`. pub fn create_kernel(prio: u8, entry: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) !*Thread { - var td = try mm.zone.gpa.create(Thread); - const stack = try mm.heap.alloc(kernel_thread_stack_size); + var td = try thread_zone.create(); + const stack = try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory); td.kern.init(@intFromPtr(stack), kernel_thread_stack_size, prio, entry, arg); td.kern.turnstile = try psp.turnstile_zone.create();