mm: make physical allocation blocking (WIP)

src/ex/workqueue.zig

@@ -146,10 +146,6 @@ pub fn enqueue(item: *WorkItem) void {
 
 /// Initialize the workqueue subsystem.
 pub fn init() !void {
-    // Ensure readying the threads doesn't preempt us.
-    const ipl = ke.ipl.raise(.Dispatch);
-    defer ke.ipl.lower(ipl);
-
     for (0..ke.ncpus) |cpu| {
         const c = percpu.remote(@intCast(cpu));
 

src/mm/heap.zig

@@ -7,23 +7,22 @@ pub fn init() void {
     mi.kernel_space.arena.init("kernel heap", mi.impl.kernel_heap_base, r.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena");
 }
 
-pub fn alloc(size: usize) mm.Error!*anyopaque {
-    const ipl = mi.kernel_space.lock.acquire();
-    defer mi.kernel_space.lock.release(ipl);
+pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
+    _ = policy;
+
+    mi.kernel_space.lock.acquire();
 
     const addr = try mi.kernel_space.arena.alloc(size, .{});
 
-    mi.kernel_space.pmap.map_range_allocating(addr, size, .{
-        .read = true,
-        .write = true,
-        .global = false,
-    });
+    mi.kernel_space.pmap.map_range_allocating(addr, size, .{ .read = true, .write = true }, .DontWaitForMemory);
+
+    mi.kernel_space.lock.release();
 
     return @ptrFromInt(addr);
 }
 
 pub fn free(va: r.VAddr, size: usize) void {
-    const ipl = mi.kernel_space.lock.acquire();
+    mi.kernel_space.lock.acquire();
     mi.tlb.reclaim_range(&mi.kernel_space, va, size);
-    mi.kernel_space.lock.release(ipl);
+    mi.kernel_space.lock.release();
 }

src/mm/phys.zig

@@ -8,13 +8,19 @@ const ke = r.ke;
 
 const log = std.log.scoped(.@"mm/phys");
 
+const free_pages_limit = 128;
+
 var bootstrapped = false;
 var early_alloc_entry_idx: usize = 0;
 var memory_map: *pl.BootInfo.MemMap = undefined;
 var early_allocs: usize = 0;
-var list_lock: ke.SpinLock = undefined;
+var list_lock: ke.SpinLock = .init();
 var free_list: rtl.List = undefined;
 
+var free_page_event: ke.Event = undefined;
+
+var free_pages: usize = 0;
+
 pub var usable_memory: std.atomic.Value(usize) = .init(0);
 
 fn early_alloc() usize {
@@ -38,32 +44,86 @@ fn early_alloc() usize {
     @panic("mm/phys: Out of memory for early allocation");
 }
 
+// Wait for pages to be available.
+// List lock is held.
+fn wait_for_pages(old_ipl: ke.Ipl) void {
+    var ipl = old_ipl;
+
+    std.debug.assert(@intFromEnum(ipl) < @intFromEnum(ke.Ipl.Dispatch));
+
+    if (free_pages >= free_pages_limit) {
+        return;
+    }
+
+    var cnt = free_pages;
+
+    while (cnt < free_pages_limit) {
+        free_page_event.reset();
+
+        list_lock.release(ipl);
+
+        _ = ke.wait.wait_one(&free_page_event.hdr, null) catch unreachable;
+
+        ipl = list_lock.acquire();
+        cnt = free_pages;
+    }
+}
+
 /// Allocate a page of physical memory.
+/// This may block if no memory is available.
 pub fn alloc() r.PAddr {
     if (!bootstrapped) {
         @branchHint(.unlikely);
         return early_alloc();
     }
+    const ipl = list_lock.acquire();
+    defer list_lock.release(ipl);
+
+    wait_for_pages(ipl);
+
+    std.debug.assert(!free_list.is_empty());
+
+    const head = free_list.first();
+    head.remove();
+
+    _ = usable_memory.fetchSub(mm.page_size, .monotonic);
+    free_pages -= 1;
+
+    const elem: *mm.PageFree = @fieldParentPtr("link", head);
+    const page: *mm.Page = @ptrCast(elem);
+    const phys_addr = mm.pfn_to_page(mm.struct_page_to_pfn(page));
+
+    return phys_addr;
+}
+
+pub fn alloc_opts(opts: struct { policy: mm.WaitPolicy }) ?r.PAddr {
+    if (!bootstrapped) {
+        @branchHint(.unlikely);
+        return early_alloc();
+    }
 
     const ipl = list_lock.acquire();
+    defer list_lock.release(ipl);
 
-    if (free_list.is_empty()) {
-        list_lock.release(ipl);
-        // FIXME: wait for free memory
-        @panic("mm/phys: Out of memory for physical allocation");
+    if (free_list.is_empty() and opts.policy == .DontWaitForMemory) {
+        return null;
+    } else if (opts.policy == .WaitForMemory) {
+        wait_for_pages(ipl);
     }
 
+    std.debug.assert(free_pages > 0);
+    std.debug.assert(!free_list.is_empty());
+
     const head = free_list.first();
     head.remove();
 
     _ = usable_memory.fetchSub(mm.page_size, .monotonic);
+    free_pages -= 1;
 
     const elem: *mm.PageFree = @fieldParentPtr("link", head);
     const page: *mm.Page = @ptrCast(elem);
     const phys_addr = mm.pfn_to_page(mm.struct_page_to_pfn(page));
 
-    list_lock.release(ipl);
-
     return phys_addr;
 }
 
@@ -74,6 +134,10 @@ pub fn free(addr: r.PAddr) void {
     const ipl = list_lock.acquire();
     _ = usable_memory.fetchAdd(mm.page_size, .monotonic);
     free_list.insert_head(&page.free.link);
+    free_pages += 1;
+    if (free_pages >= free_pages_limit) {
+        free_page_event.signal();
+    }
     list_lock.release(ipl);
 }
 
@@ -81,17 +145,28 @@ pub fn free(addr: r.PAddr) void {
 pub fn free_batch(head: *mm.Page, tail: *mm.Page, count: usize) void {
     const ipl = list_lock.acquire();
 
-    tail.free.link.next = free_list.first();
-    free_list.insert_head(&head.free.link);
+    const first = free_list.first();
+
+    head.free.link.prev = &free_list.head;
+    tail.free.link.next = first;
+    first.prev = &tail.free.link;
+    free_list.head.next = &head.free.link;
 
     _ = usable_memory.fetchAdd(count * mm.page_size, .monotonic);
 
+    free_pages += count;
+
+    if (free_pages >= free_pages_limit) {
+        free_page_event.signal();
+    }
+
     list_lock.release(ipl);
 }
 
 pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void {
     memory_map = &boot_info.memory_map;
     free_list.init();
+    free_page_event.init(.Notification);
 
     var total_usable_memory: usize = 0;
     log.info("physical memory map:", .{});
@@ -133,11 +208,16 @@ pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void {
         const map_end = std.mem.alignForward(usize, exact_end, mm.page_size);
 
         // 3. Map the virtual pages.
-        mi.kernel_space.pmap.map_range_allocating(map_start, map_end - map_start, .{
-            .read = true,
-            .write = true,
-            .global = true,
-        });
+        mi.kernel_space.pmap.map_range_allocating(
+            map_start,
+            map_end - map_start,
+            .{
+                .read = true,
+                .write = true,
+                .global = true,
+            },
+            .DontWaitForMemory,
+        );
     }
 }
 
@@ -152,7 +232,12 @@ pub fn init_pfndb() void {
         const npages = entry.size / mm.page_size;
 
         for (0..npages) |j| {
-            free(entry.base + (j * mm.page_size));
+            const page: *mm.Page = mm.pfn_to_struct_page(mm.page_to_pfn(entry.base + (j * mm.page_size)));
+
+            free_list.insert_head(&page.free.link);
+            free_pages += 1;
+
+            _ = usable_memory.fetchAdd(mm.page_size, .monotonic);
         }
     }
 

src/mm/pmap.zig

@@ -53,9 +53,10 @@ const ContiguousSource = struct {
 /// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped.
 const AllocatingSource = struct {
     flags: mm.MapFlags,
+    policy: mm.WaitPolicy,
 
     pub fn next(self: *const AllocatingSource, _: r.VAddr) MapItem {
-        const pa = mi.phys.alloc();
+        const pa = mi.phys.alloc_opts(.{ .policy = self.policy }) orelse @panic("Out of memory");
         return .{
             .pa = pa,
             .len = mm.page_size,
@@ -95,9 +96,10 @@ pub const PMap = struct {
     }
 
     /// Map a virtual address range to physical addresses allocated on demand.
-    pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags) void {
+    pub fn map_range_allocating(self: *Self, va: r.VAddr, size: usize, flags: mm.MapFlags, policy: mm.WaitPolicy) void {
         const src = AllocatingSource{
             .flags = flags,
+            .policy = policy,
         };
 
         self.map_from(va, size, src);

src/mm/private.zig

@@ -14,7 +14,7 @@ const ke = @import("root").ke;
 pub const Space = struct {
     pmap: PMap,
     arena: vmem.Arena,
-    lock: ke.QSpinLock,
+    lock: ke.Mutex,
 };
 
 pub const PfnList = extern struct {

src/mm/radix_pmap.zig

@@ -148,12 +148,14 @@ pub fn RadixPmap(comptime Impl: type) type {
                 c.advance(mm.page_size);
             }
 
+            if (list.is_empty()) return null;
+
             const head_free: *mm.PageFree = @fieldParentPtr("link", list.first());
             const tail_free: *mm.PageFree = @fieldParentPtr("link", list.last());
             const head: *mm.Page = @ptrCast(head_free);
             const tail: *mm.Page = @ptrCast(tail_free);
 
-            return if (list.is_empty()) null else .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) };
+            return .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) };
         }
 
         /// Ensure all intermediate page tables down to `target_level` exist for

src/mm/root.zig

@@ -40,6 +40,11 @@ pub const MapFlags = packed struct {
     cache_disable: bool = false,
 };
 
+pub const WaitPolicy = enum {
+    WaitForMemory,
+    DontWaitForMemory,
+};
+
 pub const Error = error{ OutOfMemory, InvalidAddress, InvalidSize };
 
 pub inline fn page_to_pfn(addr: usize) Pfn {

src/mm/tlb.zig

@@ -20,10 +20,11 @@ fn reclaim_state(state: *ke.ShootdownState) void {
     const npages = state.npages;
 
     // Reclaim the virtual address space.
-    const ipl = space.lock.acquire();
+    space.lock.acquire();
 
     space.arena.free(base, @as(usize, npages) * mm.page_size) catch unreachable;
-    space.lock.release(ipl);
+
+    space.lock.release();
 
     _ = async_shootdowns.fetchAdd(1, .monotonic);
 

src/mm/zone.zig

@@ -258,7 +258,7 @@ pub const Zone = struct {
     /// Current color for slab coloring.
     color: usize,
     /// Zone lock.
-    lock: ke.QSpinLock,
+    lock: ke.Mutex,
     /// List of full slabs in this zone.
     full_slabs: rtl.List,
     /// List of partial slabs in this zone.
@@ -344,9 +344,7 @@ pub const Zone = struct {
     }
 
     /// Allocate an object from the zone.
-    pub fn alloc(self: *Self) mm.Error!*anyopaque {
-        const ipl = ke.ipl.raise(.Dispatch);
-        defer ke.ipl.lower(ipl);
+    pub fn alloc(self: *Self, opts: struct { policy: mm.WaitPolicy = .WaitForMemory }) mm.Error!*anyopaque {
         var buf: *anyopaque = undefined;
 
         // Try grabbing an object from the magazine layer.
@@ -354,8 +352,8 @@ pub const Zone = struct {
             @branchHint(.likely);
             const cpu = &self.cpus[ke.cpu.current()];
 
-            cpu.lock.acquire_no_ipl();
-            defer cpu.lock.release_no_ipl();
+            const ipl = cpu.lock.acquire();
+            defer cpu.lock.release(ipl);
 
             while (true) {
                 if (cpu.alloc_rounds > 0) {
@@ -407,13 +405,17 @@ pub const Zone = struct {
         }
 
         // Fall back to the slab layer.
-        self.lock.acquire_no_ipl();
-        defer self.lock.release_no_ipl();
+        self.lock.acquire();
+        defer self.lock.release();
 
         var slab: *Slab = undefined;
 
         if (self.partial_slabs.is_empty()) {
-            slab = try self.slab_create();
+            self.lock.release();
+
+            slab = try self.slab_create(opts.policy);
+
+            self.lock.acquire();
             self.partial_slabs.insert_tail(&slab.link);
         } else {
             slab = @fieldParentPtr("link", self.partial_slabs.first());
@@ -446,16 +448,14 @@ pub const Zone = struct {
 
     /// Free an object back to the zone.
     pub fn free(self: *Self, obj: *anyopaque) void {
-        const ipl = ke.ipl.raise(.Dispatch);
-        defer ke.ipl.lower(ipl);
-
         if (magazines_initialized and self.use_magazines) {
             @branchHint(.likely);
 
-            const cpu = &self.cpus[ke.cpu.current()];
+            const ipl = ke.ipl.raise(.Dispatch);
+            var cpu = &self.cpus[ke.cpu.current()];
 
             cpu.lock.acquire_no_ipl();
-            defer cpu.lock.release_no_ipl();
+            defer cpu.lock.release(ipl);
 
             while (true) {
                 if (cpu.free != null and cpu.free_rounds < cpu.magazine_size) {
@@ -495,11 +495,7 @@ pub const Zone = struct {
                 }
 
                 // No empty magazine in the depot, try to allocate a new one.
-                cpu.lock.release_no_ipl();
-
-                const new_mag: ?*Magazine = @ptrCast(@alignCast(self.magtype.?.zone.alloc() catch null));
-
-                cpu.lock.acquire_no_ipl();
+                const new_mag: ?*Magazine = @ptrCast(@alignCast(self.magtype.?.zone.alloc(.{ .policy = .DontWaitForMemory }) catch null));
 
                 if (new_mag) |m| {
                     self.free_to_depot(m, &self.empty_mags);
@@ -518,8 +514,8 @@ pub const Zone = struct {
             fill_with_poison(obj, self.obj_size, free_poison);
         }
 
-        self.lock.acquire_no_ipl();
-        defer self.lock.release_no_ipl();
+        self.lock.acquire();
+        defer self.lock.release();
 
         // Find the slab for the buffer.
         const slab: *Slab = if (self.obj_size > small_slab_size) blk: {
@@ -551,8 +547,8 @@ pub const Zone = struct {
         }
     }
 
-    fn alloc_page() !*anyopaque {
-        const alloc_ret = mi.phys.alloc();
+    fn alloc_page(policy: mm.WaitPolicy) !*anyopaque {
+        const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse return error.OutOfMemory;
         return @ptrFromInt(mm.p2v(alloc_ret));
     }
 
@@ -562,8 +558,8 @@ pub const Zone = struct {
         mi.phys.free(phys);
     }
 
-    fn slab_create_small(self: *Self) mm.Error!*Slab {
-        const buf = try alloc_page();
+    fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
+        const buf = try alloc_page(policy);
 
         // We employ a simple coloring scheme; every time a slab is created,
         // the color is shifted by the alignment. This is done until we reach the
@@ -604,7 +600,7 @@ pub const Zone = struct {
         return slab;
     }
 
-    fn slab_create_large(self: *Self) mm.Error!*Slab {
+    fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
         self.color += self.alignment;
 
         if (self.color > self.max_color) {
@@ -612,7 +608,7 @@ pub const Zone = struct {
         }
 
         const capacity = (self.slab_size - self.color) / self.chunk_size;
-        const buf = try mi.heap.alloc(self.slab_size);
+        const buf = try mi.heap.alloc(self.slab_size, policy);
 
         var ret: *Slab = @ptrCast(@alignCast(try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity))));
 
@@ -648,11 +644,11 @@ pub const Zone = struct {
         return ret;
     }
 
-    fn slab_create(self: *Self) mm.Error!*Slab {
+    fn slab_create(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
         if (self.obj_size <= small_slab_size) {
-            return self.slab_create_small();
+            return self.slab_create_small(policy);
         } else {
-            return self.slab_create_large();
+            return self.slab_create_large(policy);
         }
     }
 
@@ -723,7 +719,7 @@ pub fn TypedZone(comptime T: type) type {
         }
 
         pub fn create(self: *Self) mm.Error!*T {
-            const ret = try self.zone.alloc();
+            const ret = try self.zone.alloc(.{});
             return @ptrCast(@alignCast(ret));
         }
 
@@ -794,12 +790,12 @@ fn gpa_alloc(
     if (len > 2048) {
         if (ptr_align.toByteUnits() > mm.page_size) return null;
         const pages = std.mem.alignForward(usize, len, mm.page_size);
-        const ptr = mi.heap.alloc(pages) catch return null;
+        const ptr = mi.heap.alloc(pages, .WaitForMemory) catch return null;
         return @ptrCast(ptr);
     }
 
     const zone = zone_for(len, ptr_align.toByteUnits()) orelse return null;
-    const obj = zone.alloc() catch return null;
+    const obj = zone.alloc(.{}) catch return null;
     return @ptrCast(obj);
 }
 

src/platform/pc/smp.zig

@@ -107,7 +107,7 @@ pub fn init() linksection(r.init) void {
         // copy the offset into the AP's self_offset variable
         @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id];
 
-        const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32)) catch @panic("Failed to allocate AP stack")) + r.kib(32);
+        const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32), .DontWaitForMemory) catch @panic("Failed to allocate AP stack")) + r.kib(32);
 
         start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15);
 

src/proc/thread.zig

@@ -11,6 +11,7 @@ const mm = r.mm;
 const kernel_thread_stack_size = r.kib(16);
 
 var reaper_item: ex.WorkItem = undefined;
+var thread_zone: mm.zone.TypedZone(Thread) = undefined;
 
 /// Higher-level wrapper over a thread.
 pub const Thread = struct {
@@ -28,7 +29,7 @@ fn reap_thread(obj: *anyopaque, _: ?*anyopaque) void {
 
     psp.turnstile_zone.destroy(td.kern.turnstile);
     mm.heap.free(td.kern.stack, kernel_thread_stack_size);
-    mm.zone.gpa.destroy(td);
+    thread_zone.destroy(td);
 }
 
 fn reaper_fn(_: ?*anyopaque) void {
@@ -40,6 +41,7 @@ fn activation(_: *rtl.HandoffList) void {
 }
 
 pub fn init() void {
+    thread_zone.init("thread", .{});
     ke.thread.reaper_list.* = .init(activation);
     reaper_item.init(.High, reaper_fn, null);
 }
@@ -49,8 +51,8 @@ pub fn init() void {
 /// - `entry`: entry point.
 /// - `arg`: argument passed to `entry`.
 pub fn create_kernel(prio: u8, entry: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) !*Thread {
-    var td = try mm.zone.gpa.create(Thread);
-    const stack = try mm.heap.alloc(kernel_thread_stack_size);
+    var td = try thread_zone.create();
+    const stack = try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory);
 
     td.kern.init(@intFromPtr(stack), kernel_thread_stack_size, prio, entry, arg);
     td.kern.turnstile = try psp.turnstile_zone.create();