feat(mm): add allocators

Kconfig

@@ -49,8 +49,6 @@ config BALANCE_INTERVAL
 	   default 1000
 	   help
 		  This option sets the rate (in milliseconds) at which thread load balancing is perfomed.
-
-
 config NCPUS
 	   int "Maximal number of CPUs"
 	   default 1024
@@ -62,4 +60,22 @@ config LOG_BUFFER_SHIFT
          default 14
          help
              This option sets the size of the kernel log buffer as a power of two. The actual size is calculated as 2 raised to the power of this value (i.e., 2^LOG_BUFFER_SHIFT bytes).
+
+
+menu "Memory Management"
+menu "Kernel Allocator"
+
+config SLAB_POISON
+	   bool "Poison slab objects on allocation"
+	   default y
+
+config SLAB_POISON_CHECK
+	   bool "Verify poison state on allocation"
+	   default y
+
+
+endmenu
+
+endmenu
+
 endmenu

src/mm/amd64/impl.zig

@@ -13,7 +13,9 @@ pub fn virt_to_phys(vaddr: b.VAddr) usize {
     return vaddr - 0xffff800000000000;
 }
 
-pub const pfndb_base = 0xffffea0000000000;
+pub const hhdm_base = 0xffff800000000000;
+pub const kernel_heap_base = 0xffffe00000000000;
+pub const pfndb_base = 0xffffff0000000000;
 
 pub const PagingImpl = struct {
     const Self = @This();

src/mm/heap.zig

@@ -0,0 +1,26 @@
+const b = @import("base");
+const ke = b.ke;
+const mm = b.mm;
+const mi = mm.private;
+
+var heap_lock: ke.SpinLock = .init();
+var heap_arena: mm.vmem.Arena = undefined;
+
+pub fn init() void {
+    heap_arena.init("kernel heap", mi.impl.kernel_heap_base, b.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena");
+}
+
+pub fn alloc_pages(size: usize) mm.Error!*anyopaque {
+    const ipl = heap_lock.acquire();
+    defer heap_lock.release(ipl);
+
+    const addr = try heap_arena.alloc(size, .{});
+
+    mi.kernel_pmap.map_range_allocating(addr, size, .{
+        .read = true,
+        .write = true,
+        .global = true,
+    });
+
+    return @ptrFromInt(addr);
+}

src/mm/init.zig

@@ -78,9 +78,14 @@ fn map_kernel(boot_info: *pl.BootInfo) void {
     }
 }
 
+const std = @import("std");
+
 pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
     mi.phys.init(boot_info);
     map_kernel(boot_info);
     mi.kernel_pmap.activate();
     mi.phys.init_pfndb();
+    mi.zone.early_init();
+    mi.vmem.init();
+    mi.heap.init();
 }

src/mm/phys.zig

@@ -207,7 +207,7 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
     for (0..boot_info.memory_map.entry_count) |i| {
         const entry = boot_info.memory_map.entries[i];
 
-        std.log.info("mm/phys: [{x:0>16} - {x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) });
+        std.log.info("mm/phys: [{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) });
 
         if (entry.type == .Free) {
             total_usable_memory += entry.size;

src/mm/private.zig

@@ -3,6 +3,9 @@ pub const phys = @import("phys.zig");
 pub const radix_pmap = @import("radix_pmap.zig");
 pub const pmap = @import("pmap.zig");
 pub const init = @import("init.zig");
+pub const vmem = @import("vmem.zig");
+pub const heap = @import("heap.zig");
+pub const zone = @import("zone.zig");
 pub const PMap = pmap.PMap;
 
 pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64"))

src/mm/root.zig

@@ -6,6 +6,7 @@ const std = @import("std");
 pub const page_size = 4096;
 
 pub const init = private.init.init;
+pub const vmem = private.vmem;
 
 // 16 TB of maximum physical memory.
 // This should be fine on consumer hardware for at least a decade :^)
@@ -28,6 +29,8 @@ pub const MapFlags = packed struct {
     cache_disable: bool = false,
 };
 
+pub const Error = error{OutOfMemory, InvalidAddress, InvalidSize};
+
 pub inline fn page_to_pfn(addr: usize) Pfn {
     return @intCast(addr >> 12);
 }

src/mm/vmem.zig

@@ -0,0 +1,469 @@
+//! Generic resource allocator, based on the vmem allocator described in:
+//! Jeff Bonwick and Jonathan Adams'
+//! "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources"
+//!
+//! ## Overview
+//!
+//! A vmem arena manages a contiguous range of an arbitrary resource (virtual address
+//! space, PIDs, etc.) by tracking free and allocated regions as a list of segments.
+//! Each segment has a base address and a size.
+//!
+//! ## Data structures
+//!
+//! An arena is composed of three data structures:
+//!
+//! 1. Segment list: an address-ordered doubly-linked list of all segments, both
+//!    free and allocated. Used for coalescing on free (adjacent free segments are
+//!    merged) and for NextFit traversal.
+//!
+//! 2. Power-of-two free lists: an array of 64 lists, where bucket n holds free
+//!    segments whose size falls in [2^n, 2^(n+1)).
+//!
+//! 3. Allocated segment RB-tree: a tree of allocated segments keyed by base address.
+//!    Used for O(log n) lookup on free() given only an address.
+//!
+//! ## Allocation
+//!
+//! Three policies are supported:
+//!
+//! - `InstantFit`: find the first segment in the appropriate freelist bucket. O(1) for
+//!   unconstrained allocations; degrades under alignment or address constraints.
+//!
+//! - `BestFit`: scan freelist buckets for the smallest segment that satisfies the
+//!   request. Minimizes fragmentation at the cost of a fuller scan.
+//!
+//! - `NextFit`: continue searching from where the last allocation left off, wrapping
+//!   around at the end of the arena. Avoids reusing recently-freed addresses, making
+//!   it useful for resources like PIDs where cycling through values is desirable.
+//!
+//! ## Differences from the original paper
+//!
+//! This implementation omits several features from the original Solaris vmem:
+//!   - No importing: arenas are initialized with a fixed set of spans and cannot
+//!     grow by importing from a parent arena. This simplifies the implementation
+//!     significantly and is sufficient for fixed virtual address space management.
+//!
+//!   - No span markers: without importing there is no need to track which spans
+//!     were imported, so span segment types are omitted entirely.
+//!
+//!   - No quantum caching: the original vmem uses per-size slab caches (qcaches)
+//!     to accelerate small allocations. This is unnecessary here because vmem is
+//!     only called by the slab allocator for large slab-sized chunks, not for
+//!     individual small objects. This could change in the future if we find
+//!     more uses for the allocator.
+//!
+//!   - No hash table: allocated segments are tracked in an RB-tree rather than
+//!     the dynamic hash table used by Solaris vmem, avoiding the resize
+//!     complexity.
+const std = @import("std");
+const rtl = @import("rtl");
+const mm = @import("base").mm;
+const mi = mm.private;
+
+pub const Policy = enum {
+    /// Use the smallest free segment that can satisfy the request.
+    /// This tends to minimize fragmentation.
+    BestFit,
+
+    /// Provide a good approximation of the best fit policy in guaranteed
+    /// O(1) time. This is the default policy.
+    InstantFit,
+
+    /// Use the next free segment after the most recently allocated segment.
+    NextFit,
+};
+
+pub const AllocOptions = struct {
+    policy: Policy = .InstantFit,
+    alignment: ?usize = null,
+    min: ?usize = null,
+    max: ?usize = null,
+};
+
+const Segment = struct {
+    const Type = enum {
+        Allocated,
+        Free,
+    };
+
+    const Linkage = union {
+        /// Linkage into a freelist, valid when type == .Free
+        free_link: rtl.List.Entry,
+        /// Linkage into the allocated segments tree, valid when type == .Allocated
+        tree_link: rtl.bst.Node,
+    };
+    /// The type of the segment.
+    type: Type,
+    /// The base address of the segment.
+    base: usize,
+    /// The size of the segment in bytes.
+    size: usize,
+    /// Linkage into the arena's segment list
+    link: rtl.List.Entry,
+    /// Type-specific linkage.
+    linkage: Linkage,
+
+    fn from_free_link(entry: *rtl.List.Entry) *Segment {
+        const linkage: *Segment.Linkage = @fieldParentPtr("free_link", entry);
+        return @fieldParentPtr("linkage", linkage);
+    }
+
+    fn from_tree_link(node: *rtl.bst.Node) *Segment {
+        const linkage: *Segment.Linkage = @fieldParentPtr("tree_link", node);
+        return @fieldParentPtr("linkage", linkage);
+    }
+
+    fn cmp(a_node: *rtl.bst.Node, b_node: *rtl.bst.Node) std.math.Order {
+        const a: *Segment = Segment.from_tree_link(a_node);
+        const b: *Segment = Segment.from_tree_link(b_node);
+        return std.math.order(a.base, b.base);
+    }
+};
+
+/// One freelist for each power of two size that can fit within the host's
+/// address space.
+const freelist_count = @bitSizeOf(usize);
+
+var seg_zone: mi.zone.TypedZone(Segment) = undefined;
+
+pub const Arena = struct {
+    /// Name of the arena, for debugging purposes.
+    name: []const u8,
+    /// Start of initial span.
+    base: usize,
+    /// Size of initial span.
+    size: usize,
+    /// Unit of currency.
+    quantum: usize,
+    /// List of segments, sorted by base address.
+    list: rtl.List,
+    /// Power of two freelists for free segments, indexed by log2(size).
+    freelists: [freelist_count]rtl.List,
+    /// Tree for allocated segments, indexed by base address.
+    allocated_segments: rtl.RBTree(Segment.cmp),
+    /// Last segment allocated from, for NextFit policy.
+    rotor: ?*Segment,
+
+    const Self = @This();
+
+    /// Initialize the area for use.
+    pub fn init(self: *Self, name: []const u8, base: usize, size: usize, quantum: usize) !void {
+        self.name = name;
+        self.base = base;
+        self.size = size;
+        self.quantum = quantum;
+
+        rtl.List.init(&self.list);
+
+        for (&self.freelists) |*freelist| {
+            freelist.init();
+        }
+
+        self.allocated_segments = .init();
+        self.rotor = null;
+
+        // Add initial span
+       try self.add(base, size);
+    }
+
+    pub fn deinit(self: *Self) void {
+        // free all segments in the segment list
+        var entry = self.list.first();
+        while (entry != &self.list.head) {
+            const next = entry.next;
+            const seg: *Segment = @fieldParentPtr("link", entry);
+            free_segment(seg);
+            entry = next;
+        }
+    }
+
+    /// Add a span of memory to the arena.
+    pub fn add(self: *Self, addr: usize, size: usize) !void {
+        const new_seg = try self.alloc_segment();
+        new_seg.* = Segment{
+            .type = .Free,
+            .base = addr,
+            .size = size,
+            .link = undefined,
+            .linkage = .{ .free_link = undefined },
+        };
+
+        // Insert into segment list, sorted by base address.
+        // Start from the end since we're likely to be adding higher addresses.
+        var entry = self.list.last();
+        while (entry != &self.list.head) {
+            const s: *Segment = @fieldParentPtr("link", entry);
+            if (s.base < addr) break;
+            entry = entry.prev;
+        }
+
+        new_seg.link.insert_before(entry.next);
+        self.add_segment_to_freelist(new_seg);
+    }
+
+    /// Allocate a segment of memory from the arena.
+    /// Options:
+    /// - policy: allocation policy to use. Default is .InstantFit.
+    /// - alignment: if specified, the returned segment will be aligned to this boundary.
+    /// - min: if specified, the returned segment will be at least on this address.
+    /// - max: if specified, the returned segment will be at most on this address.
+    /// Returns the base address of the allocated segment.
+    pub fn alloc(
+        self: *Self,
+        size: usize,
+        options: AllocOptions,
+    ) mm.Error!usize {
+        if (size % self.quantum != 0) {
+            return error.InvalidSize;
+        }
+
+        const result = switch (options.policy) {
+            .InstantFit => self.instant_fit(size, options),
+            .BestFit => self.best_fit(size, options),
+            .NextFit => self.next_fit(size, options),
+        };
+
+        const start, var seg = result orelse return error.OutOfMemory;
+
+        std.debug.assert(seg.type == .Free);
+        std.debug.assert(seg.size >= size);
+
+        // Remove the segment from the freelist.
+        self.remove_segment_from_freelist(seg);
+
+        // left split: alignment pushed start forward, so there's a gap between
+        // seg.base and start that needs to become a free segment.
+        // e.g. seg=[0x0, 0x10000], start=0x100:
+        //   before: [0x0, 0x10000] free
+        //   after:  [0x0, 0x100] free, [0x100, 0x10000] being processed
+        if (seg.base != start) {
+            const left_seg = try self.alloc_segment();
+            left_seg.* = Segment{
+                .type = .Free,
+                .base = seg.base,
+                .size = start - seg.base,
+                .link = undefined,
+                .linkage = .{ .free_link = undefined },
+            };
+
+            seg.base = start;
+            seg.size -= left_seg.size;
+
+            left_seg.link.insert_before(&seg.link);
+            self.add_segment_to_freelist(left_seg);
+        }
+
+        // right split: seg is larger than needed, so split into allocated + free remainder.
+        // e.g. seg=[0x100, 0x10000], size=0x1000:
+        //   before: [0x100, 0x10000] being processed
+        //   after:  [0x100, 0x1100] allocated, [0x1100, 0x10000] free
+        // the quantum check avoids creating a remainder too small to ever be useful.
+        if (seg.size != size and (seg.size - size) > self.quantum - 1) {
+            const new_seg = try self.alloc_segment();
+            new_seg.* = .{ .type = .Allocated, .base = seg.base, .size = size, .link = undefined, .linkage = .{ .tree_link = undefined } };
+
+            seg.base += size;
+            seg.size -= size;
+            seg.type = .Free;
+            seg.linkage = .{ .free_link = undefined };
+
+            new_seg.link.insert_before(&seg.link);
+            self.add_segment_to_freelist(seg);
+
+            self.allocated_segments.insert(&new_seg.linkage.tree_link) catch return error.InvalidAddress;
+
+            if (options.policy == .NextFit) self.rotor = new_seg;
+            return new_seg.base;
+        } else {
+            // seg is exactly the right size (or remainder is too small), use it directly.
+            seg.type = .Allocated;
+            seg.linkage = .{ .tree_link = undefined };
+            self.allocated_segments.insert(&seg.linkage.tree_link) catch return error.InvalidAddress;
+
+            if (options.policy == .NextFit) self.rotor = seg;
+            return seg.base;
+        }
+    }
+
+    pub fn free(self: *Self, addr: usize, size: usize) mm.Error!void {
+        // Find the allocated segment containing addr.
+        var search_node = Segment{
+            .type = .Allocated,
+            .base = addr,
+            .size = 0,
+            .link = undefined,
+            .linkage = .{ .tree_link = undefined },
+        };
+
+        const node = self.allocated_segments.tree.search(&search_node.linkage.tree_link) orelse return error.InvalidAddress;
+
+        const seg = Segment.from_tree_link(node);
+
+        if (seg.size != size) return error.InvalidAddress;
+
+        // Remove from allocated tree.
+        self.allocated_segments.delete(node);
+
+        // Coalesce to the right.
+        const next_entry = seg.link.next;
+        if (next_entry != &self.list.head) {
+            const next_seg: *Segment = @fieldParentPtr("link", next_entry);
+
+            if (self.rotor == next_seg) self.rotor = seg;
+
+            if (next_seg.type == .Free) {
+                // Remove next segment from freelist and segment list, then merge into seg.
+                self.remove_segment_from_freelist(next_seg);
+                next_seg.link.remove();
+                seg.size += next_seg.size;
+                self.free_segment(next_seg);
+            }
+        }
+
+        // Coalesce to the left.
+        const prev_entry = seg.link.prev;
+        if (prev_entry != &self.list.head) {
+            const prev_seg: *Segment = @fieldParentPtr("link", prev_entry);
+
+            if (self.rotor == prev_seg) self.rotor = seg;
+
+            if (prev_seg.type == .Free) {
+                // Remove previous segment from freelist and segment list, then merge into seg.
+                self.remove_segment_from_freelist(prev_seg);
+                prev_seg.link.remove();
+                seg.base = prev_seg.base;
+                seg.size += prev_seg.size;
+                self.free_segment(prev_seg);
+            }
+        }
+
+        // Mark as free and add to freelist.
+        seg.type = .Free;
+        seg.linkage = .{ .free_link = undefined };
+        self.add_segment_to_freelist(seg);
+    }
+
+    // Implementation for the instant fit policy.
+    fn instant_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } {
+        var idx = freelist_index(size);
+
+        if (!std.math.isPowerOfTwo(size)) idx += 1;
+
+        // Simply grab the first segment from the appropriate freelist
+        // that can satisfy the request.
+        // This is O(1) unless there are contraints.
+        while (idx < freelist_count) : (idx += 1) {
+            var it = self.freelists[idx].iterator();
+            while (it.next()) : (it.advance()) {
+                const seg = Segment.from_free_link(it.get());
+                if (self.try_to_fit(seg, size, options)) |addr| return .{ addr, seg };
+            }
+        }
+        return null;
+    }
+
+    // Implementation for the best fit policy.
+    // Find the smallest segment that can satisfy the request.
+    fn best_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } {
+        var idx = freelist_index(size);
+        var best_seg: ?*Segment = null;
+        var best_start: usize = 0;
+
+        while (idx < freelist_count) : (idx += 1) {
+            var it = self.freelists[idx].iterator();
+
+            while (it.next()) : (it.advance()) {
+                const seg = Segment.from_free_link(it.get());
+                if (self.try_to_fit(seg, size, options)) |addr| {
+                    if (best_seg == null or seg.size < best_seg.?.size) {
+                        best_seg = seg;
+                        best_start = addr;
+                    }
+                }
+            }
+
+            if (best_seg != null) break;
+        }
+        return if (best_seg) |s| .{ best_start, s } else null;
+    }
+
+    // Implementation for the next fit policy.
+    // Continue searching from the last allocated segment, wrapping around at the end of the arena.
+    fn next_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } {
+        // Start from rotor if we have one, otherwise from the beginning.
+        const start_entry = if (self.rotor) |r| r.link.next else self.list.first();
+
+        // Search from rotor to end of list.
+        var entry = start_entry;
+        while (entry != &self.list.head) : (entry = entry.next) {
+            const seg: *Segment = @fieldParentPtr("link", entry);
+            if (seg.type != .Free) continue;
+            if (self.try_to_fit(seg, size, options)) |addr| {
+                return .{ addr, seg };
+            }
+        }
+
+        // Wrap around and search from beginning to rotor.
+        entry = self.list.first();
+        while (entry != start_entry) : (entry = entry.next) {
+            const seg: *Segment = @fieldParentPtr("link", entry);
+            if (seg.type != .Free) continue;
+            if (self.try_to_fit(seg, size, options)) |addr| {
+                return .{ addr, seg };
+            }
+        }
+
+        return null;
+    }
+
+    // Try to fit the given segment to the request with the given options.
+    // Returns the fitted address if successful.
+    fn try_to_fit(self: *Self, segment: *Segment, size: usize, options: AllocOptions) ?usize {
+        var start = @max(segment.base, options.min orelse 0);
+        const end = @min(segment.base + segment.size, options.max orelse std.math.maxInt(usize));
+        const alignment = options.alignment orelse self.quantum;
+
+        if (start > end) return null;
+
+        // Align start to the required alignment.
+        start = std.mem.alignForward(usize, start, alignment);
+
+        if (start + size <= end) {
+            return start;
+        }
+
+        return null;
+    }
+
+    fn alloc_segment(self: *Self) !*Segment {
+        _ = self;
+        return seg_zone.create();
+    }
+
+    fn free_segment(self: *Self, segment: *Segment) void {
+        seg_zone.destroy(segment);
+        _ = self;
+    }
+
+    fn add_segment_to_freelist(self: *Self, segment: *Segment) void {
+        const freelist = self.freelist_for_size(segment.size);
+        freelist.insert_tail(&segment.linkage.free_link);
+    }
+
+    fn remove_segment_from_freelist(self: *Self, segment: *Segment) void {
+        _ = self;
+        segment.linkage.free_link.remove();
+    }
+
+    inline fn freelist_index(size: usize) usize {
+        return std.math.log2_int(usize, size);
+    }
+
+    inline fn freelist_for_size(self: *Self, size: usize) *rtl.List {
+        return &self.freelists[freelist_index(size)];
+    }
+};
+
+pub fn init() void {
+    seg_zone.init("seg", .{});
+}

src/mm/zone.zig

@@ -0,0 +1,724 @@
+//! Slab allocator implementation with per-CPU object caching.
+//! As described in Bonwick's "The Slab Allocator: An Object-Caching Kernel Memory Allocator"
+//! and "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources".
+const rtl = @import("rtl");
+const b = @import("base");
+const config = @import("config");
+const std = @import("std");
+const ke = b.ke;
+const mm = b.mm;
+const mi = mm.private;
+
+// TODO: when SMP is setup, do proper magazine initialization.
+
+/// A bufctl is a control structure for a single object in a slab.
+/// It is used to track free objects and link them together in the slab's free list.
+const BufCtl = struct {
+    /// Next BufCtl in the list.
+    next: ?*BufCtl,
+    /// Pointer to the object this BufCtl manages.
+    buffer: *anyopaque,
+    /// Pointer to the owning Slab.
+    slab: *Slab,
+};
+
+/// A slab is a contiguous memory region that holds multiple objects of the same type.
+const Slab = struct {
+    /// Linkage into a zone's list of slabs.
+    link: rtl.List.Entry,
+    /// Reference count indicating how many objects are in use.
+    refcount: usize,
+    /// Pointer to the first free BufCtl in this slab.
+    buflist: ?*BufCtl,
+    /// Base address of the slab.
+    base: usize,
+};
+
+/// A magazine is a per-CPU cache of objects for a zone.
+const Magazine = struct {
+    /// Next magazine in the list.
+    next: ?*Magazine,
+    /// One or more rounds.
+    rounds: [*]*anyopaque,
+};
+
+const MagazineType = struct {
+    rounds: usize,
+    alignment: usize,
+    minbuf: usize,
+    maxbuf: usize,
+    /// Magazine zone.
+    zone: *Zone,
+};
+
+/// Per-CPU state for a zone.
+const Cpu = struct {
+    lock: ke.SpinLock,
+    /// Magazine used for allocations.
+    alloc: ?*Magazine,
+    /// Magazine used for frees.
+    free: ?*Magazine,
+    /// Number of rounds in alloc magazine.
+    alloc_rounds: usize,
+    /// Number of rounds in free magazine.
+    free_rounds: usize,
+    /// Number of rounds in a full magazine.
+    magazine_size: usize,
+};
+
+const initial_hash = 32;
+
+const slab_align = 8;
+
+/// Global count of zones.
+const zones_num = 32;
+
+/// Every power-of-two-size from 8 to 2048 (inclusively).
+const generic_zones_num = 9;
+
+const generic_zone_names = [_][]const u8{
+    "zone-8",
+    "zone-16",
+    "zone-32",
+    "zone-64",
+    "zone-128",
+    "zone-256",
+    "zone-512",
+    "zone-1024",
+    "zone-2048",
+};
+
+/// 1/8th of a page.
+const small_slab_size = 512;
+
+/// This is an arbitrary number, ideally this should be determined on zone
+/// creation based on the object size and slab size,
+/// but for simplicity we just use a constant here.
+const objects_per_slab = 16;
+
+/// Zones used to allocate out-of-line bufctls for large slabs.
+var bufctl_zone: TypedZone(BufCtl) = undefined;
+var slab_zone: TypedZone(Slab) = undefined;
+
+/// Generic zones for power-of-two sizes from 8 to 2048.
+var generic_zones: [generic_zones_num]Zone = undefined;
+
+var magazines_enabled = false;
+
+const magtypes = [_]MagazineType{
+    .{ .rounds = 1, .alignment = 8, .minbuf = 3200, .maxbuf = 65536, .zone = undefined },
+    .{ .rounds = 3, .alignment = 16, .minbuf = 256, .maxbuf = 32768, .zone = undefined },
+    .{ .rounds = 7, .alignment = 32, .minbuf = 64, .maxbuf = 16384, .zone = undefined },
+    .{ .rounds = 15, .alignment = 64, .minbuf = 0, .maxbuf = 8192, .zone = undefined },
+    .{ .rounds = 31, .alignment = 64, .minbuf = 0, .maxbuf = 4096, .zone = undefined },
+    .{ .rounds = 47, .alignment = 64, .minbuf = 0, .maxbuf = 2048, .zone = undefined },
+    .{ .rounds = 63, .alignment = 64, .minbuf = 0, .maxbuf = 1024, .zone = undefined },
+    .{ .rounds = 95, .alignment = 64, .minbuf = 0, .maxbuf = 512, .zone = undefined },
+    .{ .rounds = 143, .alignment = 64, .minbuf = 0, .maxbuf = 0, .zone = undefined },
+};
+
+const alloc_poison: u32 = 0xBADDC0DE;
+const free_poison: u32 = 0xDEADBEEF;
+const is_poison_enabled = @hasDecl(config, "CONFIG_SLAB_POISON");
+const should_check_poison = @hasDecl(config, "CONFIG_SLAB_POISON_CHECK");
+
+inline fn fill_with_poison(ptr: *anyopaque, len: usize, pattern: u32) void {
+    const bytes: [*]u8 = @ptrCast(ptr);
+    var i: usize = 0;
+
+    while (i + 4 <= len) : (i += 4) {
+        std.mem.writeInt(u32, bytes[i..][0..4], pattern, .little);
+    }
+
+    if (i < len) {
+        var tail: [4]u8 = undefined;
+        std.mem.writeInt(u32, &tail, pattern, .little);
+        var j: usize = 0;
+        while (i < len) : ({
+            i += 1;
+            j += 1;
+        }) bytes[i] = tail[j];
+    }
+}
+
+inline fn check_poison(ptr: *anyopaque, len: usize, pattern: u32) bool {
+    if (!should_check_poison) return true;
+
+    const bytes: [*]const u8 = @ptrCast(ptr);
+    var i: usize = 0;
+
+    while (i + 4 <= len) : (i += 4) {
+        if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern) return false;
+    }
+
+    if (i < len) {
+        var tail: [4]u8 = undefined;
+        std.mem.writeInt(u32, &tail, pattern, .little);
+        var j: usize = 0;
+        while (i < len) : ({
+            i += 1;
+            j += 1;
+        }) {
+            if (bytes[i] != tail[j]) return false;
+        }
+    }
+
+    return true;
+}
+
+pub const Zone = struct {
+    /// Name used for debugging purposes.
+    name: []const u8,
+    /// Size of objects in this zone.
+    obj_size: usize,
+    /// Alignment of objects in this zone.
+    alignment: usize,
+    /// Size of a slab.
+    slab_size: usize,
+    /// Size of a chunk.
+    chunk_size: usize,
+    /// Offset for buf-to-bufctl conversion.
+    offset: usize,
+    /// Max color for slab coloring.
+    max_color: usize,
+    /// Current color for slab coloring.
+    color: usize,
+    /// Zone lock.
+    lock: ke.SpinLock,
+    /// List of full slabs in this zone.
+    full_slabs: rtl.List,
+    /// List of partial slabs in this zone.
+    partial_slabs: rtl.List,
+    /// Linkage into the global static zone free list.
+    next: ?*Zone,
+    /// Buffer-to-bufctl hash map.
+    bufmap: [*]?*BufCtl,
+
+    hash_mask: usize,
+    hash_shift: usize,
+
+    depot_lock: ke.SpinLock,
+
+    empty_mags: ?*Magazine,
+    full_mags: ?*Magazine,
+    magtype: ?MagazineType,
+    /// Per-CPU state.
+    cpus: [*]Cpu,
+
+    ctor: ?*const fn (obj: *anyopaque) void,
+    dtor: ?*const fn (obj: *anyopaque) void,
+
+    hash0: [initial_hash]?*BufCtl,
+
+    const Self = @This();
+
+    pub const InitOptions = struct {
+        alignment: usize = 0,
+        ctor: ?*const fn (*anyopaque) void = null,
+        dtor: ?*const fn (*anyopaque) void = null,
+    };
+
+    /// Initialize a zone.
+    pub fn init(self: *Self, name: []const u8, size: usize, options: InitOptions) void {
+        const obj_align = if (options.alignment == 0) slab_align else options.alignment;
+        var chunk_size = std.mem.alignForward(usize, size, obj_align);
+
+        const offset, chunk_size = if (size <= small_slab_size) blk: {
+            if (chunk_size - size >= @sizeOf(usize)) {
+                // Use padding at end of chunk.
+                break :blk .{ chunk_size - @sizeOf(usize), chunk_size };
+            } else {
+                // No space, extend the chunk.
+                break :blk .{ chunk_size, chunk_size + slab_align };
+            }
+        } else blk: {
+            // Out-of-line bufctl
+            break :blk .{ 0, chunk_size };
+        };
+
+        const slab_size, const max_color = if (size <= small_slab_size) blk: {
+            const ss = mm.page_size;
+            break :blk .{ ss, @rem(ss - @sizeOf(Slab), chunk_size) };
+        } else blk: {
+            const ss = calc_slab_size(chunk_size);
+            break :blk .{ ss, @rem(ss, chunk_size) };
+        };
+
+        self.full_slabs.init();
+        self.partial_slabs.init();
+
+        self.name = name;
+        self.obj_size = size;
+        self.alignment = obj_align;
+        self.slab_size = slab_size;
+        self.chunk_size = chunk_size;
+        self.max_color = max_color;
+        self.color = 0;
+        self.offset = offset;
+        self.ctor = options.ctor;
+        self.dtor = options.dtor;
+        self.hash_mask = initial_hash - 1;
+        self.hash_shift = std.math.log2_int_ceil(usize, chunk_size);
+        self.lock = .init();
+        self.bufmap = &self.hash0;
+
+        self.magtype = null;
+        for (&magtypes) |*mtype| {
+            if (mtype.maxbuf >= chunk_size) {
+                self.magtype = mtype.*;
+                break;
+            }
+        }
+    }
+
+    /// Allocate an object from the zone.
+    pub fn alloc(self: *Self) mm.Error!*anyopaque {
+        const ipl = ke.ipl.raise(.Dispatch);
+        defer ke.ipl.lower(ipl);
+        var buf: *anyopaque = undefined;
+
+        // Try grabbing an object from the magazine layer.
+        if (magazines_enabled) {
+            const cpu = &self.cpus[ke.cpu.current()];
+
+            cpu.lock.acquire_no_ipl();
+            defer cpu.lock.release_no_ipl();
+
+            while (true) {
+                if (cpu.alloc_rounds > 0) {
+                    // Fast path: just pop from the alloc magazine.
+                    cpu.alloc_rounds -= 1;
+                    buf = cpu.alloc.?.rounds[cpu.alloc_rounds];
+
+                    if (is_poison_enabled) {
+                        if (!check_poison(buf, self.obj_size, free_poison)) {
+                            @panic("slab poison mismatch: object corrupted after free");
+                        }
+
+                        fill_with_poison(buf, self.obj_size, alloc_poison);
+
+                        if (self.ctor) |ctor| {
+                            ctor(buf);
+                        }
+                    }
+
+                    return buf;
+                }
+
+                if (cpu.free_rounds > 0) {
+                    std.debug.assert(cpu.free != null);
+
+                    // Alloc magazine is empty. If the free magazine has items, swap them.
+                    // The free magazine now becomes our alloc magazine.
+                    std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free);
+                    std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds);
+                    continue;
+                }
+
+                // Both magazines are empty.
+                // Try to get a full magazine from the depot.
+                if (self.alloc_from_depot(&self.full_mags)) |full_mag| {
+                    // Discard our empty alloc magazine to the depot.
+                    if (cpu.alloc) |empty_mag| {
+                        self.free_to_depot(empty_mag, &self.empty_mags);
+                    }
+
+                    cpu.alloc = full_mag;
+                    cpu.alloc_rounds = cpu.magazine_size;
+                    continue;
+                }
+
+                // Depot has no full magazines, fall back to the slab layer..
+                break;
+            }
+        }
+
+        // Fall back to the slab layer.
+        self.lock.acquire_no_ipl();
+        defer self.lock.release_no_ipl();
+
+        var slab: *Slab = undefined;
+
+        if (self.partial_slabs.is_empty()) {
+            slab = try self.slab_create();
+            self.partial_slabs.insert_tail(&slab.link);
+        } else {
+            slab = @fieldParentPtr("link", self.partial_slabs.first());
+        }
+
+        var bufctl = slab.buflist orelse return error.OutOfMemory;
+
+        if (bufctl.next == null) {
+            slab.link.remove();
+            self.full_slabs.insert_tail(&slab.link);
+        }
+
+        slab.buflist = bufctl.next;
+        slab.refcount += 1;
+
+        if (self.obj_size > small_slab_size) {
+            const bucket = self.bufctl_from_obj(bufctl.buffer);
+
+            buf = bufctl.buffer;
+            bufctl.next = bucket.*;
+            bucket.* = bufctl;
+        } else {
+            buf = @ptrFromInt(@intFromPtr(bufctl) - self.offset);
+        }
+
+        if (is_poison_enabled) {
+            if (!check_poison(buf, self.obj_size, free_poison)) {
+                @panic("slab poison mismatch: object corrupted after free");
+            }
+
+            fill_with_poison(buf, self.obj_size, alloc_poison);
+
+            if (self.ctor) |ctor| {
+                ctor(buf);
+            }
+        }
+
+        return buf;
+    }
+
+    /// Free an object back to the zone.
+    pub fn free(self: *Self, obj: *anyopaque) void {
+        const ipl = ke.ipl.raise(.Dispatch);
+        defer ke.ipl.lower(ipl);
+
+        if (magazines_enabled) {
+            const cpu = &self.cpus[ke.cpu.current()];
+
+            cpu.lock.acquire_no_ipl();
+            defer cpu.lock.release_no_ipl();
+
+            while (true) {
+                if (cpu.free_rounds < cpu.magazine_size) {
+                    // Fast path: push directly to free magazine.
+                    if (is_poison_enabled) {
+                        if (self.dtor) |dtor| {
+                            dtor(obj);
+                        }
+
+                        fill_with_poison(obj, self.obj_size, free_poison);
+                    }
+
+                    cpu.free.?.rounds[cpu.free_rounds] = obj;
+                    cpu.free_rounds += 1;
+                    return;
+                }
+
+                if (cpu.alloc_rounds == 0 and cpu.alloc != null) {
+                    // Free magazine is full. If the alloc magazine is completely empty, swap them.
+                    // The empty alloc magazine becomes our new free magazine.
+                    std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free);
+                    std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds);
+                    continue;
+                }
+
+                // Free magazine is full, and alloc magazine has items.
+                // Try to get an empty magazine from the depot.
+                if (self.alloc_from_depot(&self.empty_mags)) |empty_mag| {
+                    // Put the Free magazine on the full list.
+                    if (cpu.free) |full_mag| {
+                        self.free_to_depot(full_mag, &self.full_mags);
+                    }
+
+                    cpu.free = empty_mag;
+                    cpu.free_rounds = 0;
+                    continue;
+                }
+
+                // No empty magazine in the depot, try to allocate a new one.
+                cpu.lock.release_no_ipl();
+
+                const new_mag: ?*Magazine = @ptrCast(@alignCast(self.magtype.?.zone.alloc() catch null));
+
+                cpu.lock.acquire_no_ipl();
+
+                if (new_mag) |m| {
+                    self.free_to_depot(m, &self.empty_mags);
+                    cpu.free = m;
+                    cpu.free_rounds = 0;
+                    continue;
+                }
+
+                // Failed to allocate a magazine. Fall back to the slab layer.
+                break;
+            }
+        }
+
+        if (is_poison_enabled) {
+            if (self.dtor) |dtor| {
+                dtor(obj);
+            }
+            fill_with_poison(obj, self.obj_size, free_poison);
+        }
+
+        self.lock.acquire_no_ipl();
+        defer self.lock.release_no_ipl();
+
+        // Find the bufctl for the buffer.
+        var bufctl: ?*BufCtl = null;
+        var slab: ?*Slab = null;
+
+        if (self.obj_size > small_slab_size) {
+            // Search through the hash map and remove it.
+            var prev = self.bufctl_from_obj(@ptrCast(obj));
+
+            while (prev.* != null) {
+                const bc = prev.*;
+
+                if (@intFromPtr(bc.?.buffer) == @intFromPtr(obj)) {
+                    prev.* = bc.?.next;
+                    slab = bc.?.slab;
+                    bufctl = bc.?;
+                    break;
+                }
+
+                prev = &bc.?.next;
+            }
+        } else {
+            bufctl = @ptrFromInt(@intFromPtr(obj) + self.offset);
+            slab = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size) + mm.page_size - @sizeOf(Slab));
+        }
+
+        if (slab == null or bufctl == null) {
+            return;
+        }
+
+        const s = slab.?;
+        const bufc = bufctl.?;
+
+        if (s.buflist == null) {
+            // There were no buffers in the slab, so it wasn't in the freelist.
+            // Now that there is a buffer, add it back to the freelist.
+            s.link.remove();
+            self.partial_slabs.insert_head(&s.link);
+        }
+
+        // Insert bufctl into the slab freelist.
+        bufc.next = s.buflist;
+        s.buflist = bufc;
+
+        s.refcount -= 1;
+
+        if (s.refcount == 0) {
+            // No more outstanding allocations, it is safe to reclaim the slab.
+            s.link.remove();
+            self.slab_destroy(s);
+            return;
+        }
+    }
+
+    inline fn bufctl_from_obj(self: *Self, buffer: *anyopaque) *?*BufCtl {
+        return &self.bufmap[(@intFromPtr(buffer) >> @intCast(self.hash_shift)) & self.hash_mask];
+    }
+
+    fn alloc_page() !*anyopaque {
+        const alloc_ret = mi.phys.alloc();
+
+        return @ptrFromInt(mm.p2v(alloc_ret));
+    }
+
+    fn free_page(p: *anyopaque) void {
+        const phys = mm.v2p(@intFromPtr(p));
+        mi.phys.free(phys);
+    }
+
+    fn slab_create_small(self: *Self) mm.Error!*Slab {
+        var buf = try alloc_page();
+
+        // We employ a simple coloring scheme; every time a slab is created,
+        // the color is shifted by the alignment. This is done until we reach the
+        // max color (which is when there is no space left in the slab buffer).
+        // This ensures uniform buffer address distribution.
+        self.color += self.alignment;
+
+        if (self.color > self.max_color) {
+            self.color = 0;
+        }
+
+        // Slab info is located at the end of the page.
+        var slab: *Slab = @ptrFromInt(@intFromPtr(buf) + mm.page_size - @sizeOf(Slab));
+
+        const capacity = (mm.page_size - @sizeOf(Slab) - self.color) / self.chunk_size;
+
+        slab.buflist = null;
+        slab.refcount = 0;
+        slab.base = @intFromPtr(buf);
+
+        buf = @ptrFromInt(@intFromPtr(buf) + self.color);
+
+        for (0..capacity) |i| {
+            var bufctl: *BufCtl = @ptrFromInt(@intFromPtr(buf) + (i * self.chunk_size) + self.offset);
+            const obj: *anyopaque = @ptrFromInt(@intFromPtr(buf) + (i * self.chunk_size));
+
+            // Add bufctl to the freelist
+            // NOTE: a bufctl field other than next must not be modified.
+            bufctl.next = slab.buflist;
+            slab.buflist = bufctl;
+
+            if (is_poison_enabled) {
+                fill_with_poison(obj, self.obj_size, free_poison);
+            } else {
+                if (self.ctor) |ctor| ctor(obj);
+            }
+        }
+
+        return slab;
+    }
+
+    fn slab_create_large(self: *Self) !*Slab {
+        self.color += self.alignment;
+
+        if (self.color > self.max_color) {
+            self.color = 0;
+        }
+
+        const capacity = (self.slab_size - self.color) / self.chunk_size;
+        const buf = try mi.heap.alloc_pages(self.slab_size);
+
+        var ret: *Slab = try slab_zone.create();
+
+        ret.buflist = null;
+        ret.base = @intFromPtr(buf);
+
+        for (0..capacity) |i| {
+            var bufctl: *BufCtl = try bufctl_zone.create();
+
+            bufctl.buffer = @ptrFromInt(@intFromPtr(buf) + self.color + i * self.chunk_size);
+            bufctl.slab = ret;
+
+            bufctl.next = ret.buflist;
+            ret.buflist = bufctl;
+
+            if (is_poison_enabled) {
+                fill_with_poison(bufctl.buffer, self.obj_size, free_poison);
+            } else {
+                if (self.ctor) |ctor| ctor(bufctl.buffer);
+            }
+        }
+
+        return ret;
+    }
+
+    fn slab_create(self: *Self) mm.Error!*Slab {
+        if (self.obj_size <= small_slab_size) {
+            return self.slab_create_small();
+        } else {
+            return self.slab_create_large();
+        }
+    }
+
+    fn slab_destroy(self: *Self, slab: *Slab) void {
+        if (!is_poison_enabled) {
+            if (self.dtor) |dtor| {
+                var bufctl = slab.buflist;
+                while (bufctl) |bc| : (bufctl = bc.next) {
+                    const obj = if (self.obj_size > small_slab_size)
+                        bc.buffer
+                    else
+                        @as(*anyopaque, @ptrFromInt(@intFromPtr(bc) - self.offset));
+                    dtor(obj);
+                }
+            }
+        }
+
+        if (self.obj_size > small_slab_size) {
+            var bufctl = slab.buflist;
+
+            while (bufctl != null) {
+                const next = bufctl.?.next;
+                bufctl_zone.destroy(bufctl.?);
+                bufctl = next;
+            }
+
+            slab_zone.destroy(slab);
+
+            // FIXME: call free_pages on slab.base here!
+        } else {
+            free_page(@ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size)));
+        }
+    }
+
+    fn maglist_alloc(list_head: *?*Magazine) ?*Magazine {
+        const ret = list_head.* orelse return null;
+        list_head.* = ret.next;
+
+        return ret;
+    }
+
+    fn maglist_free(list_head: *?*Magazine, mag: *Magazine) void {
+        mag.next = list_head.*;
+        list_head.* = mag;
+    }
+
+    fn alloc_from_depot(self: *Self, maglist: *?*Magazine) ?*Magazine {
+        self.depot_lock.acquire_no_ipl();
+        const ret = maglist_alloc(maglist);
+        self.depot_lock.release_no_ipl();
+        return ret;
+    }
+
+    fn free_to_depot(self: *Self, mag: *Magazine, maglist: *?*Magazine) void {
+        self.depot_lock.acquire_no_ipl();
+        maglist_free(maglist, mag);
+        self.depot_lock.release_no_ipl();
+    }
+
+    fn calc_slab_size(chunk_size: usize) usize {
+        // Try increasing page multiples until waste is under 12.5%.
+        var order: u5 = 0;
+        while (order <= 8) : (order += 1) {
+            const slab_size = @as(usize, mm.page_size) << order;
+            const n_objs = slab_size / chunk_size;
+            if (n_objs == 0) continue;
+            const waste = slab_size - (n_objs * chunk_size);
+            // Accept if waste < 1/8th of slab.
+            if (waste * 8 <= slab_size) return slab_size;
+        }
+        // Fallback: just fit one object, rounded up to page boundary
+        return std.mem.alignForward(usize, chunk_size, mm.page_size);
+    }
+};
+
+/// Parameterized version of Zone, useful for object caches.
+pub fn TypedZone(comptime T: type) type {
+    return struct {
+        zone: Zone,
+
+        const Self = @This();
+
+        pub const InitOptions = struct {
+            ctor: ?*const fn (*T) void = null,
+            dtor: ?*const fn (*T) void = null,
+        };
+
+        pub fn init(self: *@This(), name: []const u8, options: InitOptions) void {
+            self.zone.init(name, @sizeOf(T), .{
+                .alignment = @alignOf(T),
+                .ctor = @ptrCast(options.ctor),
+                .dtor = @ptrCast(options.dtor),
+            });
+        }
+
+        pub fn create(self: *Self) mm.Error!*T {
+            const ret = try self.zone.alloc();
+            return @ptrCast(@alignCast(ret));
+        }
+
+        pub fn destroy(self: *Self, obj: *T) void {
+            self.zone.free(obj);
+        }
+    };
+}
+
+pub fn early_init() linksection(b.init) void {
+    bufctl_zone.init("bufctl", .{});
+    slab_zone.init("slab", .{});
+
+    for (0..generic_zones.len, &generic_zones) |i, *zone| {
+        zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{});
+    }
+}