From 748854577edf45d4b63d58be160f83dd7e728b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Fri, 10 Apr 2026 18:04:34 -0400 Subject: [PATCH] feat(mm): add allocators --- Kconfig | 20 +- src/mm/amd64/impl.zig | 4 +- src/mm/heap.zig | 26 ++ src/mm/init.zig | 5 + src/mm/phys.zig | 2 +- src/mm/private.zig | 3 + src/mm/root.zig | 3 + src/mm/vmem.zig | 469 +++++++++++++++++++++++++++ src/mm/zone.zig | 724 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 1252 insertions(+), 4 deletions(-) create mode 100644 src/mm/heap.zig create mode 100644 src/mm/vmem.zig create mode 100644 src/mm/zone.zig diff --git a/Kconfig b/Kconfig index 3590454..e9d67f2 100644 --- a/Kconfig +++ b/Kconfig @@ -49,8 +49,6 @@ config BALANCE_INTERVAL default 1000 help This option sets the rate (in milliseconds) at which thread load balancing is perfomed. - - config NCPUS int "Maximal number of CPUs" default 1024 @@ -62,4 +60,22 @@ config LOG_BUFFER_SHIFT default 14 help This option sets the size of the kernel log buffer as a power of two. The actual size is calculated as 2 raised to the power of this value (i.e., 2^LOG_BUFFER_SHIFT bytes). + + +menu "Memory Management" +menu "Kernel Allocator" + +config SLAB_POISON + bool "Poison slab objects on allocation" + default y + +config SLAB_POISON_CHECK + bool "Verify poison state on allocation" + default y + + +endmenu + +endmenu + endmenu diff --git a/src/mm/amd64/impl.zig b/src/mm/amd64/impl.zig index ee2fe03..11a5ea2 100644 --- a/src/mm/amd64/impl.zig +++ b/src/mm/amd64/impl.zig @@ -13,7 +13,9 @@ pub fn virt_to_phys(vaddr: b.VAddr) usize { return vaddr - 0xffff800000000000; } -pub const pfndb_base = 0xffffea0000000000; +pub const hhdm_base = 0xffff800000000000; +pub const kernel_heap_base = 0xffffe00000000000; +pub const pfndb_base = 0xffffff0000000000; pub const PagingImpl = struct { const Self = @This(); diff --git a/src/mm/heap.zig b/src/mm/heap.zig new file mode 100644 index 0000000..98ecf73 --- /dev/null +++ b/src/mm/heap.zig @@ -0,0 +1,26 @@ +const b = @import("base"); +const ke = b.ke; +const mm = b.mm; +const mi = mm.private; + +var heap_lock: ke.SpinLock = .init(); +var heap_arena: mm.vmem.Arena = undefined; + +pub fn init() void { + heap_arena.init("kernel heap", mi.impl.kernel_heap_base, b.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena"); +} + +pub fn alloc_pages(size: usize) mm.Error!*anyopaque { + const ipl = heap_lock.acquire(); + defer heap_lock.release(ipl); + + const addr = try heap_arena.alloc(size, .{}); + + mi.kernel_pmap.map_range_allocating(addr, size, .{ + .read = true, + .write = true, + .global = true, + }); + + return @ptrFromInt(addr); +} diff --git a/src/mm/init.zig b/src/mm/init.zig index 32bc71b..cb05b0d 100644 --- a/src/mm/init.zig +++ b/src/mm/init.zig @@ -78,9 +78,14 @@ fn map_kernel(boot_info: *pl.BootInfo) void { } } +const std = @import("std"); + pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { mi.phys.init(boot_info); map_kernel(boot_info); mi.kernel_pmap.activate(); mi.phys.init_pfndb(); + mi.zone.early_init(); + mi.vmem.init(); + mi.heap.init(); } diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 346b27c..8f720d9 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -207,7 +207,7 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { for (0..boot_info.memory_map.entry_count) |i| { const entry = boot_info.memory_map.entries[i]; - std.log.info("mm/phys: [{x:0>16} - {x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) }); + std.log.info("mm/phys: [{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) }); if (entry.type == .Free) { total_usable_memory += entry.size; diff --git a/src/mm/private.zig b/src/mm/private.zig index 27ffa8a..7c90960 100644 --- a/src/mm/private.zig +++ b/src/mm/private.zig @@ -3,6 +3,9 @@ pub const phys = @import("phys.zig"); pub const radix_pmap = @import("radix_pmap.zig"); pub const pmap = @import("pmap.zig"); pub const init = @import("init.zig"); +pub const vmem = @import("vmem.zig"); +pub const heap = @import("heap.zig"); +pub const zone = @import("zone.zig"); pub const PMap = pmap.PMap; pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64")) diff --git a/src/mm/root.zig b/src/mm/root.zig index 821a377..c84caae 100644 --- a/src/mm/root.zig +++ b/src/mm/root.zig @@ -6,6 +6,7 @@ const std = @import("std"); pub const page_size = 4096; pub const init = private.init.init; +pub const vmem = private.vmem; // 16 TB of maximum physical memory. // This should be fine on consumer hardware for at least a decade :^) @@ -28,6 +29,8 @@ pub const MapFlags = packed struct { cache_disable: bool = false, }; +pub const Error = error{OutOfMemory, InvalidAddress, InvalidSize}; + pub inline fn page_to_pfn(addr: usize) Pfn { return @intCast(addr >> 12); } diff --git a/src/mm/vmem.zig b/src/mm/vmem.zig new file mode 100644 index 0000000..a93d1c5 --- /dev/null +++ b/src/mm/vmem.zig @@ -0,0 +1,469 @@ +//! Generic resource allocator, based on the vmem allocator described in: +//! Jeff Bonwick and Jonathan Adams' +//! "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources" +//! +//! ## Overview +//! +//! A vmem arena manages a contiguous range of an arbitrary resource (virtual address +//! space, PIDs, etc.) by tracking free and allocated regions as a list of segments. +//! Each segment has a base address and a size. +//! +//! ## Data structures +//! +//! An arena is composed of three data structures: +//! +//! 1. Segment list: an address-ordered doubly-linked list of all segments, both +//! free and allocated. Used for coalescing on free (adjacent free segments are +//! merged) and for NextFit traversal. +//! +//! 2. Power-of-two free lists: an array of 64 lists, where bucket n holds free +//! segments whose size falls in [2^n, 2^(n+1)). +//! +//! 3. Allocated segment RB-tree: a tree of allocated segments keyed by base address. +//! Used for O(log n) lookup on free() given only an address. +//! +//! ## Allocation +//! +//! Three policies are supported: +//! +//! - `InstantFit`: find the first segment in the appropriate freelist bucket. O(1) for +//! unconstrained allocations; degrades under alignment or address constraints. +//! +//! - `BestFit`: scan freelist buckets for the smallest segment that satisfies the +//! request. Minimizes fragmentation at the cost of a fuller scan. +//! +//! - `NextFit`: continue searching from where the last allocation left off, wrapping +//! around at the end of the arena. Avoids reusing recently-freed addresses, making +//! it useful for resources like PIDs where cycling through values is desirable. +//! +//! ## Differences from the original paper +//! +//! This implementation omits several features from the original Solaris vmem: +//! - No importing: arenas are initialized with a fixed set of spans and cannot +//! grow by importing from a parent arena. This simplifies the implementation +//! significantly and is sufficient for fixed virtual address space management. +//! +//! - No span markers: without importing there is no need to track which spans +//! were imported, so span segment types are omitted entirely. +//! +//! - No quantum caching: the original vmem uses per-size slab caches (qcaches) +//! to accelerate small allocations. This is unnecessary here because vmem is +//! only called by the slab allocator for large slab-sized chunks, not for +//! individual small objects. This could change in the future if we find +//! more uses for the allocator. +//! +//! - No hash table: allocated segments are tracked in an RB-tree rather than +//! the dynamic hash table used by Solaris vmem, avoiding the resize +//! complexity. +const std = @import("std"); +const rtl = @import("rtl"); +const mm = @import("base").mm; +const mi = mm.private; + +pub const Policy = enum { + /// Use the smallest free segment that can satisfy the request. + /// This tends to minimize fragmentation. + BestFit, + + /// Provide a good approximation of the best fit policy in guaranteed + /// O(1) time. This is the default policy. + InstantFit, + + /// Use the next free segment after the most recently allocated segment. + NextFit, +}; + +pub const AllocOptions = struct { + policy: Policy = .InstantFit, + alignment: ?usize = null, + min: ?usize = null, + max: ?usize = null, +}; + +const Segment = struct { + const Type = enum { + Allocated, + Free, + }; + + const Linkage = union { + /// Linkage into a freelist, valid when type == .Free + free_link: rtl.List.Entry, + /// Linkage into the allocated segments tree, valid when type == .Allocated + tree_link: rtl.bst.Node, + }; + /// The type of the segment. + type: Type, + /// The base address of the segment. + base: usize, + /// The size of the segment in bytes. + size: usize, + /// Linkage into the arena's segment list + link: rtl.List.Entry, + /// Type-specific linkage. + linkage: Linkage, + + fn from_free_link(entry: *rtl.List.Entry) *Segment { + const linkage: *Segment.Linkage = @fieldParentPtr("free_link", entry); + return @fieldParentPtr("linkage", linkage); + } + + fn from_tree_link(node: *rtl.bst.Node) *Segment { + const linkage: *Segment.Linkage = @fieldParentPtr("tree_link", node); + return @fieldParentPtr("linkage", linkage); + } + + fn cmp(a_node: *rtl.bst.Node, b_node: *rtl.bst.Node) std.math.Order { + const a: *Segment = Segment.from_tree_link(a_node); + const b: *Segment = Segment.from_tree_link(b_node); + return std.math.order(a.base, b.base); + } +}; + +/// One freelist for each power of two size that can fit within the host's +/// address space. +const freelist_count = @bitSizeOf(usize); + +var seg_zone: mi.zone.TypedZone(Segment) = undefined; + +pub const Arena = struct { + /// Name of the arena, for debugging purposes. + name: []const u8, + /// Start of initial span. + base: usize, + /// Size of initial span. + size: usize, + /// Unit of currency. + quantum: usize, + /// List of segments, sorted by base address. + list: rtl.List, + /// Power of two freelists for free segments, indexed by log2(size). + freelists: [freelist_count]rtl.List, + /// Tree for allocated segments, indexed by base address. + allocated_segments: rtl.RBTree(Segment.cmp), + /// Last segment allocated from, for NextFit policy. + rotor: ?*Segment, + + const Self = @This(); + + /// Initialize the area for use. + pub fn init(self: *Self, name: []const u8, base: usize, size: usize, quantum: usize) !void { + self.name = name; + self.base = base; + self.size = size; + self.quantum = quantum; + + rtl.List.init(&self.list); + + for (&self.freelists) |*freelist| { + freelist.init(); + } + + self.allocated_segments = .init(); + self.rotor = null; + + // Add initial span + try self.add(base, size); + } + + pub fn deinit(self: *Self) void { + // free all segments in the segment list + var entry = self.list.first(); + while (entry != &self.list.head) { + const next = entry.next; + const seg: *Segment = @fieldParentPtr("link", entry); + free_segment(seg); + entry = next; + } + } + + /// Add a span of memory to the arena. + pub fn add(self: *Self, addr: usize, size: usize) !void { + const new_seg = try self.alloc_segment(); + new_seg.* = Segment{ + .type = .Free, + .base = addr, + .size = size, + .link = undefined, + .linkage = .{ .free_link = undefined }, + }; + + // Insert into segment list, sorted by base address. + // Start from the end since we're likely to be adding higher addresses. + var entry = self.list.last(); + while (entry != &self.list.head) { + const s: *Segment = @fieldParentPtr("link", entry); + if (s.base < addr) break; + entry = entry.prev; + } + + new_seg.link.insert_before(entry.next); + self.add_segment_to_freelist(new_seg); + } + + /// Allocate a segment of memory from the arena. + /// Options: + /// - policy: allocation policy to use. Default is .InstantFit. + /// - alignment: if specified, the returned segment will be aligned to this boundary. + /// - min: if specified, the returned segment will be at least on this address. + /// - max: if specified, the returned segment will be at most on this address. + /// Returns the base address of the allocated segment. + pub fn alloc( + self: *Self, + size: usize, + options: AllocOptions, + ) mm.Error!usize { + if (size % self.quantum != 0) { + return error.InvalidSize; + } + + const result = switch (options.policy) { + .InstantFit => self.instant_fit(size, options), + .BestFit => self.best_fit(size, options), + .NextFit => self.next_fit(size, options), + }; + + const start, var seg = result orelse return error.OutOfMemory; + + std.debug.assert(seg.type == .Free); + std.debug.assert(seg.size >= size); + + // Remove the segment from the freelist. + self.remove_segment_from_freelist(seg); + + // left split: alignment pushed start forward, so there's a gap between + // seg.base and start that needs to become a free segment. + // e.g. seg=[0x0, 0x10000], start=0x100: + // before: [0x0, 0x10000] free + // after: [0x0, 0x100] free, [0x100, 0x10000] being processed + if (seg.base != start) { + const left_seg = try self.alloc_segment(); + left_seg.* = Segment{ + .type = .Free, + .base = seg.base, + .size = start - seg.base, + .link = undefined, + .linkage = .{ .free_link = undefined }, + }; + + seg.base = start; + seg.size -= left_seg.size; + + left_seg.link.insert_before(&seg.link); + self.add_segment_to_freelist(left_seg); + } + + // right split: seg is larger than needed, so split into allocated + free remainder. + // e.g. seg=[0x100, 0x10000], size=0x1000: + // before: [0x100, 0x10000] being processed + // after: [0x100, 0x1100] allocated, [0x1100, 0x10000] free + // the quantum check avoids creating a remainder too small to ever be useful. + if (seg.size != size and (seg.size - size) > self.quantum - 1) { + const new_seg = try self.alloc_segment(); + new_seg.* = .{ .type = .Allocated, .base = seg.base, .size = size, .link = undefined, .linkage = .{ .tree_link = undefined } }; + + seg.base += size; + seg.size -= size; + seg.type = .Free; + seg.linkage = .{ .free_link = undefined }; + + new_seg.link.insert_before(&seg.link); + self.add_segment_to_freelist(seg); + + self.allocated_segments.insert(&new_seg.linkage.tree_link) catch return error.InvalidAddress; + + if (options.policy == .NextFit) self.rotor = new_seg; + return new_seg.base; + } else { + // seg is exactly the right size (or remainder is too small), use it directly. + seg.type = .Allocated; + seg.linkage = .{ .tree_link = undefined }; + self.allocated_segments.insert(&seg.linkage.tree_link) catch return error.InvalidAddress; + + if (options.policy == .NextFit) self.rotor = seg; + return seg.base; + } + } + + pub fn free(self: *Self, addr: usize, size: usize) mm.Error!void { + // Find the allocated segment containing addr. + var search_node = Segment{ + .type = .Allocated, + .base = addr, + .size = 0, + .link = undefined, + .linkage = .{ .tree_link = undefined }, + }; + + const node = self.allocated_segments.tree.search(&search_node.linkage.tree_link) orelse return error.InvalidAddress; + + const seg = Segment.from_tree_link(node); + + if (seg.size != size) return error.InvalidAddress; + + // Remove from allocated tree. + self.allocated_segments.delete(node); + + // Coalesce to the right. + const next_entry = seg.link.next; + if (next_entry != &self.list.head) { + const next_seg: *Segment = @fieldParentPtr("link", next_entry); + + if (self.rotor == next_seg) self.rotor = seg; + + if (next_seg.type == .Free) { + // Remove next segment from freelist and segment list, then merge into seg. + self.remove_segment_from_freelist(next_seg); + next_seg.link.remove(); + seg.size += next_seg.size; + self.free_segment(next_seg); + } + } + + // Coalesce to the left. + const prev_entry = seg.link.prev; + if (prev_entry != &self.list.head) { + const prev_seg: *Segment = @fieldParentPtr("link", prev_entry); + + if (self.rotor == prev_seg) self.rotor = seg; + + if (prev_seg.type == .Free) { + // Remove previous segment from freelist and segment list, then merge into seg. + self.remove_segment_from_freelist(prev_seg); + prev_seg.link.remove(); + seg.base = prev_seg.base; + seg.size += prev_seg.size; + self.free_segment(prev_seg); + } + } + + // Mark as free and add to freelist. + seg.type = .Free; + seg.linkage = .{ .free_link = undefined }; + self.add_segment_to_freelist(seg); + } + + // Implementation for the instant fit policy. + fn instant_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { + var idx = freelist_index(size); + + if (!std.math.isPowerOfTwo(size)) idx += 1; + + // Simply grab the first segment from the appropriate freelist + // that can satisfy the request. + // This is O(1) unless there are contraints. + while (idx < freelist_count) : (idx += 1) { + var it = self.freelists[idx].iterator(); + while (it.next()) : (it.advance()) { + const seg = Segment.from_free_link(it.get()); + if (self.try_to_fit(seg, size, options)) |addr| return .{ addr, seg }; + } + } + return null; + } + + // Implementation for the best fit policy. + // Find the smallest segment that can satisfy the request. + fn best_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { + var idx = freelist_index(size); + var best_seg: ?*Segment = null; + var best_start: usize = 0; + + while (idx < freelist_count) : (idx += 1) { + var it = self.freelists[idx].iterator(); + + while (it.next()) : (it.advance()) { + const seg = Segment.from_free_link(it.get()); + if (self.try_to_fit(seg, size, options)) |addr| { + if (best_seg == null or seg.size < best_seg.?.size) { + best_seg = seg; + best_start = addr; + } + } + } + + if (best_seg != null) break; + } + return if (best_seg) |s| .{ best_start, s } else null; + } + + // Implementation for the next fit policy. + // Continue searching from the last allocated segment, wrapping around at the end of the arena. + fn next_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { + // Start from rotor if we have one, otherwise from the beginning. + const start_entry = if (self.rotor) |r| r.link.next else self.list.first(); + + // Search from rotor to end of list. + var entry = start_entry; + while (entry != &self.list.head) : (entry = entry.next) { + const seg: *Segment = @fieldParentPtr("link", entry); + if (seg.type != .Free) continue; + if (self.try_to_fit(seg, size, options)) |addr| { + return .{ addr, seg }; + } + } + + // Wrap around and search from beginning to rotor. + entry = self.list.first(); + while (entry != start_entry) : (entry = entry.next) { + const seg: *Segment = @fieldParentPtr("link", entry); + if (seg.type != .Free) continue; + if (self.try_to_fit(seg, size, options)) |addr| { + return .{ addr, seg }; + } + } + + return null; + } + + // Try to fit the given segment to the request with the given options. + // Returns the fitted address if successful. + fn try_to_fit(self: *Self, segment: *Segment, size: usize, options: AllocOptions) ?usize { + var start = @max(segment.base, options.min orelse 0); + const end = @min(segment.base + segment.size, options.max orelse std.math.maxInt(usize)); + const alignment = options.alignment orelse self.quantum; + + if (start > end) return null; + + // Align start to the required alignment. + start = std.mem.alignForward(usize, start, alignment); + + if (start + size <= end) { + return start; + } + + return null; + } + + fn alloc_segment(self: *Self) !*Segment { + _ = self; + return seg_zone.create(); + } + + fn free_segment(self: *Self, segment: *Segment) void { + seg_zone.destroy(segment); + _ = self; + } + + fn add_segment_to_freelist(self: *Self, segment: *Segment) void { + const freelist = self.freelist_for_size(segment.size); + freelist.insert_tail(&segment.linkage.free_link); + } + + fn remove_segment_from_freelist(self: *Self, segment: *Segment) void { + _ = self; + segment.linkage.free_link.remove(); + } + + inline fn freelist_index(size: usize) usize { + return std.math.log2_int(usize, size); + } + + inline fn freelist_for_size(self: *Self, size: usize) *rtl.List { + return &self.freelists[freelist_index(size)]; + } +}; + +pub fn init() void { + seg_zone.init("seg", .{}); +} diff --git a/src/mm/zone.zig b/src/mm/zone.zig new file mode 100644 index 0000000..d18a489 --- /dev/null +++ b/src/mm/zone.zig @@ -0,0 +1,724 @@ +//! Slab allocator implementation with per-CPU object caching. +//! As described in Bonwick's "The Slab Allocator: An Object-Caching Kernel Memory Allocator" +//! and "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources". +const rtl = @import("rtl"); +const b = @import("base"); +const config = @import("config"); +const std = @import("std"); +const ke = b.ke; +const mm = b.mm; +const mi = mm.private; + +// TODO: when SMP is setup, do proper magazine initialization. + +/// A bufctl is a control structure for a single object in a slab. +/// It is used to track free objects and link them together in the slab's free list. +const BufCtl = struct { + /// Next BufCtl in the list. + next: ?*BufCtl, + /// Pointer to the object this BufCtl manages. + buffer: *anyopaque, + /// Pointer to the owning Slab. + slab: *Slab, +}; + +/// A slab is a contiguous memory region that holds multiple objects of the same type. +const Slab = struct { + /// Linkage into a zone's list of slabs. + link: rtl.List.Entry, + /// Reference count indicating how many objects are in use. + refcount: usize, + /// Pointer to the first free BufCtl in this slab. + buflist: ?*BufCtl, + /// Base address of the slab. + base: usize, +}; + +/// A magazine is a per-CPU cache of objects for a zone. +const Magazine = struct { + /// Next magazine in the list. + next: ?*Magazine, + /// One or more rounds. + rounds: [*]*anyopaque, +}; + +const MagazineType = struct { + rounds: usize, + alignment: usize, + minbuf: usize, + maxbuf: usize, + /// Magazine zone. + zone: *Zone, +}; + +/// Per-CPU state for a zone. +const Cpu = struct { + lock: ke.SpinLock, + /// Magazine used for allocations. + alloc: ?*Magazine, + /// Magazine used for frees. + free: ?*Magazine, + /// Number of rounds in alloc magazine. + alloc_rounds: usize, + /// Number of rounds in free magazine. + free_rounds: usize, + /// Number of rounds in a full magazine. + magazine_size: usize, +}; + +const initial_hash = 32; + +const slab_align = 8; + +/// Global count of zones. +const zones_num = 32; + +/// Every power-of-two-size from 8 to 2048 (inclusively). +const generic_zones_num = 9; + +const generic_zone_names = [_][]const u8{ + "zone-8", + "zone-16", + "zone-32", + "zone-64", + "zone-128", + "zone-256", + "zone-512", + "zone-1024", + "zone-2048", +}; + +/// 1/8th of a page. +const small_slab_size = 512; + +/// This is an arbitrary number, ideally this should be determined on zone +/// creation based on the object size and slab size, +/// but for simplicity we just use a constant here. +const objects_per_slab = 16; + +/// Zones used to allocate out-of-line bufctls for large slabs. +var bufctl_zone: TypedZone(BufCtl) = undefined; +var slab_zone: TypedZone(Slab) = undefined; + +/// Generic zones for power-of-two sizes from 8 to 2048. +var generic_zones: [generic_zones_num]Zone = undefined; + +var magazines_enabled = false; + +const magtypes = [_]MagazineType{ + .{ .rounds = 1, .alignment = 8, .minbuf = 3200, .maxbuf = 65536, .zone = undefined }, + .{ .rounds = 3, .alignment = 16, .minbuf = 256, .maxbuf = 32768, .zone = undefined }, + .{ .rounds = 7, .alignment = 32, .minbuf = 64, .maxbuf = 16384, .zone = undefined }, + .{ .rounds = 15, .alignment = 64, .minbuf = 0, .maxbuf = 8192, .zone = undefined }, + .{ .rounds = 31, .alignment = 64, .minbuf = 0, .maxbuf = 4096, .zone = undefined }, + .{ .rounds = 47, .alignment = 64, .minbuf = 0, .maxbuf = 2048, .zone = undefined }, + .{ .rounds = 63, .alignment = 64, .minbuf = 0, .maxbuf = 1024, .zone = undefined }, + .{ .rounds = 95, .alignment = 64, .minbuf = 0, .maxbuf = 512, .zone = undefined }, + .{ .rounds = 143, .alignment = 64, .minbuf = 0, .maxbuf = 0, .zone = undefined }, +}; + +const alloc_poison: u32 = 0xBADDC0DE; +const free_poison: u32 = 0xDEADBEEF; +const is_poison_enabled = @hasDecl(config, "CONFIG_SLAB_POISON"); +const should_check_poison = @hasDecl(config, "CONFIG_SLAB_POISON_CHECK"); + +inline fn fill_with_poison(ptr: *anyopaque, len: usize, pattern: u32) void { + const bytes: [*]u8 = @ptrCast(ptr); + var i: usize = 0; + + while (i + 4 <= len) : (i += 4) { + std.mem.writeInt(u32, bytes[i..][0..4], pattern, .little); + } + + if (i < len) { + var tail: [4]u8 = undefined; + std.mem.writeInt(u32, &tail, pattern, .little); + var j: usize = 0; + while (i < len) : ({ + i += 1; + j += 1; + }) bytes[i] = tail[j]; + } +} + +inline fn check_poison(ptr: *anyopaque, len: usize, pattern: u32) bool { + if (!should_check_poison) return true; + + const bytes: [*]const u8 = @ptrCast(ptr); + var i: usize = 0; + + while (i + 4 <= len) : (i += 4) { + if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern) return false; + } + + if (i < len) { + var tail: [4]u8 = undefined; + std.mem.writeInt(u32, &tail, pattern, .little); + var j: usize = 0; + while (i < len) : ({ + i += 1; + j += 1; + }) { + if (bytes[i] != tail[j]) return false; + } + } + + return true; +} + +pub const Zone = struct { + /// Name used for debugging purposes. + name: []const u8, + /// Size of objects in this zone. + obj_size: usize, + /// Alignment of objects in this zone. + alignment: usize, + /// Size of a slab. + slab_size: usize, + /// Size of a chunk. + chunk_size: usize, + /// Offset for buf-to-bufctl conversion. + offset: usize, + /// Max color for slab coloring. + max_color: usize, + /// Current color for slab coloring. + color: usize, + /// Zone lock. + lock: ke.SpinLock, + /// List of full slabs in this zone. + full_slabs: rtl.List, + /// List of partial slabs in this zone. + partial_slabs: rtl.List, + /// Linkage into the global static zone free list. + next: ?*Zone, + /// Buffer-to-bufctl hash map. + bufmap: [*]?*BufCtl, + + hash_mask: usize, + hash_shift: usize, + + depot_lock: ke.SpinLock, + + empty_mags: ?*Magazine, + full_mags: ?*Magazine, + magtype: ?MagazineType, + /// Per-CPU state. + cpus: [*]Cpu, + + ctor: ?*const fn (obj: *anyopaque) void, + dtor: ?*const fn (obj: *anyopaque) void, + + hash0: [initial_hash]?*BufCtl, + + const Self = @This(); + + pub const InitOptions = struct { + alignment: usize = 0, + ctor: ?*const fn (*anyopaque) void = null, + dtor: ?*const fn (*anyopaque) void = null, + }; + + /// Initialize a zone. + pub fn init(self: *Self, name: []const u8, size: usize, options: InitOptions) void { + const obj_align = if (options.alignment == 0) slab_align else options.alignment; + var chunk_size = std.mem.alignForward(usize, size, obj_align); + + const offset, chunk_size = if (size <= small_slab_size) blk: { + if (chunk_size - size >= @sizeOf(usize)) { + // Use padding at end of chunk. + break :blk .{ chunk_size - @sizeOf(usize), chunk_size }; + } else { + // No space, extend the chunk. + break :blk .{ chunk_size, chunk_size + slab_align }; + } + } else blk: { + // Out-of-line bufctl + break :blk .{ 0, chunk_size }; + }; + + const slab_size, const max_color = if (size <= small_slab_size) blk: { + const ss = mm.page_size; + break :blk .{ ss, @rem(ss - @sizeOf(Slab), chunk_size) }; + } else blk: { + const ss = calc_slab_size(chunk_size); + break :blk .{ ss, @rem(ss, chunk_size) }; + }; + + self.full_slabs.init(); + self.partial_slabs.init(); + + self.name = name; + self.obj_size = size; + self.alignment = obj_align; + self.slab_size = slab_size; + self.chunk_size = chunk_size; + self.max_color = max_color; + self.color = 0; + self.offset = offset; + self.ctor = options.ctor; + self.dtor = options.dtor; + self.hash_mask = initial_hash - 1; + self.hash_shift = std.math.log2_int_ceil(usize, chunk_size); + self.lock = .init(); + self.bufmap = &self.hash0; + + self.magtype = null; + for (&magtypes) |*mtype| { + if (mtype.maxbuf >= chunk_size) { + self.magtype = mtype.*; + break; + } + } + } + + /// Allocate an object from the zone. + pub fn alloc(self: *Self) mm.Error!*anyopaque { + const ipl = ke.ipl.raise(.Dispatch); + defer ke.ipl.lower(ipl); + var buf: *anyopaque = undefined; + + // Try grabbing an object from the magazine layer. + if (magazines_enabled) { + const cpu = &self.cpus[ke.cpu.current()]; + + cpu.lock.acquire_no_ipl(); + defer cpu.lock.release_no_ipl(); + + while (true) { + if (cpu.alloc_rounds > 0) { + // Fast path: just pop from the alloc magazine. + cpu.alloc_rounds -= 1; + buf = cpu.alloc.?.rounds[cpu.alloc_rounds]; + + if (is_poison_enabled) { + if (!check_poison(buf, self.obj_size, free_poison)) { + @panic("slab poison mismatch: object corrupted after free"); + } + + fill_with_poison(buf, self.obj_size, alloc_poison); + + if (self.ctor) |ctor| { + ctor(buf); + } + } + + return buf; + } + + if (cpu.free_rounds > 0) { + std.debug.assert(cpu.free != null); + + // Alloc magazine is empty. If the free magazine has items, swap them. + // The free magazine now becomes our alloc magazine. + std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free); + std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds); + continue; + } + + // Both magazines are empty. + // Try to get a full magazine from the depot. + if (self.alloc_from_depot(&self.full_mags)) |full_mag| { + // Discard our empty alloc magazine to the depot. + if (cpu.alloc) |empty_mag| { + self.free_to_depot(empty_mag, &self.empty_mags); + } + + cpu.alloc = full_mag; + cpu.alloc_rounds = cpu.magazine_size; + continue; + } + + // Depot has no full magazines, fall back to the slab layer.. + break; + } + } + + // Fall back to the slab layer. + self.lock.acquire_no_ipl(); + defer self.lock.release_no_ipl(); + + var slab: *Slab = undefined; + + if (self.partial_slabs.is_empty()) { + slab = try self.slab_create(); + self.partial_slabs.insert_tail(&slab.link); + } else { + slab = @fieldParentPtr("link", self.partial_slabs.first()); + } + + var bufctl = slab.buflist orelse return error.OutOfMemory; + + if (bufctl.next == null) { + slab.link.remove(); + self.full_slabs.insert_tail(&slab.link); + } + + slab.buflist = bufctl.next; + slab.refcount += 1; + + if (self.obj_size > small_slab_size) { + const bucket = self.bufctl_from_obj(bufctl.buffer); + + buf = bufctl.buffer; + bufctl.next = bucket.*; + bucket.* = bufctl; + } else { + buf = @ptrFromInt(@intFromPtr(bufctl) - self.offset); + } + + if (is_poison_enabled) { + if (!check_poison(buf, self.obj_size, free_poison)) { + @panic("slab poison mismatch: object corrupted after free"); + } + + fill_with_poison(buf, self.obj_size, alloc_poison); + + if (self.ctor) |ctor| { + ctor(buf); + } + } + + return buf; + } + + /// Free an object back to the zone. + pub fn free(self: *Self, obj: *anyopaque) void { + const ipl = ke.ipl.raise(.Dispatch); + defer ke.ipl.lower(ipl); + + if (magazines_enabled) { + const cpu = &self.cpus[ke.cpu.current()]; + + cpu.lock.acquire_no_ipl(); + defer cpu.lock.release_no_ipl(); + + while (true) { + if (cpu.free_rounds < cpu.magazine_size) { + // Fast path: push directly to free magazine. + if (is_poison_enabled) { + if (self.dtor) |dtor| { + dtor(obj); + } + + fill_with_poison(obj, self.obj_size, free_poison); + } + + cpu.free.?.rounds[cpu.free_rounds] = obj; + cpu.free_rounds += 1; + return; + } + + if (cpu.alloc_rounds == 0 and cpu.alloc != null) { + // Free magazine is full. If the alloc magazine is completely empty, swap them. + // The empty alloc magazine becomes our new free magazine. + std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free); + std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds); + continue; + } + + // Free magazine is full, and alloc magazine has items. + // Try to get an empty magazine from the depot. + if (self.alloc_from_depot(&self.empty_mags)) |empty_mag| { + // Put the Free magazine on the full list. + if (cpu.free) |full_mag| { + self.free_to_depot(full_mag, &self.full_mags); + } + + cpu.free = empty_mag; + cpu.free_rounds = 0; + continue; + } + + // No empty magazine in the depot, try to allocate a new one. + cpu.lock.release_no_ipl(); + + const new_mag: ?*Magazine = @ptrCast(@alignCast(self.magtype.?.zone.alloc() catch null)); + + cpu.lock.acquire_no_ipl(); + + if (new_mag) |m| { + self.free_to_depot(m, &self.empty_mags); + cpu.free = m; + cpu.free_rounds = 0; + continue; + } + + // Failed to allocate a magazine. Fall back to the slab layer. + break; + } + } + + if (is_poison_enabled) { + if (self.dtor) |dtor| { + dtor(obj); + } + fill_with_poison(obj, self.obj_size, free_poison); + } + + self.lock.acquire_no_ipl(); + defer self.lock.release_no_ipl(); + + // Find the bufctl for the buffer. + var bufctl: ?*BufCtl = null; + var slab: ?*Slab = null; + + if (self.obj_size > small_slab_size) { + // Search through the hash map and remove it. + var prev = self.bufctl_from_obj(@ptrCast(obj)); + + while (prev.* != null) { + const bc = prev.*; + + if (@intFromPtr(bc.?.buffer) == @intFromPtr(obj)) { + prev.* = bc.?.next; + slab = bc.?.slab; + bufctl = bc.?; + break; + } + + prev = &bc.?.next; + } + } else { + bufctl = @ptrFromInt(@intFromPtr(obj) + self.offset); + slab = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size) + mm.page_size - @sizeOf(Slab)); + } + + if (slab == null or bufctl == null) { + return; + } + + const s = slab.?; + const bufc = bufctl.?; + + if (s.buflist == null) { + // There were no buffers in the slab, so it wasn't in the freelist. + // Now that there is a buffer, add it back to the freelist. + s.link.remove(); + self.partial_slabs.insert_head(&s.link); + } + + // Insert bufctl into the slab freelist. + bufc.next = s.buflist; + s.buflist = bufc; + + s.refcount -= 1; + + if (s.refcount == 0) { + // No more outstanding allocations, it is safe to reclaim the slab. + s.link.remove(); + self.slab_destroy(s); + return; + } + } + + inline fn bufctl_from_obj(self: *Self, buffer: *anyopaque) *?*BufCtl { + return &self.bufmap[(@intFromPtr(buffer) >> @intCast(self.hash_shift)) & self.hash_mask]; + } + + fn alloc_page() !*anyopaque { + const alloc_ret = mi.phys.alloc(); + + return @ptrFromInt(mm.p2v(alloc_ret)); + } + + fn free_page(p: *anyopaque) void { + const phys = mm.v2p(@intFromPtr(p)); + mi.phys.free(phys); + } + + fn slab_create_small(self: *Self) mm.Error!*Slab { + var buf = try alloc_page(); + + // We employ a simple coloring scheme; every time a slab is created, + // the color is shifted by the alignment. This is done until we reach the + // max color (which is when there is no space left in the slab buffer). + // This ensures uniform buffer address distribution. + self.color += self.alignment; + + if (self.color > self.max_color) { + self.color = 0; + } + + // Slab info is located at the end of the page. + var slab: *Slab = @ptrFromInt(@intFromPtr(buf) + mm.page_size - @sizeOf(Slab)); + + const capacity = (mm.page_size - @sizeOf(Slab) - self.color) / self.chunk_size; + + slab.buflist = null; + slab.refcount = 0; + slab.base = @intFromPtr(buf); + + buf = @ptrFromInt(@intFromPtr(buf) + self.color); + + for (0..capacity) |i| { + var bufctl: *BufCtl = @ptrFromInt(@intFromPtr(buf) + (i * self.chunk_size) + self.offset); + const obj: *anyopaque = @ptrFromInt(@intFromPtr(buf) + (i * self.chunk_size)); + + // Add bufctl to the freelist + // NOTE: a bufctl field other than next must not be modified. + bufctl.next = slab.buflist; + slab.buflist = bufctl; + + if (is_poison_enabled) { + fill_with_poison(obj, self.obj_size, free_poison); + } else { + if (self.ctor) |ctor| ctor(obj); + } + } + + return slab; + } + + fn slab_create_large(self: *Self) !*Slab { + self.color += self.alignment; + + if (self.color > self.max_color) { + self.color = 0; + } + + const capacity = (self.slab_size - self.color) / self.chunk_size; + const buf = try mi.heap.alloc_pages(self.slab_size); + + var ret: *Slab = try slab_zone.create(); + + ret.buflist = null; + ret.base = @intFromPtr(buf); + + for (0..capacity) |i| { + var bufctl: *BufCtl = try bufctl_zone.create(); + + bufctl.buffer = @ptrFromInt(@intFromPtr(buf) + self.color + i * self.chunk_size); + bufctl.slab = ret; + + bufctl.next = ret.buflist; + ret.buflist = bufctl; + + if (is_poison_enabled) { + fill_with_poison(bufctl.buffer, self.obj_size, free_poison); + } else { + if (self.ctor) |ctor| ctor(bufctl.buffer); + } + } + + return ret; + } + + fn slab_create(self: *Self) mm.Error!*Slab { + if (self.obj_size <= small_slab_size) { + return self.slab_create_small(); + } else { + return self.slab_create_large(); + } + } + + fn slab_destroy(self: *Self, slab: *Slab) void { + if (!is_poison_enabled) { + if (self.dtor) |dtor| { + var bufctl = slab.buflist; + while (bufctl) |bc| : (bufctl = bc.next) { + const obj = if (self.obj_size > small_slab_size) + bc.buffer + else + @as(*anyopaque, @ptrFromInt(@intFromPtr(bc) - self.offset)); + dtor(obj); + } + } + } + + if (self.obj_size > small_slab_size) { + var bufctl = slab.buflist; + + while (bufctl != null) { + const next = bufctl.?.next; + bufctl_zone.destroy(bufctl.?); + bufctl = next; + } + + slab_zone.destroy(slab); + + // FIXME: call free_pages on slab.base here! + } else { + free_page(@ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size))); + } + } + + fn maglist_alloc(list_head: *?*Magazine) ?*Magazine { + const ret = list_head.* orelse return null; + list_head.* = ret.next; + + return ret; + } + + fn maglist_free(list_head: *?*Magazine, mag: *Magazine) void { + mag.next = list_head.*; + list_head.* = mag; + } + + fn alloc_from_depot(self: *Self, maglist: *?*Magazine) ?*Magazine { + self.depot_lock.acquire_no_ipl(); + const ret = maglist_alloc(maglist); + self.depot_lock.release_no_ipl(); + return ret; + } + + fn free_to_depot(self: *Self, mag: *Magazine, maglist: *?*Magazine) void { + self.depot_lock.acquire_no_ipl(); + maglist_free(maglist, mag); + self.depot_lock.release_no_ipl(); + } + + fn calc_slab_size(chunk_size: usize) usize { + // Try increasing page multiples until waste is under 12.5%. + var order: u5 = 0; + while (order <= 8) : (order += 1) { + const slab_size = @as(usize, mm.page_size) << order; + const n_objs = slab_size / chunk_size; + if (n_objs == 0) continue; + const waste = slab_size - (n_objs * chunk_size); + // Accept if waste < 1/8th of slab. + if (waste * 8 <= slab_size) return slab_size; + } + // Fallback: just fit one object, rounded up to page boundary + return std.mem.alignForward(usize, chunk_size, mm.page_size); + } +}; + +/// Parameterized version of Zone, useful for object caches. +pub fn TypedZone(comptime T: type) type { + return struct { + zone: Zone, + + const Self = @This(); + + pub const InitOptions = struct { + ctor: ?*const fn (*T) void = null, + dtor: ?*const fn (*T) void = null, + }; + + pub fn init(self: *@This(), name: []const u8, options: InitOptions) void { + self.zone.init(name, @sizeOf(T), .{ + .alignment = @alignOf(T), + .ctor = @ptrCast(options.ctor), + .dtor = @ptrCast(options.dtor), + }); + } + + pub fn create(self: *Self) mm.Error!*T { + const ret = try self.zone.alloc(); + return @ptrCast(@alignCast(ret)); + } + + pub fn destroy(self: *Self, obj: *T) void { + self.zone.free(obj); + } + }; +} + +pub fn early_init() linksection(b.init) void { + bufctl_zone.init("bufctl", .{}); + slab_zone.init("slab", .{}); + + for (0..generic_zones.len, &generic_zones) |i, *zone| { + zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{}); + } +}