mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-10-27 13:58:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

mm/zone: remove all spinlocks

Browse Source
This commit is contained in:
Loïc Grégoire
2026-10-24 22:12:56 -04:00
parent 09762e88ae
commit 0ffaa8df5b
1 changed files with 33 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sys/mm/zone.zig
+33 -36
View File
@@ -424,7 +424,7 @@ const Depot = struct {
/// Per-CPU state for a zone. /// Per-CPU state for a zone.
const Cpu = struct { const Cpu = struct {
lock: ke.SpinLock, lock: ke.Mutex,
/// Magazine used for allocations. /// Magazine used for allocations.
alloc: ?*Magazine, alloc: ?*Magazine,
/// Magazine used for frees. /// Magazine used for frees.
@@ -519,7 +519,7 @@ pub const Zone = struct {
free_slabs: rtl.List, free_slabs: rtl.List,
/// Linkage into the global zone list. /// Linkage into the global zone list.
next: ?*Zone, next: ?*Zone,
depot_lock: ke.QSpinLock, depot_lock: ke.Mutex,
depot: Depot, depot: Depot,
/// Current contention rate, in contentions/second scaled by `wma_unit`. /// Current contention rate, in contentions/second scaled by `wma_unit`.
depot_contention_wma: usize, depot_contention_wma: usize,
@@ -611,7 +611,7 @@ pub const Zone = struct {
self.ctor = options.ctor; self.ctor = options.ctor;
self.dtor = options.dtor; self.dtor = options.dtor;
self.lock = ke.Mutex.init("zone"); self.lock = ke.Mutex.init("zone");
self.depot_lock = ke.QSpinLock.init("zone_depot"); self.depot_lock = ke.Mutex.init("zone_depot");
self.use_magazines = options.magazines; self.use_magazines = options.magazines;
self.depot = std.mem.zeroes(Depot); self.depot = std.mem.zeroes(Depot);
@@ -659,7 +659,7 @@ pub const Zone = struct {
self.cpus = cpus_alloc(); self.cpus = cpus_alloc();
for (0..ke.ncpus) |i| { for (0..ke.ncpus) |i| {
self.cpus[i].value.lock = ke.SpinLock.init("Depot"); self.cpus[i].value.lock = ke.Mutex.init("Depot");
self.cpus[i].value.alloc = null; self.cpus[i].value.alloc = null;
self.cpus[i].value.free = null; self.cpus[i].value.free = null;
self.cpus[i].value.alloc_rounds = 0; self.cpus[i].value.alloc_rounds = 0;
@@ -675,9 +675,8 @@ pub const Zone = struct {
@branchHint(.likely); @branchHint(.likely);
var buf: *anyopaque = undefined; var buf: *anyopaque = undefined;
var ipl = ke.ipl.raise(.Dispatch);
var cpu = &self.cpus[ke.cpu.current()].value; var cpu = &self.cpus[ke.cpu.current()].value;
cpu.lock.acquire_no_ipl(); cpu.lock.acquire();
while (true) { while (true) {
if (cpu.alloc_rounds > 0) { if (cpu.alloc_rounds > 0) {
@@ -698,7 +697,7 @@ pub const Zone = struct {
} }
} }
cpu.lock.release(ipl); cpu.lock.release();
return buf; return buf;
} }
@@ -771,7 +770,7 @@ pub const Zone = struct {
// and insert it into the depot instead, // and insert it into the depot instead,
// since there is a higher chance the impostor magazine // since there is a higher chance the impostor magazine
// contains objects that are hotter in cache. // contains objects that are hotter in cache.
cpu.lock.release(ipl); cpu.lock.release();
const raw_mag = try magazine_zone.alloc( const raw_mag = try magazine_zone.alloc(
.{ .policy = opts.policy }, .{ .policy = opts.policy },
@@ -787,11 +786,10 @@ pub const Zone = struct {
opts.policy, opts.policy,
); );
// We dropped the lock and IPL, so we might've been // We dropped the lock, now is a good time to reload the CPU
// migrated. // in case we got migrated.
ipl = ke.ipl.raise(.Dispatch);
cpu = &self.cpus[ke.cpu.current()].value; cpu = &self.cpus[ke.cpu.current()].value;
cpu.lock.acquire_no_ipl(); cpu.lock.acquire();
if (cpu.alloc == null) { if (cpu.alloc == null) {
cpu.alloc = nm; cpu.alloc = nm;
@@ -822,7 +820,7 @@ pub const Zone = struct {
// many rounds a magazine has in the `Magazine` struct. // many rounds a magazine has in the `Magazine` struct.
// This should be fine for now though. // This should be fine for now though.
cpu.lock.release(ipl); cpu.lock.release();
self.release( self.release(
self.arg, self.arg,
@@ -857,10 +855,10 @@ pub const Zone = struct {
if (magazines_initialized and self.use_magazines) { if (magazines_initialized and self.use_magazines) {
@branchHint(.likely); @branchHint(.likely);
var ipl = ke.ipl.raise(.Dispatch);
var cpu = &self.cpus[ke.cpu.current()].value; var cpu = &self.cpus[ke.cpu.current()].value;
cpu.lock.acquire_no_ipl(); cpu.lock.acquire();
const freed = while (true) { const freed = while (true) {
if (cpu.free != null and if (cpu.free != null and
cpu.free_rounds < magazine_size.load()) cpu.free_rounds < magazine_size.load())
@@ -947,9 +945,9 @@ pub const Zone = struct {
} }
// No empty magazine in the depot, try to allocate a new one. // No empty magazine in the depot, try to allocate a new one.
// We need to drop IPL and the CPU lock here because we need to // Drop the CPU lock across the allocation so as to not make
// acquire the magazine's allocator's blocking lock. // hold times too lengthy.
cpu.lock.release(ipl); cpu.lock.release();
const raw_mag = magazine_zone.alloc( const raw_mag = magazine_zone.alloc(
.{ .policy = .DontWaitForMemory }, .{ .policy = .DontWaitForMemory },
@@ -958,9 +956,8 @@ pub const Zone = struct {
const new_mag: ?*Magazine = @ptrCast(@alignCast(raw_mag)); const new_mag: ?*Magazine = @ptrCast(@alignCast(raw_mag));
// Re-grab our current context. // Re-grab our current context.
ipl = ke.ipl.raise(.Dispatch);
cpu = &self.cpus[ke.cpu.current()].value; cpu = &self.cpus[ke.cpu.current()].value;
cpu.lock.acquire_no_ipl(); cpu.lock.acquire();
if (new_mag) |m| { if (new_mag) |m| {
m.seq = ke.smr.seq_invalid; m.seq = ke.smr.seq_invalid;
@@ -974,7 +971,7 @@ pub const Zone = struct {
break false; break false;
}; };
cpu.lock.release(ipl); cpu.lock.release();
if (freed) return; if (freed) return;
} }
@@ -1139,7 +1136,7 @@ pub const Zone = struct {
/// Periodic updates on the zone. /// Periodic updates on the zone.
/// Updates the magazine working set and trims or grows them if needed. /// Updates the magazine working set and trims or grows them if needed.
fn update(self: *Zone) void { fn update(self: *Zone) void {
const ipl = self.depot_lock.acquire(); self.depot_lock.acquire();
// Update the working set size. // Update the working set size.
// This tracks the minimum number of magazines required, // This tracks the minimum number of magazines required,
@@ -1205,7 +1202,7 @@ pub const Zone = struct {
self.trim_depot = false; self.trim_depot = false;
self.depot_lock.release(ipl); self.depot_lock.release();
if (should_trim) { if (should_trim) {
self.trim(trim_depot); self.trim(trim_depot);
@@ -1295,13 +1292,13 @@ pub const Zone = struct {
// Detach magazines before invoking release callbacks. // Detach magazines before invoking release callbacks.
self.lock.acquire(); self.lock.acquire();
const ipl = self.depot_lock.acquire(); self.depot_lock.acquire();
// Take all elements from the zone depot. // Take all elements from the zone depot.
var depot = self.depot; var depot = self.depot;
self.depot = std.mem.zeroes(Depot); self.depot = std.mem.zeroes(Depot);
self.depot_lock.release(ipl); self.depot_lock.release();
// Trim all CPU depots to 0. // Trim all CPU depots to 0.
for (0..ke.ncpus) |i| { for (0..ke.ncpus) |i| {
@@ -1332,7 +1329,7 @@ pub const Zone = struct {
/// Detach excess magazines for destruction. /// Detach excess magazines for destruction.
fn trim_maglist(self: *Zone, maglist: *MagazineList) ?*Magazine { fn trim_maglist(self: *Zone, maglist: *MagazineList) ?*Magazine {
const ipl = self.depot_lock.acquire(); self.depot_lock.acquire();
// Grab the magazines and WSS under the depot lock. // Grab the magazines and WSS under the depot lock.
const target = @min( const target = @min(
@@ -1366,14 +1363,14 @@ pub const Zone = struct {
maglist.num -= removed; maglist.num -= removed;
maglist.wma -= removed * wma_unit; maglist.wma -= removed * wma_unit;
self.depot_lock.release(ipl); self.depot_lock.release();
return removed_head; return removed_head;
} }
/// Trim a CPU's depot to `target`. /// Trim a CPU's depot to `target`.
fn trim_cpu(self: *Zone, cpu: *Cpu, depot: *Depot, target: usize) void { fn trim_cpu(self: *Zone, cpu: *Cpu, depot: *Depot, target: usize) void {
const ipl = cpu.lock.acquire(); cpu.lock.acquire();
// Split the target (cpu_depot_size) in 2 for each magazine type. // Split the target (cpu_depot_size) in 2 for each magazine type.
// Bias towards full magazines because they are usually more useful // Bias towards full magazines because they are usually more useful
@@ -1409,7 +1406,7 @@ pub const Zone = struct {
} }
} }
cpu.lock.release(ipl); cpu.lock.release();
} }
fn maglist_destroy(self: *Zone, head: ?*Magazine, n: usize) void { fn maglist_destroy(self: *Zone, head: ?*Magazine, n: usize) void {
@@ -1623,9 +1620,9 @@ pub const Zone = struct {
/// zone depot, and tries to get full magazines from the zone depot up to /// zone depot, and tries to get full magazines from the zone depot up to
/// `target`. CPU lock is held. /// `target`. CPU lock is held.
fn alloc_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void { fn alloc_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void {
if (!self.depot_lock.try_acquire_no_ipl()) { if (!self.depot_lock.try_acquire()) {
// Register the contention. // Register the contention.
self.depot_lock.acquire_no_ipl(); self.depot_lock.acquire();
self.depot_contention_cur += 1; self.depot_contention_cur += 1;
} }
@@ -1668,7 +1665,7 @@ pub const Zone = struct {
} }
} }
self.depot_lock.release_no_ipl(); self.depot_lock.release();
} }
/// Try to rebalance the zone and cpu depots w.r.t to each other /// Try to rebalance the zone and cpu depots w.r.t to each other
@@ -1676,9 +1673,9 @@ pub const Zone = struct {
/// zone depot, and tries to get empty magazines from the zone depot up to /// zone depot, and tries to get empty magazines from the zone depot up to
/// `target`. CPU lock is held. /// `target`. CPU lock is held.
fn free_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void { fn free_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void {
if (!self.depot_lock.try_acquire_no_ipl()) { if (!self.depot_lock.try_acquire()) {
// Register the contention. // Register the contention.
self.depot_lock.acquire_no_ipl(); self.depot_lock.acquire();
self.depot_contention_cur += 1; self.depot_contention_cur += 1;
} }
@@ -1728,7 +1725,7 @@ pub const Zone = struct {
); );
} }
self.depot_lock.release_no_ipl(); self.depot_lock.release();
} }
}; };
@@ -1823,7 +1820,7 @@ pub fn late_init() linksection(r.init) void {
for (0..ke.ncpus) |i| { for (0..ke.ncpus) |i| {
z.cpus[i] = rtl.CachePaddedType(Cpu).init(.{ z.cpus[i] = rtl.CachePaddedType(Cpu).init(.{
.lock = ke.SpinLock.init("Magazine"), .lock = ke.Mutex.init("Magazine"),
.alloc = null, .alloc = null,
.free = null, .free = null,
.alloc_rounds = 0, .alloc_rounds = 0,