initial commit

src/rtl/barrier.zig

@@ -0,0 +1,57 @@
+//! Atomic barrier implementations on various architectures.
+//! This is needed since Zig removed @fence.
+
+const builtin = @import("builtin");
+
+/// Read Memory Barrier (Load-Load)
+/// Ensures all loads before the barrier complete before any loads after it.
+pub inline fn rmb() void {
+    switch (builtin.cpu.arch) {
+        .x86, .x86_64 => {
+            asm volatile ("" ::: .{ .memory = true });
+        },
+        .aarch64 => {
+            asm volatile ("dmb ishld" ::: .{ .memory = true });
+        },
+        .arm => {
+            asm volatile ("dmb ish" ::: .{ .memory = true });
+        },
+        .riscv64, .riscv32 => {
+            asm volatile ("fence r,r" ::: .{ .memory = true });
+        },
+        else => @compileError("unsupported architecture"),
+    }
+}
+
+/// Write Memory Barrier (Store-Store)
+/// Ensures all stores before the barrier complete before any stores after it.
+pub inline fn wmb() void {
+    switch (builtin.cpu.arch) {
+        .x86, .x86_64 => {
+            asm volatile ("" ::: .{ .memory = true });
+        },
+        .aarch64, .arm => {
+            asm volatile ("dmb ish" ::: .{ .memory = true });
+        },
+        .riscv64, .riscv32 => {
+            asm volatile ("fence rw,w" ::: .{ .memory = true });
+        },
+        else => @compileError("unsupported architecture"),
+    }
+}
+
+/// Full Memory Barrier (Load/Store - Load/Store)
+pub inline fn mb() void {
+    switch (builtin.cpu.arch) {
+        .x86, .x86_64 => {
+            asm volatile ("mfence" ::: .{ .memory = true });
+        },
+        .aarch64, .arm => {
+            asm volatile ("dmb ish" ::: .{ .memory = true });
+        },
+        .riscv64, .riscv32 => {
+            asm volatile ("fence rw,rw" ::: .{ .memory = true });
+        },
+        else => @compileError("unsupported architecture"),
+    }
+}

src/rtl/list.zig

@@ -0,0 +1,204 @@
+//! Intrusive circular linked list module.
+/// Branchless intrusive circular linked list.
+/// Embed `List.Entry` in your struct and use `@fieldParentPtr` to recover it.
+pub const List = struct {
+    head: Entry,
+
+    /// A list entry
+    pub const Entry = struct {
+        /// Next element in the list
+        next: *Entry = undefined,
+        /// Previous element
+        prev: *Entry = undefined,
+
+        /// Removes an entry from whatever list it's in
+        pub fn remove(elem: *Entry) void {
+            elem.prev.next = elem.next;
+            elem.next.prev = elem.prev;
+        }
+
+        /// Inserts elem before `before`
+        pub fn insert_before(elem: *Entry, before: *Entry) void {
+            elem.prev = before.prev;
+            elem.next = before;
+            before.prev.next = elem;
+            before.prev = elem;
+        }
+    };
+
+    /// Initializes the list
+    pub fn init(self: *List) void {
+        self.head.next = &self.head;
+        self.head.prev = &self.head;
+    }
+
+    /// Returns whether the list is empty
+    pub fn is_empty(self: *List) bool {
+        return self.head.next == &self.head;
+    }
+
+    /// Inserts elem at the end of the list
+    pub fn insert_tail(self: *List, elem: *Entry) void {
+        self.head.prev.next = elem;
+        elem.next = &self.head;
+        elem.prev = self.head.prev;
+        self.head.prev = elem;
+    }
+
+    /// Inserts elem at the beginning of the list
+    pub fn insert_head(self: *List, elem: *Entry) void {
+        self.head.next.prev = elem;
+        elem.next = self.head.next;
+        elem.prev = &self.head;
+        self.head.next = elem;
+    }
+
+    /// Returns the first element of the list
+    pub fn first(self: *List) *Entry {
+        return self.head.next;
+    }
+
+    /// Returns the last element of the list
+    pub fn last(self: *List) *Entry {
+        return self.head.prev;
+    }
+
+    pub fn iterator(self: *List) Iterator {
+        return .{ .head = &self.head, .current = self.head.next };
+    }
+
+    pub const Iterator = struct {
+        head: *Entry,
+        current: *Entry,
+
+        pub fn next(self: *Iterator) bool {
+            return self.current != self.head;
+        }
+
+        pub fn get(self: *Iterator) *Entry {
+            return self.current;
+        }
+
+        pub fn advance(self: *Iterator) void {
+            self.current = self.current.next;
+        }
+    };
+};
+
+const std = @import("std");
+
+test "init creates empty list" {
+    var list: List = undefined;
+    list.init();
+    try std.testing.expect(list.is_empty());
+}
+
+test "insert_tail adds elements in order" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+    var c = Node{ .value = 3 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+    list.insert_tail(&c.link);
+
+    var it = list.iterator();
+    var i: u32 = 1;
+    while (it.next()) : (it.advance()) {
+        const node: *Node = @fieldParentPtr("link", it.get());
+        try std.testing.expectEqual(i, node.value);
+        i += 1;
+    }
+    try std.testing.expectEqual(@as(u32, 4), i);
+}
+
+test "insert_head adds elements in reverse order" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+    var c = Node{ .value = 3 };
+
+    list.insert_head(&a.link);
+    list.insert_head(&b.link);
+    list.insert_head(&c.link);
+
+    var it = list.iterator();
+    var i: u32 = 3;
+    while (it.next()) : (it.advance()) {
+        const node: *Node = @fieldParentPtr("link", it.get());
+        try std.testing.expectEqual(i, node.value);
+        i -= 1;
+    }
+}
+
+test "remove detaches element" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+    var c = Node{ .value = 3 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+    list.insert_tail(&c.link);
+
+    b.link.remove();
+
+    var it = list.iterator();
+    const first: *Node = @fieldParentPtr("link", it.get());
+    try std.testing.expectEqual(@as(u32, 1), first.value);
+    it.advance();
+    const second: *Node = @fieldParentPtr("link", it.get());
+    try std.testing.expectEqual(@as(u32, 3), second.value);
+    it.advance();
+    try std.testing.expect(!it.next());
+}
+
+test "insert_before inserts in correct position" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 3 };
+    var mid = Node{ .value = 2 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+
+    mid.link.insert_before(&b.link);
+
+    var it = list.iterator();
+    var i: u32 = 1;
+    while (it.next()) : (it.advance()) {
+        const node: *Node = @fieldParentPtr("link", it.get());
+        try std.testing.expectEqual(i, node.value);
+        i += 1;
+    }
+}
+
+test "first and last return correct elements" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+
+    const first: *Node = @fieldParentPtr("link", list.first());
+    const last: *Node = @fieldParentPtr("link", list.last());
+    try std.testing.expectEqual(@as(u32, 1), first.value);
+    try std.testing.expectEqual(@as(u32, 2), last.value);
+}

src/rtl/pairing_heap.zig

@@ -0,0 +1,331 @@
+//! Intrusive Pairing Heap implementation.
+const std = @import("std");
+
+pub const Node = struct {
+    child: ?*Node = null,
+    next: ?*Node = null,
+    prev: ?*Node = null,
+};
+
+pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *Node) std.math.Order) type {
+    return struct {
+        const Self = @This();
+
+        /// Number of elements in the heap
+        size: usize = 0,
+
+        /// Topmost element of the heap
+        root: ?*Node = null,
+
+        fn meld(a: *Node, b: *Node) *Node {
+            const a_wins = switch (order) {
+                .min => cmp(a, b) != .gt, // a <= b
+                .max => cmp(a, b) == .gt, // a > b
+            };
+
+            if (a_wins) {
+                if (a.child) |c| {
+                    c.prev = b;
+                }
+
+                b.next = a.child;
+                a.child = b;
+                b.prev = a;
+                return a;
+            }
+
+            if (b.child) |c| {
+                c.prev = a;
+            }
+
+            a.prev = b;
+            a.next = b.child;
+            b.child = a;
+            return b;
+        }
+
+        fn merge_pairs(node: *Node) *Node {
+            // First pass: pair up siblings left to right
+            var list: ?*Node = node;
+            var pairs: ?*Node = null;
+
+            while (list) |a| {
+                const b = a.next orelse {
+                    a.next = pairs;
+                    a.prev = null;
+                    pairs = a;
+                    break;
+                };
+                list = b.next;
+
+                a.next = null;
+                a.prev = null;
+                b.next = null;
+                b.prev = null;
+
+                const merged = meld(a, b);
+                merged.next = pairs;
+                merged.prev = null;
+                pairs = merged;
+            }
+
+            // Second pass: fold pairs right to left
+            var result = pairs.?;
+            var rest = result.next;
+            result.next = null;
+            while (rest) |p| {
+                rest = p.next;
+                p.next = null;
+                result = meld(result, p);
+            }
+            return result;
+        }
+
+        pub fn init() Self {
+            return .{
+                .size = 0,
+                .root = null,
+            };
+        }
+
+        /// Insert `node` into the heap.
+        pub fn insert(self: *Self, node: *Node) void {
+            node.* = .{
+                .child = null,
+                .next = null,
+                .prev = null,
+            };
+
+            self.size += 1;
+
+            if (self.root == null) {
+                self.root = node;
+                return;
+            }
+
+            self.root = meld(self.root.?, node);
+        }
+
+        /// Return and remove the topmost element.
+        pub fn pop(self: *Self) ?*Node {
+            const root = self.root orelse return null;
+            const child = root.child;
+
+            root.child = null;
+            self.size -= 1;
+
+            if (child) |c| {
+                c.prev = null;
+                self.root = merge_pairs(c);
+            } else {
+                self.root = null;
+            }
+
+            return root;
+        }
+
+        /// Remove `node` from the heap.
+        pub fn remove(self: *Self, node: *Node) void {
+            if (self.root == node) {
+                _ = self.pop();
+                return;
+            }
+
+            std.debug.assert(self.root != null);
+
+            self.size -= 1;
+
+            // Unlink the node
+            if (node.next) |n| {
+                n.prev = node.prev;
+            }
+
+            if (node.prev) |p| {
+                if (p.child == node) {
+                    // node was leftmost child
+                    p.child = node.next;
+                } else {
+                    // node was a sibling
+                    p.next = node.next;
+                }
+            }
+
+            node.prev = null;
+            node.next = null;
+
+            // Merge node's children back into the heap
+            if (node.child) |c| {
+                c.prev = null;
+                self.root = meld(self.root.?, merge_pairs(c));
+            }
+
+            node.child = null;
+        }
+    };
+}
+const TestNode = struct {
+    value: u32,
+    node: Node = .{},
+};
+
+fn my_cmp(a: *Node, b: *Node) std.math.Order {
+    const ta: *TestNode = @fieldParentPtr("node", a);
+    const tb: *TestNode = @fieldParentPtr("node", b);
+    return std.math.order(ta.value, tb.value);
+}
+
+const MinHeap = PairingHeap(.min, my_cmp);
+const MaxHeap = PairingHeap(.max, my_cmp);
+
+test "empty heap" {
+    var heap = MinHeap.init();
+    try std.testing.expect(heap.root == null);
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "single element insert and pop" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 42 };
+
+    heap.insert(&a.node);
+    try std.testing.expectEqual(1, heap.size);
+
+    const top: *TestNode = @fieldParentPtr("node", heap.root.?);
+    try std.testing.expectEqual(42, top.value);
+
+    _ = heap.pop();
+    try std.testing.expect(heap.root == null);
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "min heap ordering" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 3 };
+    var b = TestNode{ .value = 1 };
+    var c = TestNode{ .value = 2 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+    try std.testing.expectEqual(3, heap.size);
+
+    const first: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(1, first.value);
+
+    const second: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(2, second.value);
+
+    const third: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(3, third.value);
+
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "max heap ordering" {
+    var heap = MaxHeap.init();
+    var a = TestNode{ .value = 3 };
+    var b = TestNode{ .value = 1 };
+    var c = TestNode{ .value = 2 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+
+    const first: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(3, first.value);
+
+    const second: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(2, second.value);
+
+    const third: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(1, third.value);
+}
+
+test "remove middle node" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+    var b = TestNode{ .value = 2 };
+    var c = TestNode{ .value = 3 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+
+    heap.remove(&b.node);
+    try std.testing.expectEqual(2, heap.size);
+
+    const first: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(1, first.value);
+
+    const second: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(3, second.value);
+
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "remove root" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+    var b = TestNode{ .value = 2 };
+    var c = TestNode{ .value = 3 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+
+    heap.remove(&a.node);
+    try std.testing.expectEqual(2, heap.size);
+
+    const top: *TestNode = @fieldParentPtr("node", heap.root.?);
+    try std.testing.expectEqual(2, top.value);
+}
+
+test "remove last element" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+
+    heap.insert(&a.node);
+    heap.remove(&a.node);
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.root == null);
+}
+
+test "duplicate values" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+    var b = TestNode{ .value = 1 };
+    var c = TestNode{ .value = 1 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+    try std.testing.expectEqual(3, heap.size);
+
+    _ = heap.pop();
+    _ = heap.pop();
+    _ = heap.pop();
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "large sequence" {
+    var heap = MinHeap.init();
+    var nodes: [16]TestNode = undefined;
+
+    for (&nodes, 0..) |*n, i| {
+        n.* = .{ .value = @intCast(16 - i) };
+        heap.insert(&n.node);
+    }
+
+    try std.testing.expectEqual(16, heap.size);
+
+    var prev: u32 = 0;
+    while (heap.pop()) |n| {
+        const t: *TestNode = @fieldParentPtr("node", n);
+        try std.testing.expect(t.value >= prev);
+        prev = t.value;
+    }
+    try std.testing.expectEqual(0, heap.size);
+}

src/rtl/root.zig

@@ -0,0 +1,11 @@
+const std = @import("std");
+
+pub const List = @import("list.zig").List;
+pub const SeqLock = @import("seqlock.zig").SeqLock;
+pub const barrier = @import("barrier.zig");
+pub const pairing_heap = @import("pairing_heap.zig");
+pub const PairingHeap = pairing_heap.PairingHeap;
+
+test {
+    std.testing.refAllDecls(@This());
+}

src/rtl/seqlock.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+const rtl = @import("rtl");
+
+pub fn SeqLock(comptime T: type) type {
+    return struct {
+        const Self = @This();
+
+        raw: T,
+        sequence: std.atomic.Value(usize),
+
+        pub fn init(val: T) Self {
+            return .{ .raw = val, .sequence = .init(0) };
+        }
+
+        pub fn load(self: *Self) T {
+            var ret: T = undefined;
+
+            while (true) {
+                const seq = self.sequence.load(.acquire);
+                if (seq & 1 != 0) {
+                    std.atomic.spinLoopHint();
+                    continue;
+                }
+
+                ret = self.raw;
+
+                // Ensure the sequence read happens *after* the data is fully loaded.
+                rtl.barrier.rmb();
+
+                if (self.sequence.load(.monotonic) == seq)
+                    break;
+
+                std.atomic.spinLoopHint();
+            }
+
+            return ret;
+        }
+
+        pub fn store(self: *Self, data: T) void {
+            const seq = self.sequence.load(.monotonic);
+
+            self.sequence.store(seq + 1, .monotonic);
+
+            // Ensure the data write happens *after* the sequence is incremented.
+            rtl.fence.wmb();
+
+            self.raw = data;
+
+            self.sequence.store(seq + 2, .release);
+        }
+    };
+}