diff --git a/src/rtl/barrier.zig b/src/rtl/barrier.zig index 573a6db..74b5247 100644 --- a/src/rtl/barrier.zig +++ b/src/rtl/barrier.zig @@ -1,6 +1,7 @@ //! Atomic barrier implementations on various architectures. //! This is needed since Zig removed @fence. +const std = @import("std"); const builtin = @import("builtin"); /// Read Memory Barrier (Load-Load) @@ -30,11 +31,14 @@ pub inline fn wmb() void { .x86, .x86_64 => { asm volatile ("" ::: .{ .memory = true }); }, - .aarch64, .arm => { + .aarch64 => { + asm volatile ("dmb ishst" ::: .{ .memory = true }); + }, + .arm => { asm volatile ("dmb ish" ::: .{ .memory = true }); }, .riscv64, .riscv32 => { - asm volatile ("fence rw,w" ::: .{ .memory = true }); + asm volatile ("fence w,w" ::: .{ .memory = true }); }, else => @compileError("unsupported architecture"), } @@ -55,3 +59,51 @@ pub inline fn mb() void { else => @compileError("unsupported architecture"), } } + +/// Copy to dest from src with atomic loads. +/// Both addresses need to be aligned. +pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.builtin.AtomicOrder) void { + const T = @TypeOf(dest.*); + const Word = switch (@alignOf(T)) { + 1 => u8, + 2 => u16, + 4 => u32, + else => u64, + }; + + const word_size = @sizeOf(Word); + const n = @sizeOf(T); + comptime std.debug.assert(n % word_size == 0); + + const src_words: [*]const volatile Word = @ptrCast(src); + const dest_words: [*]Word = @ptrCast(dest); + + var i: usize = 0; + while (i < n / word_size) : (i += 1) { + dest_words[i] = @atomicLoad(Word, &src_words[i], ordering); + } +} + +/// Copy to dest from src with atomic stores. +/// Both addresses need to be aligned. +pub fn atomic_store_memcpy(dest: anytype, src: anytype, comptime ordering: std.builtin.AtomicOrder) void { + const T = @TypeOf(dest.*); + const Word = switch (@alignOf(T)) { + 1 => u8, + 2 => u16, + 4 => u32, + else => u64, + }; + + const word_size = @sizeOf(Word); + const n = @sizeOf(T); + comptime std.debug.assert(n % word_size == 0); + + const src_words: [*]const volatile Word = @ptrCast(src); + const dest_words: [*]Word = @ptrCast(dest); + + var i: usize = 0; + while (i < n / word_size) : (i += 1) { + @atomicStore(Word, &dest_words[i], src_words[i], ordering); + } +} diff --git a/src/rtl/seqlock.zig b/src/rtl/seqlock.zig index d3afc61..c9cc65b 100644 --- a/src/rtl/seqlock.zig +++ b/src/rtl/seqlock.zig @@ -22,7 +22,8 @@ pub fn SeqLock(comptime T: type) type { continue; } - ret = self.raw; + // Relaxed load memcpy is needed here since data races are UB. + rtl.barrier.atomic_load_memcpy(&ret, &self.raw, .monotonic); // Ensure the sequence read happens *after* the data is fully loaded. rtl.barrier.rmb(); @@ -44,7 +45,8 @@ pub fn SeqLock(comptime T: type) type { // Ensure the data write happens *after* the sequence is incremented. rtl.fence.wmb(); - self.raw = data; + // Relaxed store memcpy is needed here since data races are UB. + rtl.barrier.atomic_store_memcpy(&self.raw, &data, .monotonic); self.sequence.store(seq + 2, .release); }