Zag/src/rtl/barrier.zig

//! Atomic barrier implementations on various architectures.
//! This is needed since Zig removed @fence.

const builtin = @import("builtin");

const std = @import("std");
const assert = std.debug.assert;

pub inline fn fence(comptime ordering: std.builtin.AtomicOrder) void {
    switch (ordering) {
        .seq_cst, .acq_rel => mb(),
        .acquire => acq(),
        .release => rel(),
        else => @compileError("Unsupported ordering"),
    }
}

// Full memory barrier
inline fn mb() void {
    switch (builtin.cpu.arch) {
        .x86_64 => {
            // Faster than mfence
            asm volatile ("lock addl $0, (%%rsp)" ::: .{ .memory = true });
        },
        .x86 => {
            asm volatile ("lock addl $0, (%%esp)" ::: .{ .memory = true });
        },
        .aarch64, .arm => {
            asm volatile ("dmb ish" ::: .{ .memory = true });
        },
        .riscv64, .riscv32 => {
            asm volatile ("fence rw,rw" ::: .{ .memory = true });
        },
        else => @compileError("unsupported architecture"),
    }
}

// Acquire barrier
inline fn acq() void {
    switch (builtin.cpu.arch) {
        .x86, .x86_64 => {
            asm volatile ("" ::: .{ .memory = true });
        },
        .aarch64, .arm => {
            asm volatile ("dmb ishld" ::: .{ .memory = true });
        },
        .riscv64, .riscv32 => {
            asm volatile ("fence r,rw" ::: .{ .memory = true });
        },
        else => @compileError("unsupported architecture"),
    }
}

// Release barrier
inline fn rel() void {
    switch (builtin.cpu.arch) {
        .x86, .x86_64 => {
            asm volatile ("" ::: .{ .memory = true });
        },
        .aarch64, .arm => {
            asm volatile ("dmb ish" ::: .{ .memory = true });
        },
        .riscv64, .riscv32 => {
            asm volatile ("fence rw,w" ::: .{ .memory = true });
        },
        else => @compileError("unsupported architecture"),
    }
}

/// Copy to dest from src with atomic loads.
/// Both addresses need to be aligned.
pub fn atomic_load_memcpy(
    dest: anytype,
    src: anytype,
    comptime ordering: std.builtin.AtomicOrder,
) void {
    const T = @TypeOf(dest.*);
    const Word = switch (@alignOf(T)) {
        1 => u8,
        2 => u16,
        4 => u32,
        else => u64,
    };

    const word_size = @sizeOf(Word);
    const n = @sizeOf(T);
    comptime assert(n % word_size == 0);

    const src_words: [*]const volatile Word = @ptrCast(src);
    const dest_words: [*]Word = @ptrCast(dest);

    var i: usize = 0;
    while (i < n / word_size) : (i += 1) {
        dest_words[i] = @atomicLoad(Word, &src_words[i], ordering);
    }
}

/// Copy to dest from src with atomic stores.
/// Both addresses need to be aligned.
pub fn atomic_store_memcpy(
    dest: anytype,
    src: anytype,
    comptime ordering: std.builtin.AtomicOrder,
) void {
    const T = @TypeOf(dest.*);
    const Word = switch (@alignOf(T)) {
        1 => u8,
        2 => u16,
        4 => u32,
        else => u64,
    };

    const word_size = @sizeOf(Word);
    const n = @sizeOf(T);
    comptime assert(n % word_size == 0);

    const src_words: [*]const volatile Word = @ptrCast(src);
    const dest_words: [*]Word = @ptrCast(dest);

    var i: usize = 0;
    while (i < n / word_size) : (i += 1) {
        @atomicStore(Word, &dest_words[i], src_words[i], ordering);
    }
}