Zag/src/kern/dpc.zig

//! Deferred procedure calls (DPC).

const config = @import("config");
const rtl = @import("rtl");
const r = @import("root");
const ke = r.ke;
const kep = ke.private;

const std = @import("std");
const assert = std.debug.assert;

/// Deferred Procedure Call (DPC) structure.
/// Used for scheduling work to be done when IPL is lowered below `.Dispatch`.
pub const Dpc = struct {
    /// Entry into per-CPU DPC queue
    link: rtl.List.Entry,
    /// Routine to call
    func: *const fn (*Dpc, ?*anyopaque) void,
    /// Argument passed to the routine
    arg: ?*anyopaque,
    /// Whether or not the DPC is currently inserted
    inserted: std.atomic.Value(bool),

    /// Initialize a DPC for `func`.
    pub fn init(func: *const fn (*Dpc, ?*anyopaque) void) Dpc {
        return .{
            .link = undefined,
            .func = func,
            .arg = undefined,
            .inserted = std.atomic.Value(bool).init(false),
        };
    }
};

const PerCpu = struct {
    /// Queue of DPCs on this CPU.
    queue: rtl.List,
    /// Lock over this CPU's DPC queue.
    lock: ke.SpinLock,
};

const pcpu = ke.CpuLocal(PerCpu, .{
    .lock = undefined,
    .queue = undefined,
});

fn pcpu_init() linksection(r.init) void {
    pcpu.local().lock = ke.SpinLock.init("dpc");
    pcpu.local().queue.init();
}

comptime {
    _ = r.percpu_init_set.insert(&pcpu_init);
}

/// Enqueue a DPC.
/// If the DPC is already enqueued, this function is a no-op.
pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
    const ipl = ke.ipl.raise(.High);
    const mycpu = ke.cpu.current();
    const dpc_cpu = pcpu.local();

    if (dpc.inserted.cmpxchgStrong(false, true, .acquire, .monotonic) == null) {
        dpc.arg = arg;

        // Insert the DPC on this CPU's DPC queue
        dpc_cpu.lock.acquire_no_ipl();
        dpc_cpu.queue.insert_tail(&dpc.link);

        // Mark the DPC as pending on this CPU
        kep.ipl.set_softint_pending(mycpu, .Dispatch);

        dpc_cpu.lock.release_no_ipl();
    }

    ke.ipl.lower(ipl);
}

fn dispatch_queue(cpu: u32) void {
    const ipl_cpu = kep.ipl.cpu_ipl.local();
    const dpc_cpu = pcpu.local();
    const sched_cpu = kep.sched.percpu.local();

    ipl_cpu.* = .Dispatch;

    // Mark as handled, this must be done before enabling interrupts or we will
    // race.
    kep.ipl.clear_softint_pending(cpu, .Dispatch);

    _ = kep.impl.enable_interrupts();

    while (true) {
        const ipl = dpc_cpu.lock.acquire_at(.High);
        var dpc: *Dpc = undefined;

        if (dpc_cpu.queue.is_empty()) {
            // No more DPCs to process.
            dpc_cpu.lock.release(ipl);
            break;
        }

        // Pop the head
        var first_elem = dpc_cpu.queue.first();
        first_elem.remove();

        dpc = @fieldParentPtr("link", first_elem);
        dpc.inserted.store(false, .release);

        // An interrupt which would enqueue this DPC could occur between
        // loading the argument and calling the routine, which is why we capture
        // it here to ensure that we get the intended context.
        const arg = dpc.arg;

        dpc_cpu.lock.release(ipl);
        assert(ke.ipl.current() == .Dispatch);

        dpc.func(dpc, arg);
    }

    if (sched_cpu.start_timer) {
        ke.timer.set(
            &sched_cpu.resched_timer,
            .from(r.Milliseconds.init(config.sched_timeslice)),
            .{ .dpc = &sched_cpu.resched_dpc },
        );
    }

    if (sched_cpu.preemption_reason == .HigherPriority) {
        // Reload the quantum for the new thread.
        ke.timer.cancel(&sched_cpu.resched_timer);
        ke.timer.set(
            &sched_cpu.resched_timer,
            .from(r.Milliseconds.init(config.sched_timeslice)),
            .{ .dpc = &sched_cpu.resched_dpc },
        );
    }

    sched_cpu.start_timer = false;
    sched_cpu.preemption_reason = .None;

    kep.shootdown.process_shootdowns();

    if (sched_cpu.next_thread != null) {
        const curtd = sched_cpu.current_thread orelse unreachable;

        if (!curtd.smr_sections.is_empty()) {
            kep.smr.mark_thread_stalled(curtd);
        }

        kep.sched.handle_preemption(sched_cpu);
    }

    _ = kep.impl.disable_interrupts();
}

/// Dispatch the DPC queue on the current CPU.
pub fn dispatch() void {
    // DPC processing is done at IPL Dispatch.
    // Don't call lower/raise here because they might call us again recursively.
    const int_state = kep.impl.disable_interrupts();
    const old_ipl = kep.ipl.cpu_ipl.local().*;
    var mycpu = ke.cpu.current();

    while (kep.ipl.is_softint_pending(.Dispatch)) {
        dispatch_queue(mycpu);
        // Our CPU might have changed.
        mycpu = ke.cpu.current();
    }

    kep.ipl.cpu_ipl.local().* = old_ipl;
    kep.impl.restore_interrupts(int_state);
}