Zag/src/kern/amd64/impl.zig

//! amd64-specific kernel code.

const r = @import("root");
const ke = r.ke;
const kep = ke.private;
const pl = r.pl;
const amd64 = r.arch;

const std = @import("std");

pub const tlb_max_pages = 33;

pub const init = @import("init.zig");
pub const early_init = init.early_init;

pub export var cpu_self_offset: usize linksection(r.percpu) = 0;
pub export var cpu_offsets: [*]usize = undefined;

const ThreadFrame = extern struct {
    rbp: u64 align(1),
    rbx: u64 align(1),
    r12: u64 align(1),
    r13: u64 align(1),
    r14: u64 align(1),
    r15: u64 align(1),
    rip: u64 align(1),
};

const Context = ke.Thread.Context;

extern fn asm_thread_entry() void;

extern fn amd64_context_load(td: *ThreadContext) callconv(.c) void;

extern fn amd64_context_switch(
    old: *Context,
    new: *Context,
    lock: *u8,
    switching: *bool,
) callconv(.c) void;

extern fn amd64_context_switch_cont_n(
    old: *Context,
    new: *Context,
    lock: *u8,
    switching: *bool,
) callconv(.c) void;

extern fn amd64_call_continuation(
    td: *ke.Thread,
    ctx: *Context,
    func: usize,
    arg: ?*anyopaque,
) callconv(.c) noreturn;

export fn ki_free_old_stack(stack_top: usize) callconv(.c) void {
    kep.thread.stack_cache_free(stack_top);
}

/// Trampoline to match the zig ABI.
export fn ki_continuation_trampoline(
    func: usize,
    arg: ?*anyopaque,
) noreturn {
    ke.ipl.lower(.Passive);
    const f: *const fn (?*anyopaque) void = @ptrFromInt(func);
    f(arg);
    @panic("Continuation returned");
}

pub const ThreadContext = extern struct {
    rsp: u64 align(1) = 0xDEADC0D3,

    export fn thread_entry(entry: usize, arg: usize) callconv(.c) void {
        const entry_fn: *const fn (?*anyopaque) void = @ptrFromInt(entry);
        ke.ipl.lower(.Passive);
        entry_fn(@ptrFromInt(arg));
    }

    pub fn init_with_stack(
        stack: r.VAddr,
    ) @This() {
        return .{
            .rsp = stack,
        };
    }

    pub fn reset(
        self: *@This(),
        stack_top: r.VAddr,
        entry: *const fn (?*anyopaque) void,
        arg: ?*anyopaque,
    ) @This() {
        var sp: usize = stack_top;

        sp = sp & ~@as(usize, 15);
        sp = sp - @sizeOf(ThreadFrame);
        sp = (sp & ~@as(usize, 15)) - 8;

        const frame: *ThreadFrame = @ptrFromInt(sp);

        frame.* = std.mem.zeroes(ThreadFrame);

        self.rsp = @intFromPtr(frame);
        frame.rip = @intFromPtr(&asm_thread_entry);
        frame.r12 = @intFromPtr(entry);
        frame.r13 = @intFromPtr(arg);

        return self.*;
    }

    pub fn load(self: *ThreadContext) void {
        amd64_context_load(self);
    }
};

pub fn switch_normal_to_normal(
    old: *Context,
    new: *Context,
) void {
    const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old));
    amd64_context_switch(
        old,
        new,
        &thread.lock.inner.locked.raw,
        &thread.switching.raw,
    );
}

pub fn switch_cont_to_normal(
    old: *Context,
    new: *Context,
) void {
    const td: *ke.Thread = @alignCast(@fieldParentPtr("context", old));

    amd64_context_switch_cont_n(
        old,
        new,
        &td.lock.inner.locked.raw,
        &td.switching.raw,
    );
}

pub fn call_continuation(
    ctx: *Context,
    continuation: ke.Continuation,
) noreturn {
    const td: *ke.Thread = @alignCast(@fieldParentPtr("context", ctx));

    amd64_call_continuation(
        td,
        ctx,
        @intFromPtr(continuation.func),
        continuation.arg,
    );
}

inline fn percpu_ptr_for(variable: anytype, cpu: u32) @TypeOf(variable) {
    return @ptrFromInt(@intFromPtr(variable) +% cpu_offsets[cpu]);
}

pub inline fn percpu_ptr_other(variable: anytype, id: u32) @TypeOf(variable) {
    return percpu_ptr_for(variable, id);
}

pub inline fn halt() void {
    asm volatile ("hlt");
}

pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) {
    const offset = asm volatile ("mov %%gs:(%[self]), %[out]"
        : [out] "=r" (-> u64),
        : [self] "r" (&cpu_self_offset),
    );

    return @ptrFromInt(offset +% @intFromPtr(variable));
}

pub inline fn set_hardware_ipl(ipl: ke.Ipl) void {
    const cr8_value: u64 = switch (ipl) {
        .Passive, .Dispatch => 0,
        .Device => 13,
        .High => 15,
    };
    asm volatile ("mov %[ipl], %%cr8"
        :
        : [ipl] "r" (cr8_value),
    );
}

pub inline fn enable_interrupts() void {
    amd64.sti();
}

pub inline fn disable_interrupts() bool {
    const ie = amd64.rflags().interrupt_enable;

    amd64.cli();

    return ie;
}

pub inline fn restore_interrupts(state: bool) void {
    if (state) {
        enable_interrupts();
    } else {
        _ = disable_interrupts();
    }
}

pub inline fn flush_full_tlb() void {
    const prev_cr3 = amd64.read_cr(3);
    // Reload CR3 to flush the entire TLB.
    // This works because we assume global pages are never unmapped.
    // If they are in the future, this should toggle CR4.PGE instead.
    amd64.write_cr(3, prev_cr3);
}

pub inline fn flush_tlb(va: r.VAddr) void {
    amd64.invlpg(va);
}

// Inline asm so the compiler has a chance to inline it.
pub inline fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
    return asm volatile (
        \\movb %%gs:cpu_ipl, %%cl
        \\movb %%cl, (%[old_ptr])
        \\movb %[new_val], %%gs:cpu_ipl
        \\cmpb %%cl, %[new_val]
        \\setl %[result]
        : [result] "=r" (-> bool),
        : [new_val] "r" (new),
          [old_ptr] "r" (old),
        : .{ .rcx = true, .memory = true });
}

pub inline fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
    return asm volatile (
        \\movb %%gs:cpu_ipl, %%cl
        \\movb %%cl, (%[old_ptr])
        \\movb %[new_val], %%gs:cpu_ipl
        \\cmpb %[new_val], %%cl
        \\setl %[result]
        : [result] "=r" (-> bool),
        : [new_val] "r" (new),
          [old_ptr] "r" (old),
        : .{ .rcx = true, .memory = true });
}