riscv64: initial risc-v port

src/arch/riscv64/root.zig

@@ -0,0 +1,26 @@
+pub const name: []const u8 = "riscv64";
+pub const BootInfo = struct {};
+
+pub inline fn read_csr(comptime csr: []const u8) u64 {
+    return asm volatile ("csrr %[out], " ++ csr
+        : [out] "=r" (-> u64),
+    );
+}
+
+pub inline fn write_csr(comptime csr: []const u8, value: u64) void {
+    asm volatile ("csrw " ++ csr ++ ", %[value]"
+        :
+        : [value] "r" (value),
+    );
+}
+
+pub inline fn sfence_vma() void {
+    asm volatile ("sfence.vma" ::: .{ .memory = true });
+}
+
+pub inline fn sfence_vma_page(va: usize) void {
+    asm volatile ("sfence.vma %[va], zero"
+        :
+        : [va] "r" (va),
+        : .{ .memory = true });
+}

src/bootvid/bootvid.zig

@@ -144,6 +144,7 @@ pub fn disable() void {
 }
 
 pub fn crash() void {
+    if (!enabled) return;
     cursor.x = 0;
     cursor.y = 0;
 
@@ -172,9 +173,13 @@ pub fn write_char(c: u8) void {
 }
 
 pub fn init(boot_info: *r.BootInfo) void {
-    framebuffer = @ptrFromInt(boot_info.framebuffer.?.address);
-    framebuffer_width = boot_info.framebuffer.?.width;
-    framebuffer_height = boot_info.framebuffer.?.height;
+    const fb = boot_info.framebuffer orelse {
+        enabled = false;
+        return;
+    };
+    framebuffer = @ptrFromInt(fb.address);
+    framebuffer_width = fb.width;
+    framebuffer_height = fb.height;
 
     load_font();
 

src/kern/panic.zig

@@ -57,7 +57,7 @@ fn is_kernel_text(addr: usize) bool {
 }
 
 fn walk_stack_frame(base: usize) void {
-    var frame: ?*StackFrame = @ptrFromInt(base);
+    var frame: ?*StackFrame = @ptrFromInt(if (config.arch == .riscv64) base - 16 else base);
     var depth: usize = 0;
     while (frame) |f| : (depth += 1) {
         if (depth > 64) break;
@@ -67,7 +67,10 @@ fn walk_stack_frame(base: usize) void {
         }
         const sym = get_symbol_name(ret_addr) orelse Symbol{ .name = "???", .offset = 0 };
         std.log.err("  #{d} {s}+0x{x} - 0x{x}", .{ depth, sym.name, sym.offset, ret_addr });
-        frame = f.prev;
+        frame = if (config.arch == .riscv64 and f.prev != null)
+            @ptrFromInt(@intFromPtr(f.prev.?) - 16)
+        else
+            f.prev;
     }
 }
 

src/kern/private.zig

@@ -6,6 +6,7 @@ const ke = r.ke;
 
 pub const impl = switch (config.arch) {
     .amd64 => @import("amd64/impl.zig"),
+    .riscv64 => @import("riscv64/impl.zig"),
     else => @compileError("unsupported architecture"),
 };
 

src/kern/riscv64/impl.zig

@@ -0,0 +1,145 @@
+const r = @import("root");
+const ke = r.ke;
+const rv64 = r.arch;
+
+pub const tlb_max_pages = 32;
+
+pub const ThreadContext = extern struct {
+    sp: usize,
+    ra: usize = 0,
+    s: [12]usize = .{0} ** 12,
+
+    pub fn init_with_stack(stack: r.VAddr) @This() {
+        return .{ .sp = stack };
+    }
+
+    pub fn reset(self: *@This(), stack_top: r.VAddr, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() {
+        self.sp = stack_top & ~@as(usize, 15);
+        self.ra = @intFromPtr(&thread_start);
+        self.s = .{0} ** 12;
+        self.s[1] = @intFromPtr(entry);
+        self.s[2] = @intFromPtr(arg);
+        return self.*;
+    }
+
+    pub fn load(self: *@This()) noreturn {
+        riscv_context_load(self);
+    }
+};
+
+extern fn riscv_context_load(ctx: *ThreadContext) callconv(.c) noreturn;
+extern fn riscv_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void;
+extern fn riscv_context_switch_cont(old: *ke.Thread.Context, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void;
+extern fn thread_start() callconv(.c) noreturn;
+
+export fn riscv_thread_entry(entry_addr: usize, arg_addr: usize) callconv(.c) noreturn {
+    const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_addr);
+    entry(@ptrFromInt(arg_addr));
+    @panic("Thread entry returned");
+}
+
+export fn riscv_free_old_stack(stack: usize) callconv(.c) void {
+    ke.private.thread.stack_cache_free(stack);
+}
+
+export fn riscv_run_continuation(func_addr: usize, arg: ?*anyopaque) callconv(.c) noreturn {
+    ke.ipl.lower(.Passive);
+    const func: *const fn (?*anyopaque) void = @ptrFromInt(func_addr);
+    func(arg);
+    @panic("Continuation returned");
+}
+
+pub fn switch_normal_to_normal(old: *ke.Thread.Context, new: *ke.Thread.Context) void {
+    const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old));
+    riscv_context_switch(&old.impl, &new.impl, &thread.lock.inner.locked.raw, &thread.switching.raw);
+}
+
+pub fn switch_cont_to_normal(old: *ke.Thread.Context, new: *ke.Thread.Context) void {
+    const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old));
+    riscv_context_switch_cont(old, &new.impl, &thread.lock.inner.locked.raw, &thread.switching.raw);
+}
+
+pub fn call_continuation(ctx: *ke.Thread.Context, continuation: ke.Continuation) noreturn {
+    const sp = ctx.stack_top & ~@as(usize, 15);
+    asm volatile (
+        \\mv sp, %[stack]
+        \\mv a0, %[func]
+        \\mv a1, %[arg]
+        \\tail riscv_run_continuation
+        :
+        : [stack] "r" (sp),
+          [func] "r" (continuation.func),
+          [arg] "r" (continuation.arg),
+        : .{ .memory = true });
+    unreachable;
+}
+
+pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) {
+    return variable;
+}
+
+pub inline fn percpu_ptr_other(variable: anytype, id: u32) @TypeOf(variable) {
+    // TODO: start up harts
+    if (id != 0) @panic("secondary harts are not supported");
+    return variable;
+}
+
+pub fn early_init() void {
+    _ = disable_interrupts();
+}
+
+pub inline fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
+    const current = ke.private.ipl.cpu_ipl.local();
+    old.* = current.*;
+    current.* = new;
+    return new.value() < old.value();
+}
+
+pub inline fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
+    const current = ke.private.ipl.cpu_ipl.local();
+    old.* = current.*;
+    current.* = new;
+    return new.value() > old.value();
+}
+
+pub fn set_hardware_ipl(level: ke.Ipl) void {
+    if (level.value() > ke.Ipl.Dispatch.value()) {
+        asm volatile ("csrc sie, %[mask]"
+            :
+            : [mask] "r" (@as(usize, 32)),
+            : .{ .memory = true });
+    } else {
+        asm volatile ("csrs sie, %[mask]"
+            :
+            : [mask] "r" (@as(usize, 32)),
+            : .{ .memory = true });
+    }
+}
+
+pub inline fn disable_interrupts() bool {
+    const old = asm volatile ("csrrc %[old], sstatus, %[mask]"
+        : [old] "=r" (-> usize),
+        : [mask] "r" (@as(usize, 2)),
+        : .{ .memory = true });
+    return old & 2 != 0;
+}
+
+pub inline fn enable_interrupts() void {
+    asm volatile ("csrsi sstatus, 2" ::: .{ .memory = true });
+}
+
+pub inline fn restore_interrupts(state: bool) void {
+    if (state) enable_interrupts() else _ = disable_interrupts();
+}
+
+pub inline fn halt() void {
+    asm volatile ("wfi");
+}
+
+pub inline fn flush_full_tlb() void {
+    rv64.sfence_vma();
+}
+
+pub inline fn flush_tlb(va: r.VAddr) void {
+    rv64.sfence_vma_page(va);
+}

src/kern/riscv64/locore.s

@@ -0,0 +1,71 @@
+.section .text
+.global riscv_context_switch
+.type riscv_context_switch, @function
+riscv_context_switch:
+    sd sp, 0(a0)
+    sd ra, 8(a0)
+    sd s0, 16(a0)
+    sd s1, 24(a0)
+    sd s2, 32(a0)
+    sd s3, 40(a0)
+    sd s4, 48(a0)
+    sd s5, 56(a0)
+    sd s6, 64(a0)
+    sd s7, 72(a0)
+    sd s8, 80(a0)
+    sd s9, 88(a0)
+    sd s10, 96(a0)
+    sd s11, 104(a0)
+    mv a0, a1
+    j riscv_context_resume
+
+.global riscv_context_load
+.type riscv_context_load, @function
+riscv_context_load:
+    mv a2, zero
+    mv a3, zero
+    j riscv_context_resume
+
+riscv_context_resume:
+    ld sp, 0(a0)
+    ld ra, 8(a0)
+    ld s0, 16(a0)
+    ld s1, 24(a0)
+    ld s2, 32(a0)
+    ld s3, 40(a0)
+    ld s4, 48(a0)
+    ld s5, 56(a0)
+    ld s6, 64(a0)
+    ld s7, 72(a0)
+    ld s8, 80(a0)
+    ld s9, 88(a0)
+    ld s10, 96(a0)
+    ld s11, 104(a0)
+    beqz a3, 1f
+    sb zero, 0(a3)
+1:
+    beqz a2, 2f
+    sb zero, 0(a2)
+2:
+    ret
+
+.global riscv_context_switch_cont
+.type riscv_context_switch_cont, @function
+riscv_context_switch_cont:
+    mv s2, a2
+    mv s3, a3
+    mv s4, a1
+    ld a0, 112(a0)
+    ld sp, 0(s4)
+    call riscv_free_old_stack
+    mv a0, s4
+    mv a2, s2
+    mv a3, s3
+    j riscv_context_resume
+
+.global thread_start
+.type thread_start, @function
+thread_start:
+    mv a0, s1
+    mv a1, s2
+    tail riscv_thread_entry

src/loader/riscv64/impl.zig

@@ -0,0 +1,122 @@
+const riscv64 = @import("arch");
+const r = @import("root");
+const pmap = @import("../pmap.zig");
+
+pub const levels = [_]pmap.PMapLevel{
+    .{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K
+    .{ .shift = 21, .mask = 0x1ff, .leaf = true }, // 2M
+    .{ .shift = 30, .mask = 0x1ff, .leaf = true }, // 1G
+    .{ .shift = 39, .mask = 0x1ff, .leaf = false }, // root
+};
+
+pub const virtual_bits: usize = 48;
+
+pub const Pte = packed struct(u64) {
+    present: bool,
+    read: bool,
+    write: bool,
+    exec: bool,
+    user: bool,
+    global: bool,
+    access: bool,
+    dirty: bool,
+    rsw: u2,
+    addr: u44,
+    reserved: u7,
+    pbmt: u2,
+    n: bool,
+
+    pub fn address(self: Pte) usize {
+        return @as(usize, self.addr) << 12;
+    }
+
+    pub fn is_present(self: Pte) bool {
+        return self.present;
+    }
+
+    pub fn load(table: *Pte) Pte {
+        return @atomicLoad(Pte, table, .monotonic);
+    }
+
+    pub fn zero() Pte {
+        return Pte{
+            .present = false,
+            .read = false,
+            .write = false,
+            .exec = false,
+            .user = false,
+            .global = false,
+            .access = false,
+            .dirty = false,
+            .rsw = 0,
+            .addr = 0,
+            .reserved = 0,
+            .pbmt = 0,
+            .n = false,
+        };
+    }
+};
+
+pub inline fn make_table_pte(pa: usize) Pte {
+    return Pte{
+        .present = true,
+        .read = false,
+        .write = false,
+        .exec = false,
+        .user = false,
+        .global = false,
+        .access = false,
+        .dirty = false,
+        .rsw = 0,
+        .addr = @truncate(pa >> 12),
+        .reserved = 0,
+        .pbmt = 0,
+        .n = false,
+    };
+}
+
+pub inline fn make_leaf_pte(pa: usize, flags: r.mem.MapFlags, level: usize) Pte {
+    _ = level;
+    return Pte{
+        .present = true,
+        .read = flags.read or flags.write,
+        .write = flags.write,
+        .exec = flags.execute,
+        .user = flags.user,
+        .global = flags.global,
+        .access = true,
+        .dirty = flags.write,
+        .rsw = 0,
+        .addr = @truncate(pa >> 12),
+        .reserved = 0,
+        .pbmt = 0,
+        .n = false,
+    };
+}
+
+pub inline fn activate(root_pa: usize) void {
+    riscv64.write_csr("satp", (@as(u64, 9) << 60) | (root_pa >> 12));
+    riscv64.sfence_vma();
+}
+
+pub inline fn flush(va: usize) void {
+    riscv64.sfence_vma_page(va);
+}
+
+pub fn is_leaf_level_enabled(level: usize) bool {
+    if (!levels[level].leaf) return false;
+    return true;
+}
+
+pub fn debug_write(c: u8) void {
+    // OpenSBI debug ext
+    asm volatile ("ecall"
+        :
+        : [char] "{a0}" (@as(usize, c)),
+          [extension] "{a7}" (@as(usize, 1)),
+        : .{ .memory = true });
+}
+
+pub fn init() void {
+    r.loader_info.arch_info = .{};
+}

src/loader/riscv64/jump.s

@@ -0,0 +1,7 @@
+.section .text
+.global jump_to_kernel
+.type jump_to_kernel, @function
+jump_to_kernel:
+    mv t0, a1
+    mv sp, a2
+    jr t0

src/loader/root.zig

@@ -82,6 +82,7 @@ pub const ImplSchema = struct {
 
 pub const arch = switch (config.arch) {
     .amd64 => @import("amd64/impl.zig"),
+    .riscv64 => @import("riscv64/impl.zig"),
     else => @compileError("unsupported architecture"),
 };
 

src/mm/private.zig

@@ -30,6 +30,7 @@ pub const PMapLevel = struct {
 
 pub const impl = switch (config.arch) {
     .amd64 => @import("amd64/impl.zig"),
+    .riscv64 => @import("riscv64/impl.zig"),
     else => @compileError("unsupported architecture"),
 };
 

src/mm/riscv64/impl.zig

@@ -0,0 +1,122 @@
+const rv64 = @import("arch");
+const r = @import("root");
+const mm = r.mm;
+const mmp = mm.private;
+
+pub const hhdm_minimum_max_address = r.gib(4);
+
+pub fn phys_to_virt(addr: r.PAddr) r.VAddr {
+    return addr + 0xffff800000000000;
+}
+
+pub fn virt_to_phys(vaddr: r.VAddr) usize {
+    return vaddr - 0xffff800000000000;
+}
+
+pub const hhdm_base = 0xffff800000000000;
+pub const kernel_heap_base = 0xffffc00000000000;
+pub const pfndb_base = 0xffffd00000000000;
+
+pub const levels = [_]mmp.PMapLevel{
+    .{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K
+    .{ .shift = 21, .mask = 0x1ff, .leaf = true }, // 2M
+    .{ .shift = 30, .mask = 0x1ff, .leaf = true }, // 1G
+    .{ .shift = 39, .mask = 0x1ff, .leaf = false }, // root
+};
+
+pub const Pte = packed struct(u64) {
+    present: bool,
+    read: bool,
+    write: bool,
+    exec: bool,
+    user: bool,
+    global: bool,
+    access: bool,
+    dirty: bool,
+    rsw: u2,
+    addr: u44,
+    reserved: u7,
+    pbmt: u2,
+    n: bool,
+
+    pub fn address(self: Pte) r.PAddr {
+        return @as(r.PAddr, self.addr) << 12;
+    }
+
+    pub fn is_present(self: Pte) bool {
+        return self.present;
+    }
+
+    pub fn load(table: *Pte) Pte {
+        return @atomicLoad(Pte, table, .monotonic);
+    }
+
+    pub fn zero() Pte {
+        return Pte{
+            .present = false,
+            .read = false,
+            .write = false,
+            .exec = false,
+            .user = false,
+            .global = false,
+            .access = false,
+            .dirty = false,
+            .rsw = 0,
+            .addr = 0,
+            .reserved = 0,
+            .pbmt = 0,
+            .n = false,
+        };
+    }
+};
+
+pub inline fn make_table_pte(pa: r.PAddr) Pte {
+    return Pte{
+        .present = true,
+        .read = false,
+        .write = false,
+        .exec = false,
+        .user = false,
+        .global = false,
+        .access = false,
+        .dirty = false,
+        .rsw = 0,
+        .addr = @truncate(pa >> 12),
+        .reserved = 0,
+        .pbmt = 0,
+        .n = false,
+    };
+}
+
+pub inline fn make_leaf_pte(pa: r.PAddr, flags: mm.MapFlags, level: usize) Pte {
+    _ = level;
+    return Pte{
+        .present = true,
+        .read = flags.read or flags.write,
+        .write = flags.write,
+        .exec = flags.execute,
+        .user = flags.user,
+        .global = flags.global,
+        .access = true,
+        .dirty = flags.write,
+        .rsw = 0,
+        .addr = @truncate(pa >> 12),
+        .reserved = 0,
+        .pbmt = 0,
+        .n = false,
+    };
+}
+
+pub inline fn activate(root_pa: r.PAddr) void {
+    rv64.write_csr("satp", (@as(u64, 9) << 60) | (root_pa >> 12));
+    rv64.sfence_vma();
+}
+
+pub fn is_leaf_level_enabled(level: usize) bool {
+    if (!levels[level].leaf) return false;
+    return true;
+}
+
+pub fn init_kernel() void {
+    mmp.kernel_space.pmap.root_pa = (rv64.read_csr("satp") & ((@as(u64, 1) << 44) - 1)) << 12;
+}

src/platform/riscv64/impl.zig

@@ -0,0 +1,83 @@
+const r = @import("root");
+const rv64 = r.arch;
+const ke = r.ke;
+const std = @import("std");
+extern fn riscv_trap() callconv(.c) void;
+
+// for QEMU virt. for real HW this needs to be parsed from the DTB.
+const timebase_hz = 10_000_000;
+var time_counter: ke.TimeCounter = .{
+    .read_count = read_time,
+    .frequency = timebase_hz,
+    .name = "riscv time",
+    .quality = 100,
+    .mask = std.math.maxInt(u64),
+    .p = 0,
+    .n = 0,
+};
+
+fn read_time() u64 {
+    return rv64.read_csr("time");
+}
+
+pub const name: []const u8 = "QEMU virt";
+
+pub fn early_init() void {
+    rv64.write_csr("stvec", @intFromPtr(&riscv_trap));
+}
+
+pub fn late_init() void {
+    ke.time.register_source(&time_counter);
+    asm volatile ("csrs sie, %[mask]"
+        :
+        : [mask] "r" (@as(usize, 32)),
+        : .{ .memory = true });
+}
+
+export fn riscv_trap_handler(cause: usize, pc: usize) callconv(.c) void {
+    if (cause == ((@as(usize, 1) << 63) | 5)) {
+        const old = ke.ipl.set_hardware(.Device);
+        set_timer(std.math.maxInt(u64));
+        ke.private.timer.clock();
+        _ = ke.ipl.set_hardware(old);
+        if (old == .Passive and ke.private.ipl.is_softint_pending(.Dispatch)) {
+            ke.private.dpc.dispatch();
+        }
+        return;
+    }
+    std.debug.panic("trap: cause = 0x{x}, pc = 0x{x}, tval = 0x{x}", .{ cause, pc, rv64.read_csr("stval") });
+}
+
+fn set_timer(deadline: u64) void {
+    asm volatile ("ecall"
+        :
+        : [deadline] "{a0}" (deadline),
+          [function] "{a6}" (@as(usize, 0)),
+          [extension] "{a7}" (@as(usize, 0x54494d45)),
+        : .{ .memory = true });
+}
+
+pub fn debug_write(c: u8) void {
+    // SBI legacy console putchar.
+    asm volatile ("ecall"
+        :
+        : [char] "{a0}" (@as(usize, c)),
+          [extension] "{a7}" (@as(usize, 1)),
+        : .{ .memory = true });
+}
+
+pub fn debug_read() u8 {
+    while (true) {
+        const c = asm volatile ("ecall"
+            : [char] "={a0}" (-> usize),
+            : [extension] "{a7}" (@as(usize, 2)),
+            : .{ .memory = true });
+        if (c != std.math.maxInt(usize)) return @truncate(c);
+    }
+}
+
+pub fn send_ipi(_: u32) void {}
+pub fn arm_timer(ns: r.Nanoseconds) void {
+    const ticks = @max(1, @as(u64, @intCast((@as(u128, ns.value) * timebase_hz) / std.time.ns_per_s)));
+    set_timer(read_time() +% ticks);
+}

src/platform/riscv64/trap.s

@@ -0,0 +1,74 @@
+.section .text
+.global riscv_trap
+.type riscv_trap, @function
+riscv_trap:
+    addi sp, sp, -272
+    sd ra, 0(sp)
+    sd gp, 8(sp)
+    sd tp, 16(sp)
+    sd t0, 24(sp)
+    sd t1, 32(sp)
+    sd t2, 40(sp)
+    sd s0, 48(sp)
+    sd s1, 56(sp)
+    sd a0, 64(sp)
+    sd a1, 72(sp)
+    sd a2, 80(sp)
+    sd a3, 88(sp)
+    sd a4, 96(sp)
+    sd a5, 104(sp)
+    sd a6, 112(sp)
+    sd a7, 120(sp)
+    sd s2, 128(sp)
+    sd s3, 136(sp)
+    sd s4, 144(sp)
+    sd s5, 152(sp)
+    sd s6, 160(sp)
+    sd s7, 168(sp)
+    sd s8, 176(sp)
+    sd s9, 184(sp)
+    sd s10, 192(sp)
+    sd s11, 200(sp)
+    sd t3, 208(sp)
+    sd t4, 216(sp)
+    sd t5, 224(sp)
+    sd t6, 232(sp)
+    csrr t0, sepc
+    sd t0, 240(sp)
+    csrr a0, scause
+    mv a1, t0
+    call riscv_trap_handler
+    ld t0, 240(sp)
+    csrw sepc, t0
+    ld ra, 0(sp)
+    ld gp, 8(sp)
+    ld tp, 16(sp)
+    ld t0, 24(sp)
+    ld t1, 32(sp)
+    ld t2, 40(sp)
+    ld s0, 48(sp)
+    ld s1, 56(sp)
+    ld a0, 64(sp)
+    ld a1, 72(sp)
+    ld a2, 80(sp)
+    ld a3, 88(sp)
+    ld a4, 96(sp)
+    ld a5, 104(sp)
+    ld a6, 112(sp)
+    ld a7, 120(sp)
+    ld s2, 128(sp)
+    ld s3, 136(sp)
+    ld s4, 144(sp)
+    ld s5, 152(sp)
+    ld s6, 160(sp)
+    ld s7, 168(sp)
+    ld s8, 176(sp)
+    ld s9, 184(sp)
+    ld s10, 192(sp)
+    ld s11, 200(sp)
+    ld t3, 208(sp)
+    ld t4, 216(sp)
+    ld t5, 224(sp)
+    ld t6, 232(sp)
+    addi sp, sp, 272
+    sret

src/platform/root.zig

@@ -7,6 +7,7 @@ const rtl = @import("rtl");
 
 pub const impl = switch (config.arch) {
     .amd64 => @import("pc/impl.zig"),
+    .riscv64 => @import("riscv64/impl.zig"),
     else => @compileError("unsupported architecture"),
 };