diff --git a/build/run.zig b/build/run.zig index 37be7d4..12a17cf 100644 --- a/build/run.zig +++ b/build/run.zig @@ -44,9 +44,12 @@ pub fn addRun(b: *std.Build, kernel: *std.Build.Step.Compile, plat: config.Platf if (debug) { qemu.addArgs(&.{ - "-d", "int", - "-D", "qemu.log", - "-no-reboot", "-no-shutdown", + "-d", + "int", + "-D", + "qemu.log", + "-no-reboot", + "-no-shutdown", }); } diff --git a/src/ex/fireworks.zig b/src/ex/fireworks.zig index 72a8879..26911db 100644 --- a/src/ex/fireworks.zig +++ b/src/ex/fireworks.zig @@ -1,13 +1,10 @@ //! Fireworks test ported from the Boron operating system. const std = @import("std"); -const base = @import("root"); -const ke = base.ke; +const r = @import("root"); +const ke = r.ke; +const mm = r.mm; const sintab = @import("sintab.zig"); -const c = @cImport({ - @cInclude("stdlib.h"); -}); - var pixel_buffer: [*]u8 = undefined; var fb_width: usize = 0; var fb_height: usize = 0; @@ -130,8 +127,6 @@ fn particle(param: ?*anyopaque) void { data.act_y = parent_data.act_y; const explosion_range = parent_data.explosion_range; - c.free(param); - const angle = @rem(rand(), 65536); data.vel_x = mul_fp_fp(cos(angle), rand_fp_sign()) * explosion_range; data.vel_y = mul_fp_fp(sin(angle), rand_fp_sign()) * explosion_range; @@ -169,10 +164,10 @@ fn particle(param: ?*anyopaque) void { } fn make_thread(entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) *ke.Thread { - var ret: *ke.Thread = @ptrCast(@alignCast(c.malloc(@sizeOf(ke.Thread)))); - const stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf"); + var ret: *ke.Thread = mm.zone.gpa.create(ke.Thread) catch @panic("oom"); + const stack = mm.heap.alloc(r.kib(16)) catch @panic("wtf"); - ret.init(@intFromPtr(stack.ptr), 16384, entry, arg); + ret.init(@intFromPtr(stack), r.kib(16), entry, arg); ret.priority = ke.Thread.Priority.default; ret.priority_class = .Batch; @@ -235,7 +230,7 @@ fn explodeable(_: ?*anyopaque) void { const part_count: usize = @intCast(@rem(rand(), 100) + 100); for (0..part_count) |_| { - const param: *FireworkData = @ptrCast(@alignCast(c.malloc(@sizeOf(FireworkData)))); + const param: *FireworkData = mm.zone.gpa.create(FireworkData) catch @panic("oom"); param.* = data; spawn_particle(param); } @@ -244,7 +239,7 @@ fn explodeable(_: ?*anyopaque) void { } pub fn start(param: ?*anyopaque) void { - const boot_info: *base.pl.BootInfo = @ptrCast(@alignCast(param)); + const boot_info: *r.pl.BootInfo = @ptrCast(@alignCast(param)); rand_gen ^= rand_tsc_based(); diff --git a/src/ex/init.zig b/src/ex/init.zig index 8c89dfc..59ea5f8 100644 --- a/src/ex/init.zig +++ b/src/ex/init.zig @@ -3,11 +3,10 @@ const r = @import("root"); const pl = r.pl; const ke = r.ke; const mm = r.mm; +const ex = r.ex; const amd64 = @import("arch"); -fn callback(_: ?*anyopaque) void { - std.log.info("Timer callback called", .{}); -} +var thread0: ke.Thread = undefined; pub fn init(boot_info: *pl.BootInfo) void { mm.init(boot_info); @@ -16,12 +15,9 @@ pub fn init(boot_info: *pl.BootInfo) void { mm.late_init(); ke.sched.late_init(); - var timer: ke.Timer = undefined; - var dpc: ke.Dpc = .init(callback); - - timer.init(); - - ke.timer.set(&timer, std.time.ns_per_ms * 50, &dpc); + const sp = mm.heap.alloc(r.kib(16)) catch @panic("oom"); + thread0.init(@intFromPtr(sp), r.kib(16), ex.fireworks.start, boot_info); + ke.sched.enqueue(&thread0); while (true) { std.atomic.spinLoopHint(); diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 00b24f6..5f36acf 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -11,92 +11,54 @@ pub const early_init = init.early_init; pub export var cpu_self_offset: usize linksection(r.percpu) = 0; pub export var cpu_offsets: [*]usize = undefined; -pub const ThreadContext = struct { - rdi: u64, - rsi: u64, - rbx: u64, - rsp: u64, - rbp: u64, - r12: u64, - r13: u64, - r14: u64, - r15: u64, +const ThreadFrame = extern struct { + rbp: u64 align(1), + rbx: u64 align(1), + r12: u64 align(1), + r13: u64 align(1), + r14: u64 align(1), + r15: u64 align(1), + rip: u64 align(1), +}; - fn thread_trampoline(entry: usize, arg: usize) callconv(.c) void { +extern fn asm_thread_entry() void; + +extern fn do_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8) callconv(.c) void; + +pub const ThreadContext = extern struct { + rsp: u64 align(1), + frame: ThreadFrame, + + export fn thread_entry(entry: usize, arg: usize) callconv(.c) void { const entry_fn: *const fn (?*anyopaque) void = @ptrFromInt(entry); + ke.ipl.lower(.Passive); entry_fn(@ptrFromInt(arg)); // TODO: This shouldn't be reached!!! } pub fn init(stack: r.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() { var ctx: @This() = undefined; - ctx.rdi = @intFromPtr(entry); - ctx.rsi = @intFromPtr(arg); + var sp: usize = stack + stack_size; - var sp = stack + stack_size; - { - sp -= @sizeOf(usize); - @as(*usize, @ptrFromInt(sp)).* = 0; - - sp -= @sizeOf(usize); - @as(*usize, @ptrFromInt(sp)).* = @intFromPtr(&thread_trampoline); + if (sp % 16 != 0) { + sp -= (sp % 16); } - ctx.rsp = sp; + + const frame: *ThreadFrame = @ptrFromInt(sp - @sizeOf(ThreadFrame)); + + frame.* = std.mem.zeroes(ThreadFrame); + + ctx.rsp = @intFromPtr(frame); + frame.rip = @intFromPtr(&asm_thread_entry); + frame.r12 = @intFromPtr(entry); + frame.r13 = @intFromPtr(arg); return ctx; } - export fn do_switch_to() callconv(.naked) void { - const asm_template = std.fmt.comptimePrint( - \\movq %%rbx, {d}(%%rdi) - \\movq %%rsp, {d}(%%rdi) - \\movq %%rbp, {d}(%%rdi) - \\movq %%r12, {d}(%%rdi) - \\movq %%r13, {d}(%%rdi) - \\movq %%r14, {d}(%%rdi) - \\movq %%r15, {d}(%%rdi) - \\ - \\movq {d}(%%rsi), %%rbx - \\movq {d}(%%rsi), %%rsp - \\movq {d}(%%rsi), %%rbp - \\movq {d}(%%rsi), %%r12 - \\movq {d}(%%rsi), %%r13 - \\movq {d}(%%rsi), %%r14 - \\movq {d}(%%rsi), %%r15 - \\movq {d}(%%rsi), %%rdi - \\movq {d}(%%rsi), %%rsi - \\ - \\ret - , .{ - @offsetOf(ThreadContext, "rbx"), - @offsetOf(ThreadContext, "rsp"), - @offsetOf(ThreadContext, "rbp"), - @offsetOf(ThreadContext, "r12"), - @offsetOf(ThreadContext, "r13"), - @offsetOf(ThreadContext, "r14"), - @offsetOf(ThreadContext, "r15"), - - @offsetOf(ThreadContext, "rbx"), - @offsetOf(ThreadContext, "rsp"), - @offsetOf(ThreadContext, "rbp"), - @offsetOf(ThreadContext, "r12"), - @offsetOf(ThreadContext, "r13"), - @offsetOf(ThreadContext, "r14"), - @offsetOf(ThreadContext, "r15"), - @offsetOf(ThreadContext, "rdi"), - @offsetOf(ThreadContext, "rsi"), - }); - - asm volatile (asm_template); - } - pub fn switch_to(self: *ThreadContext, new: *ThreadContext) callconv(.c) void { - asm volatile ( - \\call do_switch_to - : - : [_] "{rdi}" (self), - [_] "{rsi}" (new), - ); + const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", self)); + do_context_switch(self, new, &thread.lock.locked); } }; @@ -118,9 +80,14 @@ pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) { } pub inline fn set_hardware_ipl(ipl: ke.Ipl) void { + const cr8_value: u64 = switch (ipl) { + .Passive, .Dispatch => 0, + .Device => 13, + .High => 15, + }; asm volatile ("mov %[ipl], %%cr8" : - : [ipl] "r" (@as(u64, @intFromEnum(ipl))), + : [ipl] "r" (cr8_value), ); } @@ -144,6 +111,6 @@ pub inline fn restore_interrupts(state: bool) void { } } -pub inline fn send_resched_ipi(_: u32) void { - @panic("TODO send_resched_ipi"); +pub inline fn send_resched_ipi(target: u32) void { + pl.impl.send_resched_ipi(target); } diff --git a/src/kern/amd64/init.zig b/src/kern/amd64/init.zig index a81d8b7..b74c72b 100644 --- a/src/kern/amd64/init.zig +++ b/src/kern/amd64/init.zig @@ -86,11 +86,16 @@ fn early_cpu_init() linksection(r.init) void { } pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn { + amd64.write_msr(.GsBase, ki.impl.cpu_offsets[cpu_id]); + early_cpu_init(); ki.cpu.init_cpu(cpu_id); pl.impl.init_ap(); + ki.sched.percpu.local().current_thread = pl.impl.smp.start_thread.local().*; + _ = booted.fetchAdd(1, .monotonic); + ke.ipl.lower(.Passive); while (true) { std.atomic.spinLoopHint(); diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index 8aca96b..3c06bec 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -87,7 +87,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys pl.impl.apic.eoi(); } - ki.impl.set_hardware_ipl(ipl); + _ = ke.ipl.set_hardware(ipl); if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) { ki.dpc.dispatch(ke.cpu.current()); diff --git a/src/kern/amd64/locore.s b/src/kern/amd64/locore.s index 0b9fc7b..249a46a 100644 --- a/src/kern/amd64/locore.s +++ b/src/kern/amd64/locore.s @@ -131,3 +131,42 @@ __interrupt_vectors: .rept 256 .quad isr\()\+ .endr + + +.global do_context_switch +do_context_switch: + /* Save callee-saved registers. */ + push %r15 + push %r14 + push %r13 + push %r12 + push %rbx + push %rbp + + /* Save stack pointer. */ + mov %rsp, (%rdi) + + /* Load new stack pointer. */ + mov (%rsi), %rsp + + /* Unlock passed lock. */ + xorl %ecx, %ecx + xchgb %cl, (%rdx) + + /* Restore callee-saved registers. */ + pop %rbp + pop %rbx + pop %r12 + pop %r13 + pop %r14 + pop %r15 + ret + + +.extern thread_entry +.global asm_thread_entry +asm_thread_entry: + mov %r12, %rdi + mov %r13, %rsi + call thread_entry + ud2 diff --git a/src/kern/init.zig b/src/kern/init.zig index dbbf471..4d17d43 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -7,7 +7,7 @@ const ki = ke.private; const ex = r.ex; var thread0: ke.Thread = undefined; -var stack: [8192]u8 align(16) = undefined; +var stack: [r.kib(16)]u8 align(16) = undefined; pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void { ki.impl.early_init(); @@ -15,10 +15,12 @@ pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void { pl.early_init(boot_info); ki.cpu.init_cpu(0); - thread0.init(@intFromPtr(&stack), 8192, ki.sched.idle, null); + thread0.init(@intFromPtr(&stack), r.kib(16), ki.sched.idle, null); thread0.priority = 0; thread0.priority_class = .Idle; + ki.sched.percpu.local().current_thread = &thread0; + std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ pl.name, arch.name, boot_info.cmdline }); ex.private.init(boot_info); diff --git a/src/kern/sched.zig b/src/kern/sched.zig index 72142d3..734f2aa 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -795,26 +795,26 @@ fn yield(cpu: u32) void { sched_cpu.queues_lock.acquire_no_ipl(); - const cur = cpu.current_thread; - var next = cpu.next_thread; + const cur = sched_cpu.current_thread; + var next = sched_cpu.next_thread; if (next != null) { - cpu.next_thread = null; + sched_cpu.next_thread = null; } else { // Pick a new thread to run. - next = select_thread(null, cpu, false); + next = select_thread(null, sched_cpu, false); } if (next == null) { // Nothing to run, go idle. - next = cpu.idle_thread; + next = sched_cpu.idle_thread; sched_cpu.steal_work = true; } sched_cpu.queues_lock.release_no_ipl(); // Switch into the thread. - do_switch(cpu, cur.?, next.?); + do_switch(sched_cpu, cur.?, next.?); // cur lock dropped } diff --git a/src/kern/timer.zig b/src/kern/timer.zig index ea879bc..f4f3e73 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -172,13 +172,16 @@ fn handle_expiry(_: ?*anyopaque) void { return; } + if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) { + // Timer must've been canceled. + cpu.lock.release_no_ipl(); + continue; + } + _ = cpu.timers.pop(); cpu.lock.release_no_ipl(); - if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) { - // Timer must've been canceled. - continue; - } + const maybe_dpc = timer.dpc; timer.hdr.lock.acquire_no_ipl(); @@ -186,12 +189,12 @@ fn handle_expiry(_: ?*anyopaque) void { timer.hdr.signaled = 1; timer.cpu = null; - // Wake whomever was waiting on the timer. - ki.wait.satisfy_wait(&timer.hdr); - timer.state.store(.Stopped, .release); - if (timer.dpc) |dpc| ke.dpc.enqueue(dpc, null); + // Wake whomever was waiting on the timer. + ki.wait.satisfy_wait(&timer.hdr); timer.hdr.lock.release_no_ipl(); + + if (maybe_dpc) |dpc| ke.dpc.enqueue(dpc, null); } } diff --git a/src/kern/wait.zig b/src/kern/wait.zig index 2ef5240..a3e2aa5 100644 --- a/src/kern/wait.zig +++ b/src/kern/wait.zig @@ -5,6 +5,7 @@ const rtl = @import("rtl"); const std = @import("std"); const r = @import("root"); const ke = r.ke; +const ki = ke.private; /// Header for waitable objects. /// This must be added to any structured which is considered waitable. @@ -86,7 +87,7 @@ pub fn wait_one(object: *DispatchHeader, timeout: ?r.Nanoseconds) !usize { pub fn wait_any(objects: []*DispatchHeader, timeout: ?r.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize { const ipl = ke.ipl.raise(.Dispatch); defer ke.ipl.lower(ipl); - const curtd = ke.curcpu().current_thread.?; + const curtd = ki.sched.percpu.local().current_thread.?; const obj_count = objects.len; const has_timeout = timeout != null; const blocks = waitblocks orelse blk: { diff --git a/src/mm/init.zig b/src/mm/init.zig index 8ea7e09..cef377a 100644 --- a/src/mm/init.zig +++ b/src/mm/init.zig @@ -28,6 +28,15 @@ pub const map_kernel = if (!is_um) const rodata_size = rodata_end - rodata_start; const data_size = data_end - data_start; + const start_phys = kaddr.physical_base; + const start_virt = kaddr.virtual_base; + const until_size = text_start - start_virt; + mi.kernel_pmap.map_contiguous_range(start_virt, start_phys, until_size, .{ + .read = true, + .write = true, + .global = true, + }); + mi.kernel_pmap.map_contiguous_range(text_start, kaddr.physical_base + (text_start - kaddr.virtual_base), text_size, .{ .read = true, .execute = true, diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 442c992..1fcd567 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -51,6 +51,15 @@ var batch_pool: std.atomic.Value(GlobalHead) = .init(.{ .tag = 0, }); +export const phys_percpu_init linksection(r.percpu_init) = &init_cpu; + +fn init_cpu() callconv(.c) void { + const cpu = percpu.local(); + cpu.active_batch = null; + cpu.active_count = 0; + cpu.depot_count = 0; +} + /// Push a batch of free pages to the global pool. fn push_to_pool(batch: *mm.Page) void { var old_head = batch_pool.load(.acquire); diff --git a/src/mm/zone.zig b/src/mm/zone.zig index 882340a..1a2aa27 100644 --- a/src/mm/zone.zig +++ b/src/mm/zone.zig @@ -766,12 +766,14 @@ pub fn late_init() linksection(r.init) void { z.cpus = gpa.alloc(Cpu, ke.ncpus) catch @panic("Failed to allocate per-CPU magazine state"); for (0..ke.ncpus) |i| { - z.cpus[i].lock = .init(); - z.cpus[i].alloc = null; - z.cpus[i].free = null; - z.cpus[i].alloc_rounds = 0; - z.cpus[i].free_rounds = 0; - z.cpus[i].magazine_size = z.magtype.?.rounds; + z.cpus[i] = .{ + .lock = .init(), + .alloc = null, + .free = null, + .alloc_rounds = 0, + .free_rounds = 0, + .magazine_size = z.magtype.?.rounds, + }; } } zone = z.next; diff --git a/src/platform/pc/apic.zig b/src/platform/pc/apic.zig index 9522b08..77ca25b 100644 --- a/src/platform/pc/apic.zig +++ b/src/platform/pc/apic.zig @@ -142,10 +142,11 @@ pub fn send_sipi(apic_id: u32, startup_page: u8) void { } pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void { - const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector; if (in_x2apic_mode.local().*) { + const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector; x2apic_write(.Icr, icr_value); } else { + const icr_value = (@as(u32, delivery_mode) << 8) | vector; // Write to ICR1 first, because writing to the low word causes // the IPI to be sent. xapic_write(.Icr1, @as(u32, apic_id) << 24); diff --git a/src/platform/pc/impl.zig b/src/platform/pc/impl.zig index eea73d3..930f2f8 100644 --- a/src/platform/pc/impl.zig +++ b/src/platform/pc/impl.zig @@ -5,7 +5,7 @@ const std = @import("std"); const tsc = @import("tsc.zig"); const pvclock = @import("pvclock.zig"); pub const apic = @import("apic.zig"); -const smp = @import("smp.zig"); +pub const smp = @import("smp.zig"); const ki = r.ke.private; @@ -105,3 +105,7 @@ pub fn debug_write(c: u8) void { pub fn arm_timer(deadline: r.Nanoseconds) void { apic.arm_timer(deadline); } + +pub fn send_resched_ipi(target_cpu: u32) void { + apic.send_ipi(smp.cpu_id_to_apic_id[target_cpu], 240, 0); +} diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index 2786f65..6487976 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -17,10 +17,12 @@ extern var AP_TRAMPOLINE_DATA: u8; extern var __percpu_start: u8; extern var __percpu_end: u8; -export var cpu_id_to_apic_id: [config.CONFIG_NCPUS]u32 = undefined; +pub export var cpu_id_to_apic_id: [config.CONFIG_NCPUS]u32 = undefined; const start_stack = ke.ExportedCpuLocal(usize, 0, "ap_start_stack"); +pub const start_thread = ke.CpuLocal(*ke.Thread, undefined); + var aps_booted: std.atomic.Value(usize) = .init(0); const ApData = extern struct { @@ -35,6 +37,16 @@ fn ap_entry(cpu_id: u32) callconv(.c) noreturn { ki.impl.init.ap_entry(cpu_id, &aps_booted); } +fn make_thread(entrypoint: *const fn (?*anyopaque) void, stack: usize) *ke.Thread { + const td = mm.zone.gpa.create(ke.Thread) catch @panic("Failed to allocate thread for AP"); + + td.init(stack, r.kib(32), entrypoint, null); + + td.priority = 0; + td.priority_class = .Idle; + return td; +} + pub fn init() linksection(r.init) void { const trampoline_start = @intFromPtr(&AP_TRAMPOLINE_START); const trampoline_size = @intFromPtr(&AP_TRAMPOLINE_END) - trampoline_start; @@ -94,10 +106,12 @@ pub fn init() linksection(r.init) void { // copy the offset into the AP's self_offset variable @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id]; - const stack_top = @intFromPtr(mm.heap.alloc(r.kib(16)) catch @panic("Failed to allocate AP stack")) + r.kib(16); + const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32)) catch @panic("Failed to allocate AP stack")) + r.kib(32); start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15); + start_thread.remote(@intCast(cpu_id)).* = make_thread(@ptrCast(&ap_entry), stack_top - r.kib(32)); + rtl.barrier.wmb(); // Send the INIT-SIPI-SIPI sequence to start the AP. @@ -109,6 +123,8 @@ pub fn init() linksection(r.init) void { apic.send_sipi(apic_id, 0x08); } + log.info("Waiting for APs...", .{}); + while (aps_booted.load(.acquire) < apic.apics.items.len) { std.atomic.spinLoopHint(); } diff --git a/src/platform/um/impl.zig b/src/platform/um/impl.zig index 386b7aa..54c21e7 100644 --- a/src/platform/um/impl.zig +++ b/src/platform/um/impl.zig @@ -38,6 +38,7 @@ pub const GlobalState = struct { pub const CpuState = struct { pthread: c.pthread_t, + td: *ke.Thread, }; pub var global_state: GlobalState = undefined; @@ -122,6 +123,8 @@ fn other_cpu_entry(_: ?*anyopaque) callconv(.c) ?*anyopaque { ki.cpu.init_cpu(@intCast(my_cpu_id)); + ki.sched.percpu.local().current_thread = percpu.local().td; + timer.init_cpu(); while (true) {} @@ -174,6 +177,10 @@ pub fn late_init(_: *pl.BootInfo) linksection(r.init) void { @memcpy(block, @as([*]u8, @ptrCast(&__percpu_start))[0..size]); + const td = allocator.create(ke.Thread) catch @panic("oom"); + make_thread(idle, td); + percpu.remote(@intCast(i)).td = td; + var attr: c.pthread_attr_t = undefined; _ = c.pthread_attr_init(&attr);