diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 0854447..8e189fb 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -99,25 +99,19 @@ pub const ThreadContext = struct { } }; -pub fn curcpu() *ke.Cpu { - const asm_template = std.fmt.comptimePrint("movq %%gs:{d}, %[ret]", .{@offsetOf(Cpu, "self_ptr")}); - const impl_cpu = asm volatile (asm_template - : [ret] "=r" (-> *Cpu), - ); +inline fn percpu_ptr_for(variable: anytype, cpu: u32) @TypeOf(variable) { + _ = cpu; // TODO + @panic("TODO percpu_ptr_for"); - return @fieldParentPtr("impl", impl_cpu); + // return @ptrFromInt(@intFromPtr(variable) +% cpu.impl.percpu_offset); } -inline fn percpu_ptr_for(variable: anytype, cpu: *ke.Cpu) @TypeOf(variable) { - return @ptrFromInt(@intFromPtr(variable) +% cpu.impl.percpu_offset); -} - -pub inline fn percpu_ptr_other(variable: anytype, id: usize) @TypeOf(variable) { - return percpu_ptr_for(variable, ke.cpus[id]); +pub inline fn percpu_ptr_other(variable: anytype, id: u32) @TypeOf(variable) { + return percpu_ptr_for(variable, id); } pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) { - return percpu_ptr_for(variable, curcpu()); + @panic("TODO percpu_ptr"); } pub inline fn set_hardware_ipl(ipl: ke.Ipl) void { @@ -147,6 +141,6 @@ pub inline fn restore_interrupts(state: bool) void { } } -pub inline fn send_resched_ipi(_: *ke.Cpu) void { +pub inline fn send_resched_ipi(_: u32) void { @panic("TODO send_resched_ipi"); } diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig index ee672a5..3317ff6 100644 --- a/src/kern/cpu.zig +++ b/src/kern/cpu.zig @@ -5,69 +5,33 @@ const config = @import("config"); const ke = b.ke; const ki = b.ke.private; -comptime { - if (!@hasDecl(ki.impl, "Cpu")) @compileError("impl must provide Cpu"); -} - -pub const PreemptionReason = enum(u8) { - None, - /// A higher priority thread was readied. - HigherPriority, -}; - extern var __init_array_percpu_start: u8; extern var __init_array_percpu_end: u8; -/// Global Per-CPU state. -pub const Cpu = struct { - /// Implementation-dependent CPU data. - impl: ki.impl.Cpu, - /// Unique ID of this CPU. - id: u32, - /// Current IPL on this CPU. - ipl: ke.Ipl, - /// Currently running thread. - current_thread: ?*ke.Thread, - /// Per-CPU idle thread. - idle_thread: ?*ke.Thread, - /// Thread selected for preemption. - next_thread: ?*ke.Thread, - /// The reason why the current thread was preempted. - preemption_reason: PreemptionReason, - /// Flag indicating the start of the scheduling timer. - start_timer: bool, - /// DPCs - resched_dpc: ke.Dpc, - resched_timer: ke.Timer, +const id = CpuLocal(u32, 0); +var curr_id: std.atomic.Value(u32) = .init(0); - /// Initialize `cpu` for usage with its idle thread being `thread`. - pub fn init(cpu: *Cpu, thread: *ke.Thread) void { - cpu.* = .{ - .id = cpu.id, - .ipl = .Passive, - .idle_thread = thread, - .current_thread = thread, - .resched_dpc = .init(ki.sched.clock), - .next_thread = null, - .preemption_reason = .None, - .start_timer = false, - .resched_timer = undefined, - .impl = cpu.impl, - }; +/// Initialize a CPU. Must be called on all CPUs. +pub fn init_cpu() void { + const start = @intFromPtr(&__init_array_percpu_start); + const end = @intFromPtr(&__init_array_percpu_end); + const count = (end - start) / @sizeOf(*const fn () void); + const funcs: [*]const *const fn () callconv(.c) void = @ptrFromInt(start); - // Call per-cpu init functions. - const start = @intFromPtr(&__init_array_percpu_start); - const end = @intFromPtr(&__init_array_percpu_end); - const count = (end - start) / @sizeOf(*const fn () void); - const funcs: [*]const *const fn () callconv(.c) void = @ptrFromInt(start); - - for (0..count) |i| { - funcs[i](); - } - - cpu.resched_timer.init(); + if (curr_id.load(.monotonic) >= config.CONFIG_NCPUS) { + std.debug.panic("cpu: Too many CPUs booted! Maximum supported is {}", .{config.CONFIG_NCPUS}); } -}; + + id.local().* = curr_id.fetchAdd(1, .monotonic); + + for (0..count) |i| { + funcs[i](); + } +} + +pub fn current() u32 { + return id.local().*; +} comptime { if (!@hasDecl(ki.impl, "percpu_ptr")) @compileError("impl must provide percpu_ptr()"); @@ -85,8 +49,8 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type { } /// Return a pointer to remote CPU data. - pub fn remote(cpu: *ke.Cpu) *T { - return ki.impl.percpu_ptr_other(&storage, cpu.id); + pub fn remote(cpu: u32) *T { + return ki.impl.percpu_ptr_other(&storage, cpu); } }; } diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index c81e923..7e3e32d 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -50,7 +50,7 @@ fn pcpu_init() linksection(b.init) callconv(.c) void { /// If the DPC is already enqueued, this function is a no-op. pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { const ipl = ke.ipl.raise(.High); - const cpu = ke.curcpu(); + const mycpu = ke.cpu.current(); const dpc_cpu = pcpu.local(); if (!dpc.inserted) { @@ -61,7 +61,7 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { dpc_cpu.queue.insert_tail(&dpc.link); // Mark the DPC as pending on this CPU - ki.ipl.set_softint_pending(cpu, .Dispatch); + ki.ipl.set_softint_pending(mycpu, .Dispatch); dpc.inserted = true; dpc_cpu.lock.release_no_ipl(); @@ -70,9 +70,12 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { ke.ipl.lower(ipl); } -fn dispatch_queue(cpu: *ke.Cpu) void { - cpu.ipl = .Dispatch; +fn dispatch_queue(cpu: u32) void { + const ipl_cpu = ki.ipl.percpu.remote(cpu); const dpc_cpu = pcpu.remote(cpu); + const sched_cpu = ki.sched.percpu.remote(cpu); + + ipl_cpu.ipl = .Dispatch; // Mark as handled, this must be done before enabling interrupts or we will race. ki.ipl.clear_softint_pending(cpu, .Dispatch); @@ -101,47 +104,46 @@ fn dispatch_queue(cpu: *ke.Cpu) void { const arg = dpc.arg; dpc_cpu.lock.release(ipl); - std.debug.assert(cpu.ipl == .Dispatch); + std.debug.assert(ke.ipl.current() == .Dispatch); dpc.func(arg); } - if (cpu.start_timer) { - ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc); + if (sched_cpu.start_timer) { + ke.timer.set(&sched_cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &sched_cpu.resched_dpc); } - if (cpu.preemption_reason == .HigherPriority) { + if (sched_cpu.preemption_reason == .HigherPriority) { // Reload the quantum for the new thread. - ke.timer.cancel(&cpu.resched_timer); - ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc); + ke.timer.cancel(&sched_cpu.resched_timer); + ke.timer.set(&sched_cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &sched_cpu.resched_dpc); } - cpu.start_timer = false; - cpu.preemption_reason = .None; + sched_cpu.start_timer = false; + sched_cpu.preemption_reason = .None; - if (cpu.next_thread != null) { - ki.sched.handle_preemption(cpu); + if (sched_cpu.next_thread != null) { + ki.sched.handle_preemption(sched_cpu); } _ = ki.impl.disable_interrupts(); } /// Dispatch the DPC queue on `cpu`. -pub fn dispatch(cpu: *ke.Cpu) void { +pub fn dispatch(cpu: u32) void { // DPC processing is done at Ipl.Dispatch // We can't call lower/raise here because they might call us again recursively. var mycpu = cpu; - const old_ipl = cpu.ipl; + const old_ipl = ki.ipl.percpu.remote(mycpu).ipl; const int_state = ki.impl.disable_interrupts(); while (ki.ipl.is_softint_pending(.Dispatch)) { dispatch_queue(mycpu); - // Our CPU might have changed. - mycpu = ke.curcpu(); + mycpu = ke.cpu.current(); } - cpu.ipl = old_ipl; + ki.ipl.percpu.remote(mycpu).ipl = old_ipl; ki.impl.restore_interrupts(int_state); } diff --git a/src/kern/init.zig b/src/kern/init.zig index 17c0236..6cfae56 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -11,7 +11,7 @@ var stack: [8192]u8 align(16) = undefined; pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { std.log.info("hello, world", .{}); pl.early_init(); - ki.bootstrap_cpu.init(&thread0); + ki.cpu.init_cpu(); thread0.init(@intFromPtr(&stack), 8192, ki.sched.idle, null); thread0.priority = 0; diff --git a/src/kern/ipl.zig b/src/kern/ipl.zig index 9ee6631..c84905e 100644 --- a/src/kern/ipl.zig +++ b/src/kern/ipl.zig @@ -19,15 +19,28 @@ pub const Ipl = enum(u8) { } }; -/// Bitmask of pending software interrupts on this CPU. -const pending_softints = ke.CpuLocal(std.atomic.Value(u8), .init(0)); +const PerCpu = struct { + /// Current IPL on this CPU. + ipl: Ipl, + /// Bitmask of pending software interrupts on this CPU. + pending_softints: std.atomic.Value(u8), +}; + +pub const percpu = ke.CpuLocal(PerCpu, .{ + .ipl = .Passive, + .pending_softints = std.atomic.Value(u8).init(0), +}); + +pub fn current() Ipl { + return percpu.local().ipl; +} /// Raise IPL to `new`. /// Panics if the current IPL is higher than requested. pub fn raise(new: Ipl) Ipl { const ints = ki.impl.disable_interrupts(); - var cpu = ke.curcpu(); + var cpu = percpu.local(); const old = cpu.ipl; if (@intFromEnum(new) < @intFromEnum(old)) { @@ -49,7 +62,8 @@ pub fn raise(new: Ipl) Ipl { /// Panics if the current IPL is lower than requested. pub fn lower(new: Ipl) void { const ints = ki.impl.disable_interrupts(); - var cpu = ke.curcpu(); + var cpu = percpu.local(); + const cpu_id = ke.cpu.current(); const old = cpu.ipl; if (@intFromEnum(new) > @intFromEnum(old)) { @@ -65,7 +79,7 @@ pub fn lower(new: Ipl) void { ki.impl.restore_interrupts(ints); if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(cpu); + ki.dpc.dispatch(cpu_id); } } @@ -75,7 +89,7 @@ comptime { /// Set the hardware IPL to `new`. pub fn set_hardware(new: Ipl) Ipl { - var cpu = ke.curcpu(); + var cpu = percpu.local(); const old = cpu.ipl; cpu.ipl = new; @@ -86,15 +100,15 @@ pub fn set_hardware(new: Ipl) Ipl { } /// Mark a software interrupt of IPL `ipl` on `cpu` as pending. -pub fn set_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void { +pub fn set_softint_pending(cpu: u32, ipl: Ipl) void { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); - _ = pending_softints.remote(cpu).bitSet(@intCast(@intFromEnum(ipl)), .monotonic); + _ = percpu.remote(cpu).pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic); } /// Mark a software interrupt of IPL `ipl` on `cpu` as handled. -pub fn clear_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void { +pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); - _ = pending_softints.remote(cpu).bitReset(@intCast(@intFromEnum(ipl)), .monotonic); + _ = percpu.remote(cpu).pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic); } /// Check whether a software interrupt of IPL `ipl` on the current CPU is pending. @@ -102,5 +116,5 @@ pub fn is_softint_pending(ipl: Ipl) bool { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); const shift: u3 = @intCast(@intFromEnum(ipl)); const bit: u8 = @as(u8, 1) << shift; - return pending_softints.local().load(.monotonic) & bit != 0; + return percpu.local().pending_softints.load(.monotonic) & bit != 0; } diff --git a/src/kern/private.zig b/src/kern/private.zig index 52a1369..f680827 100644 --- a/src/kern/private.zig +++ b/src/kern/private.zig @@ -15,7 +15,6 @@ else @compileError("unsupported architecture"); pub const init = init_mod.init; -pub var bootstrap_cpu: ke.Cpu = undefined; // === Exported Modules === pub const ipl = @import("ipl.zig"); diff --git a/src/kern/root.zig b/src/kern/root.zig index 1ae4dc3..8f08fed 100644 --- a/src/kern/root.zig +++ b/src/kern/root.zig @@ -13,7 +13,6 @@ pub const Dpc = p.dpc.Dpc; pub const TimeCounter = p.time.TimeCounter; pub const Timer = p.timer.Timer; pub const Console = p.log.Console; -pub const Cpu = p.cpu.Cpu; pub const CpuLocal = p.cpu.CpuLocal; pub const CpuMask = p.cpu.CpuMask; @@ -24,6 +23,7 @@ pub const ipl = struct { pub const raise = p.ipl.raise; pub const lower = p.ipl.lower; pub const set_hardware = p.ipl.set_hardware; + pub const current = p.ipl.current; }; pub const thread = struct { @@ -61,15 +61,9 @@ pub const log = struct { pub const log = p.log.log; }; -comptime { - if (!@hasDecl(p.impl, "curcpu")) @compileError("impl must provide curcpu()"); -} - -/// Return the current CPU -pub const curcpu: fn () *Cpu = p.impl.curcpu; +pub const cpu = struct { + pub const current = p.cpu.current; +}; /// Number of CPUs on the system pub var ncpus: usize = 0; - -/// Array of CPUs -pub var cpus: []*Cpu = undefined; diff --git a/src/kern/sched.zig b/src/kern/sched.zig index 4575576..fbbb8d0 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -116,6 +116,12 @@ const ThreadStatus = packed struct(u8) { priority: u7, }; +pub const PreemptionReason = enum(u8) { + None, + /// A higher priority thread was readied. + HigherPriority, +}; + /// Per-CPU scheduler data. pub const PerCpu = struct { /// Current load on this CPU (number of ready threads) @@ -137,33 +143,46 @@ pub const PerCpu = struct { /// Atomic byte for getting the current thread's status locklessly. /// 1 bit for thread interactivity, 7 for priority. current_thread_status: std.atomic.Value(ThreadStatus), + /// DPC for rescheduling. + resched_dpc: ke.Dpc, + /// Timer for quantum expiration. + resched_timer: ke.Timer, + /// Flag indicating the start of the scheduling timer. + start_timer: bool, + /// The reason why the current thread was preempted. + preemption_reason: PreemptionReason, + /// Currently running thread. + current_thread: ?*ke.Thread, + /// Per-CPU idle thread. + idle_thread: ?*ke.Thread, + /// Thread selected for preemption. + next_thread: ?*ke.Thread, }; -const percpu_data = ke.CpuLocal(PerCpu, undefined); +pub const percpu = ke.CpuLocal(PerCpu, undefined); /// Handle preemption. /// This is called in DPC dispatch when it notices `next_thread` is set. -pub fn handle_preemption(cpu: *ke.Cpu) void { - const sched_cpu = percpu_data.remote(cpu); - sched_cpu.queues_lock.acquire_no_ipl(); +pub fn handle_preemption(cpu: *PerCpu) void { + cpu.queues_lock.acquire_no_ipl(); const cur = cpu.current_thread.?; const next = cpu.next_thread orelse { - sched_cpu.queues_lock.release_no_ipl(); + cpu.queues_lock.release_no_ipl(); return; }; cpu.next_thread = null; - sched_cpu.queues_lock.release_no_ipl(); + cpu.queues_lock.release_no_ipl(); cur.lock.acquire_no_ipl(); if (cur != cpu.idle_thread and cur.state != .Blocked) { // Put it back in the queue. - sched_cpu.queues_lock.acquire_no_ipl(); - insert_in_queue(sched_cpu, cur, false); - sched_cpu.queues_lock.release_no_ipl(); + cpu.queues_lock.acquire_no_ipl(); + insert_in_queue(cpu, cur, false); + cpu.queues_lock.release_no_ipl(); } do_switch(cpu, cur, next); @@ -173,22 +192,21 @@ pub fn handle_preemption(cpu: *ke.Cpu) void { /// Called every time slice in DPC context. pub fn clock(_: ?*anyopaque) void { - const cpu = ke.curcpu(); - const sched_cpu = percpu_data.remote(cpu); - std.debug.assert(cpu.ipl == .Dispatch); + const cpu = percpu.local(); + std.debug.assert(ki.ipl.current() == .Dispatch); const curtd = cpu.current_thread orelse return; curtd.lock.acquire_no_ipl(); defer curtd.lock.release_no_ipl(); - sched_cpu.queues_lock.acquire_no_ipl(); - defer sched_cpu.queues_lock.release_no_ipl(); + cpu.queues_lock.acquire_no_ipl(); + defer cpu.queues_lock.release_no_ipl(); // Advance the insert index every tick, while keeping a separation of 1 with runidx. // This ensures fairness. - if (sched_cpu.runidx == sched_cpu.insidx) { - sched_cpu.insidx = (sched_cpu.insidx + 1) % runqueues_n; + if (cpu.runidx == cpu.insidx) { + cpu.insidx = (cpu.insidx + 1) % runqueues_n; } if (curtd.priority_class == .Batch) { @@ -223,7 +241,7 @@ pub fn enqueue(td: *ke.Thread) void { /// Block the currently running thread. pub fn block() void { const ipl = ke.ipl.raise(.Dispatch); - const td = ke.curcpu().current_thread.?; + const td = percpu.local().current_thread.?; td.lock.acquire_no_ipl(); @@ -231,7 +249,7 @@ pub fn block() void { td.sleep_start = ke.time.read_time_nano(); td.runq = null; - yield(ke.curcpu()); + yield(ke.cpu.current()); // td lock released ke.ipl.lower(ipl); @@ -260,9 +278,9 @@ export const sched_percpu_init linksection(b.percpu_init) = &init_cpu; /// Initialize a CPU for use by the scheduler. fn init_cpu() linksection(b.init) callconv(.c) void { - var percpu = percpu_data.local(); + var cpu = percpu.local(); - percpu.* = .{ + cpu.* = .{ .insidx = 0, .runidx = 0, .calendar_queue = .{ .status = 0, .queues = undefined }, @@ -272,16 +290,24 @@ fn init_cpu() linksection(b.init) callconv(.c) void { .steal_work = false, .load = .init(0), .current_thread_status = .init(.{ .interactive = false, .priority = 0 }), + .resched_dpc = .init(ki.sched.clock), + .resched_timer = undefined, + .start_timer = false, + .preemption_reason = .None, + .current_thread = null, + .idle_thread = null, + .next_thread = null, }; - percpu.idle_queue.init(); + cpu.resched_timer.init(); + cpu.idle_queue.init(); for (0..runqueues_n) |i| { - percpu.calendar_queue.queues[i].init(); + cpu.calendar_queue.queues[i].init(); } for (0..runqueues_n) |i| { - percpu.realtime_queue.queues[i].init(); + cpu.realtime_queue.queues[i].init(); } } @@ -623,18 +649,17 @@ fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool { // Return whether or not `td` should preempt the thread running on `cpu`. // This is done in a lock-free manner. -fn should_preempt_cpu(td: *ke.Thread, cpu: *ke.Cpu) bool { - const status = percpu_data.remote(cpu).current_thread_status.load(.monotonic); +fn should_preempt_cpu(td: *ke.Thread, cpu: *PerCpu) bool { + const status = cpu.current_thread_status.load(.monotonic); return should_prio_preempt(td, status.priority, status.interactive); } // Enqueue a thread on a CPU. -fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void { - var c = percpu_data.remote(cpu); +fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { + const cpu = percpu.remote(c); + cpu.queues_lock.acquire_no_ipl(); - c.queues_lock.acquire_no_ipl(); - - defer c.queues_lock.release_no_ipl(); + defer cpu.queues_lock.release_no_ipl(); if (cpu.current_thread == null or should_preempt(td, cpu.current_thread.?)) { // We can preempt the current thread, first check whether or not there @@ -647,7 +672,7 @@ fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void { if (should_preempt(td, n)) { // This thread can preempt the next thread, // put the next thread back into a queue as the first element. - insert_in_queue(c, n, true); + insert_in_queue(cpu, n, true); } else { // The next thread is higher priority, we can't preempt it. can_preempt = false; @@ -659,11 +684,11 @@ fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void { cpu.next_thread = td; cpu.preemption_reason = .HigherPriority; - ki.ipl.set_softint_pending(cpu, .Dispatch); + ki.ipl.set_softint_pending(c, .Dispatch); - if (cpu != ke.curcpu()) { + if (c != ke.cpu.current()) { // Send an IPI. - ki.impl.send_resched_ipi(cpu); + ki.impl.send_resched_ipi(c); } // queues_lock dropped @@ -673,41 +698,42 @@ fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void { // Normal enqueue path. // Add the thread to its appropriate queue. - insert_in_queue(c, td, false); + insert_in_queue(cpu, td, false); if (cpu.resched_timer.state.load(.monotonic) == .Stopped) { cpu.start_timer = true; - ki.ipl.set_softint_pending(cpu, .Dispatch); + ki.ipl.set_softint_pending(c, .Dispatch); } // queues_lock dropped } // Find a suitable CPU to run the thread. -fn pick_cpu(td: *ke.Thread) *ke.Cpu { +fn pick_cpu(td: *ke.Thread) u32 { // CPU selection policy in order: // 1. Pick the last CPU the thread ran on if it would run immediately or if it's pinned. // 2. Pick the least loaded CPU on which the thread can run immediately. // 3. Pick the least loaded CPU overall. if (td.last_cpu) |cpu| { - if (should_preempt_cpu(td, cpu) or td.pinned) return cpu; + const sched_cpu = percpu.remote(cpu); + if (should_preempt_cpu(td, sched_cpu) or td.pinned) return cpu; } // Check all CPUs, keeping track of the least loaded one overall and the least loaded preemptible one. - var least: ?*ke.Cpu = null; - var least_preempt: ?*ke.Cpu = null; + var least: ?u32 = null; + var least_preempt: ?u32 = null; for (0..ke.ncpus) |i| { - const cpu = ke.cpus[i]; - const data = percpu_data.remote(cpu); + const cpu: u32 = @truncate(i); + const data = percpu.remote(cpu); - if (least == null or data.load.load(.monotonic) < percpu_data.remote(least.?).load.load(.monotonic)) { + if (least == null or data.load.load(.monotonic) < percpu.remote(least.?).load.load(.monotonic)) { least = cpu; } - if (should_preempt_cpu(td, cpu)) { - if (least_preempt == null or data.load.load(.monotonic) < percpu_data.remote(least_preempt.?).load.load(.monotonic)) { + if (should_preempt_cpu(td, data)) { + if (least_preempt == null or data.load.load(.monotonic) < percpu.remote(least_preempt.?).load.load(.monotonic)) { least_preempt = cpu; } } @@ -718,12 +744,10 @@ fn pick_cpu(td: *ke.Thread) *ke.Cpu { } // Find a thread to preempt `cur` with. -fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread { - const sched_cpu = percpu_data.remote(cpu); - +fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, migrate: bool) ?*ke.Thread { // First try to pick from the realtime queues. - if (pick_realtime_thread(sched_cpu, if (cur) |c| c.priority else 0, migrate)) |td| { - _ = sched_cpu.load.fetchSub(1, .monotonic); + if (pick_realtime_thread(cpu, if (cur) |c| c.priority else 0, migrate)) |td| { + _ = cpu.load.fetchSub(1, .monotonic); return td; } @@ -733,8 +757,8 @@ fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread { } // Then try picking a batch thread. - if (pick_batch_thread(sched_cpu, migrate)) |td| { - _ = sched_cpu.load.fetchSub(1, .monotonic); + if (pick_batch_thread(cpu, migrate)) |td| { + _ = cpu.load.fetchSub(1, .monotonic); return td; } @@ -744,8 +768,8 @@ fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread { } // Finally try picking from the idle queue. - if (pick_idle_thread(sched_cpu, migrate)) |td| { - _ = sched_cpu.load.fetchSub(1, .monotonic); + if (pick_idle_thread(cpu, migrate)) |td| { + _ = cpu.load.fetchSub(1, .monotonic); return td; } @@ -754,21 +778,21 @@ fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread { } // Execute a context switch on `cpu`. -fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void { +fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { next.state = .Running; cpu.current_thread = next; - next.last_cpu = cpu; + next.last_cpu = ke.cpu.current(); // Stash thread status. - percpu_data.remote(cpu).current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic); + cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic); // Do machine-dependent switch. cur.context.switch_to(&next.context); } // Yield on `cpu` -fn yield(cpu: *ke.Cpu) void { - const sched_cpu = percpu_data.remote(cpu); +fn yield(cpu: u32) void { + const sched_cpu = percpu.remote(cpu); sched_cpu.queues_lock.acquire_no_ipl(); @@ -796,8 +820,8 @@ fn yield(cpu: *ke.Cpu) void { // cur lock dropped } -fn find_most_loaded(exclude: ?*ke.CpuMask) ?*ke.Cpu { - var most: ?*ke.Cpu = null; +fn find_most_loaded(exclude: ?*ke.CpuMask) ?u32 { + var most: ?u32 = null; var most_load: usize = 0; var equal = true; @@ -808,14 +832,14 @@ fn find_most_loaded(exclude: ?*ke.CpuMask) ?*ke.Cpu { } } - const cpu = ke.cpus[i]; - const load = percpu_data.remote(cpu).load.load(.monotonic); + const cpu = percpu.remote(@truncate(i)); + const load = cpu.load.load(.monotonic); if (most == null) { - most = cpu; + most = @truncate(i); most_load = load; } else if (load > most_load) { - most = cpu; + most = @truncate(i); most_load = load; equal = false; } else if (load < most_load) { @@ -831,8 +855,8 @@ fn find_most_loaded(exclude: ?*ke.CpuMask) ?*ke.Cpu { return most; } -fn find_least_loaded(exclude: ?*ke.CpuMask) ?*ke.Cpu { - var least: ?*ke.Cpu = null; +fn find_least_loaded(exclude: ?*ke.CpuMask) ?u32 { + var least: ?u32 = null; var least_load: usize = 0; var equal = true; @@ -843,14 +867,14 @@ fn find_least_loaded(exclude: ?*ke.CpuMask) ?*ke.Cpu { } } - const cpu = ke.cpus[i]; - const load = percpu_data.remote(cpu).load.load(.monotonic); + const cpu = percpu.remote(@truncate(i)); + const load = cpu.load.load(.monotonic); if (least == null) { - least = cpu; + least = @truncate(i); least_load = load; } else if (load < least_load) { - least = cpu; + least = @truncate(i); least_load = load; equal = false; } else if (load > least_load) { @@ -866,18 +890,17 @@ fn find_least_loaded(exclude: ?*ke.CpuMask) ?*ke.Cpu { return least; } -fn steal_thread_from_cpu(cpu: *ke.Cpu) ?*ke.Thread { - const sched_cpu = percpu_data.remote(cpu); - sched_cpu.queues_lock.acquire_no_ipl(); +fn steal_thread_from_cpu(cpu: *PerCpu) ?*ke.Thread { + cpu.queues_lock.acquire_no_ipl(); const td = select_thread(null, cpu, true); - sched_cpu.queues_lock.release_no_ipl(); + cpu.queues_lock.release_no_ipl(); return td; } -fn steal_work(cpu: *ke.Cpu) ?*ke.Thread { +fn steal_work(cpu: u32) ?*ke.Thread { const most = find_most_loaded(null); if (most == cpu) { @@ -886,11 +909,12 @@ fn steal_work(cpu: *ke.Cpu) ?*ke.Thread { } if (most) |other| { - if (percpu_data.remote(other).load.load(.monotonic) < steal_threshold) { + const sched_other = percpu.remote(other); + if (sched_other.load.load(.monotonic) < steal_threshold) { return null; } - return steal_thread_from_cpu(other); + return steal_thread_from_cpu(sched_other); } return null; @@ -898,21 +922,20 @@ fn steal_work(cpu: *ke.Cpu) ?*ke.Thread { /// Idle loop for a CPU. Must be set up in the idle thread. pub fn idle(_: ?*anyopaque) noreturn { - const cpu = ke.curcpu(); - while (true) { - const percpu = percpu_data.local(); + const cpu = percpu.local(); + const cpu_id = ke.cpu.current(); - if (percpu.steal_work) { + if (cpu.steal_work) { // When first going idle, try stealing work from the most loaded CPU. - percpu.steal_work = false; + cpu.steal_work = false; const ipl = ke.ipl.raise(.Dispatch); // Steal a thread. - if (steal_work(cpu)) |td| { + if (steal_work(cpu_id)) |td| { // Enqueue it on this CPU. - enqueue_on_cpu(cpu, td); + enqueue_on_cpu(cpu_id, td); } ke.ipl.lower(ipl); @@ -931,13 +954,13 @@ fn balance(_: ?*anyopaque) void { while (true) { const high = find_most_loaded(&high_mask); - if (high == null or percpu_data.remote(high.?).load.load(.monotonic) == 0) { + if (high == null or percpu.remote(high.?).load.load(.monotonic) == 0) { // No highest loaded CPU. break; } // Don't steal from this CPU again. - high_mask.set(high.?.id); + high_mask.set(high.?); low_mask = high_mask; if (high_mask.is_full()) { @@ -953,7 +976,8 @@ fn balance(_: ?*anyopaque) void { } // Steal a thread from the high CPU and put it on the low CPU. - const td = steal_thread_from_cpu(high.?); + const sched_high = percpu.remote(high.?); + const td = steal_thread_from_cpu(sched_high); if (td) |thread| { thread.lock.acquire_no_ipl(); @@ -962,7 +986,7 @@ fn balance(_: ?*anyopaque) void { } // Don't shuffle threads around. - high_mask.set(low.?.id); + high_mask.set(low.?); } const offset = lcg.next() % balance_interval; diff --git a/src/kern/thread.zig b/src/kern/thread.zig index 94ccf8e..45f1bac 100644 --- a/src/kern/thread.zig +++ b/src/kern/thread.zig @@ -77,8 +77,7 @@ pub const Thread = struct { /// Linkage into a scheduler run queue. runq_link: rtl.List.Entry, /// Last CPU this thread ran on. - last_cpu: ?*ke.Cpu, - + last_cpu: ?u32, /// Run queue this thread is in runq: ?*ki.sched.RunQueue, /// Index into the run queue this thread is currently in. diff --git a/src/kern/timer.zig b/src/kern/timer.zig index f2c5bef..5eb1f38 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -139,7 +139,7 @@ pub fn clock() void { // Called in a DPC when a timer has expired. fn handle_expiry(_: ?*anyopaque) void { - std.debug.assert(ke.curcpu().ipl == .Dispatch); + std.debug.assert(ki.ipl.current() == .Dispatch); const cpu = percpu.local(); while (true) { diff --git a/src/kern/um/impl.zig b/src/kern/um/impl.zig index fda9811..cb73c77 100644 --- a/src/kern/um/impl.zig +++ b/src/kern/um/impl.zig @@ -4,13 +4,6 @@ const c = b.pl.impl.c; const ke = b.ke; const ki = ke.private; -pub const Cpu = struct { pthread: c.pthread_t }; - -/// Return the current CPU. -pub fn curcpu() *ke.Cpu { - return b.pl.impl.my_cpu; -} - fn set_signals(enabled: bool) bool { var newmask: c.sigset_t = undefined; var oldmask: c.sigset_t = undefined; @@ -26,10 +19,12 @@ fn set_signals(enabled: bool) bool { return c.sigismember(&oldmask, c.SIGUSR1) == 0; } +threadlocal var hard_ipl: ke.Ipl = .Passive; + /// Set the hardware IPL. pub fn set_hardware_ipl(ipl: ke.Ipl) void { // This is a no-op since we handle this in enable_interrupts(). - _ = ipl; + hard_ipl = ipl; } /// Disable interrupts and return the previous state. @@ -39,18 +34,18 @@ pub fn disable_interrupts() bool { /// Enable interrupts and return the previous state. pub fn enable_interrupts() bool { - const ipl_allows_ints = @intFromEnum(ke.curcpu().ipl) <= @intFromEnum(ke.Ipl.get_max_software()); + const ipl_allows_ints = @intFromEnum(hard_ipl) <= @intFromEnum(ke.Ipl.get_max_software()); return set_signals(ipl_allows_ints); } /// Restore an interrupt state. pub fn restore_interrupts(val: bool) void { - const ipl_allows_ints = @intFromEnum(ke.curcpu().ipl) <= @intFromEnum(ke.Ipl.get_max_software()); + const ipl_allows_ints = @intFromEnum(hard_ipl) <= @intFromEnum(ke.Ipl.get_max_software()); _ = set_signals(val and ipl_allows_ints); } -pub fn send_resched_ipi(cpu: *ke.Cpu) void { - _ = c.pthread_kill(cpu.impl.pthread, c.SIGUSR1); +pub fn send_resched_ipi(cpu: u32) void { + _ = c.pthread_kill(b.pl.impl.percpu.remote(cpu).pthread, c.SIGUSR1); } pub const ThreadContext = struct { @@ -109,5 +104,5 @@ pub inline fn percpu_ptr_other(variable: anytype, cpu_id: usize) @TypeOf(variabl } pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) { - return @ptrFromInt(@intFromPtr(variable) +% b.pl.impl.cpu_offsets[b.pl.impl.my_cpu.id]); + return @ptrFromInt(@intFromPtr(variable) +% b.pl.impl.cpu_offsets[b.pl.impl.my_cpu_id]); } diff --git a/src/platform/amd64/impl.zig b/src/platform/amd64/impl.zig index dde1590..108322d 100644 --- a/src/platform/amd64/impl.zig +++ b/src/platform/amd64/impl.zig @@ -27,18 +27,6 @@ fn com1_write(c: u8) void { pub fn early_init() linksection(b.init) void { com1_init(); - const impl_cpu = &ki.bootstrap_cpu.impl; - impl_cpu.self_ptr = impl_cpu; - impl_cpu.percpu_offset = 0; - - const impl_cpu_ptr = @intFromPtr(impl_cpu); - asm volatile ("wrmsr" - : - : [_] "{eax}" (@as(u32, @intCast(impl_cpu_ptr & 0xFFFFFFFF))), - [_] "{edx}" (@as(u32, @truncate(impl_cpu_ptr >> 32))), - [_] "{ecx}" (0xC000_0101), - ); - cpu.detect_cpu_features(); const f = cpu.cpu_features; diff --git a/src/platform/boot/limine/entry.zig b/src/platform/boot/limine/entry.zig index 0d6da95..48cff08 100644 --- a/src/platform/boot/limine/entry.zig +++ b/src/platform/boot/limine/entry.zig @@ -87,10 +87,7 @@ export fn kmain() callconv(.c) void { build_memory_map(); - var cpus: [1]*ke.Cpu = .{&ke.private.bootstrap_cpu}; - ke.ncpus = 1; - ke.cpus = &cpus; ki.init(&boot_info); } diff --git a/src/platform/um/impl.zig b/src/platform/um/impl.zig index 5e965b9..3d7c260 100644 --- a/src/platform/um/impl.zig +++ b/src/platform/um/impl.zig @@ -34,8 +34,13 @@ pub const GlobalState = struct { phys_base: usize, }; +pub const CpuState = struct { + pthread: c.pthread_t, +}; + pub var global_state: GlobalState = undefined; -pub threadlocal var my_cpu: *ke.Cpu = undefined; +pub threadlocal var my_cpu_id: usize = 0; +pub const percpu = ke.CpuLocal(CpuState, undefined); pub fn debug_write(char: u8) void { _ = linux.write(linux.STDERR_FILENO, @ptrCast(&char), 1); @@ -62,12 +67,8 @@ pub fn early_init() linksection(b.init) void { // Ensure we can use per-cpu data. cpu_offsets = @constCast(&bootstrap_offsets); - // Set current CPU to the bootstrap CPU. - my_cpu = &ki.bootstrap_cpu; - - my_cpu.impl.pthread = c.pthread_self(); - - my_cpu.id = 0; + percpu.local().pthread = c.pthread_self(); + my_cpu_id = 0; const memfd: i32 = @intCast(linux.memfd_create("phys_memory", 0)); @@ -101,8 +102,8 @@ pub fn early_init() linksection(b.init) void { } fn sigusr1_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void { - if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(ke.curcpu()); + if (@intFromEnum(ki.ipl.current()) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) { + ki.dpc.dispatch(ke.cpu.current()); } } @@ -117,17 +118,15 @@ fn make_thread(entry: *const fn (?*anyopaque) void, td: *ke.Thread) void { var cpus_up = std.atomic.Value(usize).init(0); -fn other_cpu_entry(arg: ?*anyopaque) callconv(.c) ?*anyopaque { - my_cpu = @ptrCast(@alignCast(arg.?)); +fn other_cpu_entry(_: ?*anyopaque) callconv(.c) ?*anyopaque { + my_cpu_id = cpus_up.fetchAdd(1, .monotonic) + 1; - my_cpu.impl.pthread = c.pthread_self(); + percpu.local().pthread = c.pthread_self(); - my_cpu.init(my_cpu.idle_thread.?); + ki.cpu.init_cpu(); timer.init_cpu(); - _ = cpus_up.fetchAdd(1, .monotonic); - while (true) {} return null; @@ -167,9 +166,7 @@ pub fn late_init() linksection(b.init) void { var other_threads = allocator.alloc(c.pthread_t, other_count) catch @panic("oom"); - ke.cpus = allocator.alloc(*ke.Cpu, ke.ncpus) catch @panic("oom"); cpu_offsets = allocator.alloc(usize, ke.ncpus) catch @panic("oom"); - ke.cpus[0] = &ki.bootstrap_cpu; cpu_offsets[0] = 0; const size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); @@ -178,15 +175,8 @@ pub fn late_init() linksection(b.init) void { const block = allocator.alloc(u8, size) catch @panic("oom"); cpu_offsets[i] = @intFromPtr(block.ptr) -% @intFromPtr(&__percpu_start); - ke.cpus[i] = ki.impl.percpu_ptr_other(&cpu, i); - @memcpy(block, @as([*]u8, @ptrCast(&__percpu_start))[0..size]); - ke.cpus[i].idle_thread = allocator.create(ke.Thread) catch @panic("oom"); - ke.cpus[i].id = @intCast(i); - - make_thread(idle, ke.cpus[i].idle_thread.?); - var attr: c.pthread_attr_t = undefined; _ = c.pthread_attr_init(&attr); @@ -199,7 +189,7 @@ pub fn late_init() linksection(b.init) void { &other_threads[i - 1], &attr, other_cpu_entry, - ke.cpus[i], + null, ); if (r != 0) { diff --git a/src/platform/um/timer.zig b/src/platform/um/timer.zig index 5330b94..a4a6971 100644 --- a/src/platform/um/timer.zig +++ b/src/platform/um/timer.zig @@ -16,7 +16,7 @@ fn timer_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callco _ = ke.ipl.set_hardware(old_ipl); if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(ke.curcpu()); + ki.dpc.dispatch(ke.cpu.current()); } }