Add turnstiles, more sched fixes

include/zinnia/mutex.h

@@ -4,7 +4,10 @@
 
 #include <stdint.h>
 
-typedef uintptr_t mutex_t;
+typedef _Atomic(uintptr_t) mutex_t;
 
 void mutex_lock(mutex_t* mutex);
+
+bool mutex_trylock(mutex_t* mutex);
+
 void mutex_unlock(mutex_t* mutex);

include/zinnia/sched.h

@@ -6,8 +6,10 @@
 #include <zinnia/errno.h>
 #include <zinnia/init.h>
 #include <zinnia/initgraph.h>
+#include <zinnia/spin.h>
 #include <zinnia/stack.h>
 #include <zinnia/tailq.h>
+#include <zinnia/timer.h>
 #include <zinnia/types.h>
 #include <bits/sched.h>
 #include <stddef.h>
@@ -23,8 +25,24 @@ enum task_state {
     TASK_STATE_DEAD,    // Task is killed.
 };
 
-#define TASK_NAME_MAX     128
-#define KERNEL_STACK_SIZE 0x4000
+enum sched_prio : uint8_t {
+    SCHED_PRIO_MIN = 0,
+
+    SCHED_PRIO_DEFAULT = 32,
+    SCHED_PRIO_WORK_HIGH = 48,
+    SCHED_PRIO_IRQ = 56,
+
+    SCHED_PRIO_MAX = 63,
+    SCHED_PRIO_COUNT = SCHED_PRIO_MAX + 1,
+};
+
+constexpr size_t SCHED_TIME_SLICE_NS = 10'000'000;
+constexpr uint8_t SCHED_FEEDBACK_DEPTH = 8;
+constexpr uint64_t SCHED_BOOST_INTERVAL_NS = 1'000'000'000;
+
+struct turnstile;
+
+constexpr size_t TASK_NAME_MAX = 128;
 
 // A task is the smallest unit of the scheduler.
 struct task {
@@ -35,11 +53,28 @@ struct task {
     struct arch_task_context context;
     struct usercopy_region* usercopy_region;
     struct stack kernel_stack;
-    uintptr_t user_stack;
+    uintptr_t user_stack; // Saved stack pointer from userspace.
     size_t time_slice;
-    int8_t priority;
+    uint8_t penalty;
+    uint64_t feedback_epoch;
     atomic_bool wake_pending;
 
+    struct spinlock lock;
+
+    atomic_bool on_cpu;                // Whether a CPU is still running on this task's context.
+    uint8_t base_prio;                 // Set by sched_set_priority().
+    uint8_t lent_prio;                 // Highest priority lent by tasks waiting on locks this task owns.
+    _Atomic(enum sched_prio) priority; // The effective priority of this task.
+    struct scheduler* last_rq;         // The most recent run queue this task was scheduled on.
+
+    struct turnstile* turnstile;             // Turnstile owned by this task.
+    _Atomic(struct turnstile*) blocked_ts;   // The turnstile this task is blocked on.
+    _Atomic(const void*) blocked_obj;        // The lock that `blocked_ts` belongs to.
+    tailq_head(struct turnstile) pi_lenders; // Turnstiles of locks this task owns.
+    tailq_link(struct task) ts_link;         // Link in the waiters of `blocked_ts`.
+    uint8_t ts_queue;
+    atomic_bool ts_woken;
+
     tailq_link(struct task) next;
 };
 
@@ -48,7 +83,24 @@ struct scheduler {
     struct task* current;
     struct task* idle_task;
     size_t preempt_level;
-    tailq_head(struct task) run_queue;
+
+    struct spinlock lock;
+
+    // Bit N is set if `run_queue[N]` is not empty.
+    uint64_t ready_mask;
+    tailq_head(struct task) run_queue[SCHED_PRIO_COUNT];
+
+    // Current priority, or negative if the CPU is idle.
+    _Atomic(int32_t) current_priority;
+
+    atomic_bool kick_pending;
+
+    struct scheduler* next_cpu;
+
+    struct timer preempt_timer;
+    bool need_resched;
+    uint64_t slice_start;
+    uint64_t boost_epoch;
 };
 
 typedef void (*task_fn_t)(uintptr_t arg0);
@@ -69,6 +121,16 @@ void sched_block(struct scheduler* sched);
 
 // Makes a blocked task runnable again, or marks the wake as pending if it hasn't blocked yet.
 void sched_wake(struct task* task);
+
+// Same as sched_wake(), but `task->lock` must be held.
+void sched_wake_locked(struct task* task);
+
+[[noreturn]]
+void sched_exit();
+
+// Sets the base priority of `task`.
+void sched_set_priority(struct task* task, uint8_t priority);
+
 [[noreturn]]
 void sched_to_user(uintptr_t ctx);
 [[noreturn]]

include/zinnia/timer.h

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/tailq.h>
+#include <stdint.h>
+
+struct timer_queue;
+
+struct timer {
+    uint64_t deadline;
+    void (*func)(struct timer* timer);
+    _Atomic(struct timer_queue*) queue;
+    atomic_bool running;
+    tailq_link(struct timer) link;
+};
+
+void timer_init(struct timer* timer, void (*func)(struct timer* timer));
+void timer_arm(struct timer* timer, uint64_t deadline_ns);
+bool timer_cancel(struct timer* timer);
+
+void timer_cpu_init();
+void timer_interrupt();

include/zinnia/turnstile.h

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/tailq.h>
+#include <stdint.h>
+
+struct task;
+struct turnstile_chain;
+
+enum turnstile_queue {
+    TURNSTILE_EXCLUSIVE,
+    TURNSTILE_SHARED,
+
+    TURNSTILE_NUM_QUEUES,
+};
+
+struct turnstile {
+    // The lock object that this turnstile is assigned to.
+    // If not in use, it's set to `nullptr`.
+    const void* obj;
+
+    // The task that inherits the priority from the waiters.
+    struct task* owner;
+
+    // Priority of the most urgent waiter in any queue.
+    _Atomic(uint8_t) top_priority;
+
+    tailq_head(struct task) waiters[TURNSTILE_NUM_QUEUES];
+
+    // Spare turnstiles donated by waiters.
+    struct turnstile* spare;
+
+    // Next active turnstile in the hash chain.
+    struct turnstile* chain_next;
+
+    // Link for `task::pi_lenders`.
+    tailq_link(struct turnstile) lender_link;
+};
+
+struct turnstile* turnstile_alloc();
+void turnstile_free(struct turnstile* ts);
+
+struct turnstile_chain* turnstile_lock(const void* obj);
+void turnstile_unlock(struct turnstile_chain* chain);
+
+void turnstile_block(struct turnstile_chain* chain, const void* obj, struct task* owner, enum turnstile_queue queue);
+
+struct task* turnstile_first(
+    struct turnstile_chain* chain,
+    const void* obj,
+    enum turnstile_queue queue,
+    bool* have_more
+);
+
+void turnstile_wake(struct turnstile_chain* chain, const void* obj, struct task* task, struct task* new_owner);
+
+void turnstile_reprioritize(struct task* task);