irq: Add irq_tasks, move NVMe to it

arch/x86_64/idt.c

@@ -103,12 +103,10 @@ void interrupt_handler(struct arch_context* ctx) {
         break;
     case IDT_TIMER:
         lapic_eoi(&arch_cpu->lapic);
+        arch_sched_preempt_disable();
         timer_interrupt();
-        if (cpu->sched.need_resched) {
-            arch_sched_preempt_disable();
-            if (arch_sched_preempt_enable())
-                sched_reschedule(&cpu->sched);
-        }
+        if (arch_sched_preempt_enable() && (cpu->sched.need_resched || cpu->sched.owe_preempt))
+            sched_reschedule(&cpu->sched);
         break;
     default:
         if (isr < 0x20)
@@ -116,7 +114,10 @@ void interrupt_handler(struct arch_context* ctx) {
         assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
         struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
         assert(line, "Unhandled interrupt on ISR %lu", isr);
+        arch_sched_preempt_disable();
         irq_line_raise(line);
+        if (arch_sched_preempt_enable() && cpu->sched.owe_preempt)
+            sched_reschedule(&cpu->sched);
     }
 }
 

dev/acpi/uacpi_glue.c

@@ -253,7 +253,6 @@ uacpi_status uacpi_kernel_install_interrupt_handler(
     if (!acpi_irq)
         return UACPI_STATUS_OUT_OF_MEMORY;
     acpi_irq->handler.raise = acpi_irq_raise;
-    acpi_irq->handler.line = line;
     acpi_irq->func = handler;
     acpi_irq->ctx = ctx;
 

dev/block/nvme/pci.c

@@ -23,12 +23,11 @@ struct nvme_irq_handler {
     struct nvme_ctrl* controller;
 };
 
-static enum irq_status nvme_irq_raise(struct irq_handler* self) {
+static void nvme_irq_thread(struct irq_handler* self) {
     struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler);
     struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller);
     if (queue)
         nvme_io_queue_drain(queue);
-    return IRQ_STATUS_HANDLED;
 }
 
 static errno_t map_bar(struct pci_bar* bar, void** out) {
@@ -84,9 +83,16 @@ static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* ma
         struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0);
         if (!handler)
             return ENOMEM;
-        handler->handler.raise = nvme_irq_raise;
+        handler->handler.thread = nvme_irq_thread;
         handler->controller = controller;
-        irq_line_attach(irq_line, &handler->handler);
+
+        char name[32];
+        ksnprintf(name, sizeof(name), "nvme-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
+        status = irq_line_attach_task(irq_line, &handler->handler, name);
+        if (status) {
+            mem_free(handler);
+            return status;
+        }
         if (irq_line->unmask)
             irq_line->unmask(irq_line);
     }

include/zinnia/irq.h

@@ -5,9 +5,7 @@
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
 #include <zinnia/list.h>
-#include <zinnia/spin.h>
 #include <zinnia/types.h>
-#include <zinnia/workqueue.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -33,6 +31,7 @@ enum irq_mode : uint8_t {
 };
 
 struct irq_line;
+struct irq_task;
 
 struct irq_handler {
     enum irq_status (*raise)(struct irq_handler* self);
@@ -40,9 +39,7 @@ struct irq_handler {
     list_link(struct irq_handler*) next;
 
     struct irq_line* line;
-    struct workqueue* wq;
-    struct work work;
-    bool holds_mask;
+    _Atomic(bool) need;
 };
 
 struct irq_line {
@@ -55,8 +52,7 @@ struct irq_line {
     list_head(struct irq_handler*) handlers;
     enum irq_mode mode;
     enum irq_trigger_mode trigger;
-    struct spinlock thread_lock;
-    size_t threads_pending;
+    struct irq_task* task;
 };
 
 struct msi_line {
@@ -76,7 +72,7 @@ void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum
 void irq_line_raise(struct irq_line* line);
 void irq_line_attach(struct irq_line* line, struct irq_handler* handler);
 void irq_line_detach(struct irq_line* line, struct irq_handler* handler);
-errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name);
+errno_t irq_line_attach_task(struct irq_line* line, struct irq_handler* handler, const char* name);
 
 void irq_lock();
 void irq_unlock();

include/zinnia/sched.h

@@ -102,6 +102,7 @@ struct scheduler {
 
     struct timer preempt_timer;
     bool need_resched;
+    bool owe_preempt;
     uint64_t slice_start;
     uint64_t boost_epoch;
 };

lib/irq.c

@@ -1,10 +1,14 @@
 // SPDX-License-Identifier: ISC
 
+#include <zinnia/alloc.h>
 #include <zinnia/assert.h>
+#include <zinnia/atomic.h>
 #include <zinnia/irq.h>
 #include <zinnia/list.h>
+#include <zinnia/mutex.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/process.h>
 #include <zinnia/sched.h>
 #include <zinnia/utils.h>
 #include <arch/cpu.h>
@@ -34,8 +38,12 @@ void irq_lock() {
 
 void irq_unlock() {
     struct irq_percpu* state = percpu_ref(irq_data);
-    if (--state->depth == 0 && state->saved_state)
+    if (--state->depth == 0 && state->saved_state) {
+        struct scheduler* sched = &percpu_get()->sched;
+        if (sched->owe_preempt && !sched->preempt_level && sched->current)
+            sched_reschedule(sched);
         arch_irq_set_state(true);
+    }
 }
 
 void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
@@ -45,37 +53,47 @@ void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum
     irq_unlock();
 }
 
-static bool holds_mask_while_threaded(struct irq_line* line) {
+struct irq_task {
+    struct irq_line* line;
+    struct task* task;
+    _Atomic(bool) need;
+    _Atomic(bool) masked;
+    _Atomic(bool) running;
+};
+
+static mutex_t irq_task_lock;
+
+static bool holds_mask(struct irq_line* line) {
     return line->trigger == IRQ_TRIGGER_LEVEL && (line->mode & IRQ_MODE_MASKABLE) && line->mask && line->unmask;
 }
 
-static void wake_thread(struct irq_line* line, struct irq_handler* handler) {
-    if (holds_mask_while_threaded(line)) {
-        spin_lock(&line->thread_lock);
-        if (!handler->holds_mask) {
-            handler->holds_mask = true;
-            if (line->threads_pending++ == 0)
-                line->mask(line);
-        }
-        spin_unlock(&line->thread_lock);
-    }
+static void irq_task_fn(uintptr_t arg) {
+    struct irq_task* it = (struct irq_task*)arg;
+    struct irq_line* line = it->line;
 
-    work_queue(handler->wq, &handler->work);
+    while (true) {
+        atomic_store(&it->running, true, ATOMIC_SEQ_CST);
+        while (atomic_exchange(&it->need, false, ATOMIC_ACQ_REL)) {
+            struct irq_handler* handler;
+            list_foreach(handler, &line->handlers, next) {
+                if (atomic_exchange(&handler->need, false, ATOMIC_ACQ_REL))
+                    handler->thread(handler);
+            }
+            if (atomic_exchange(&it->masked, false, ATOMIC_ACQ_REL))
+                line->unmask(line);
+        }
+        atomic_store(&it->running, false, ATOMIC_SEQ_CST);
+
+        if (!atomic_load(&it->need, ATOMIC_ACQUIRE))
+            sched_block(&percpu_get()->sched);
+    }
 }
 
-static void irq_thread_work(struct work* work) {
-    struct irq_handler* handler = containerof(work, struct irq_handler, work);
-    struct irq_line* line = handler->line;
-
-    handler->thread(handler);
-
-    spin_lock(&line->thread_lock);
-    if (handler->holds_mask) {
-        handler->holds_mask = false;
-        if (--line->threads_pending == 0)
-            line->unmask(line);
-    }
-    spin_unlock(&line->thread_lock);
+static void irq_task_schedule(struct irq_task* it) {
+    if (holds_mask(it->line) && !atomic_exchange(&it->masked, true, ATOMIC_ACQ_REL))
+        it->line->mask(it->line);
+    atomic_store(&it->need, true, ATOMIC_RELEASE);
+    sched_wake(it->task);
 }
 
 void irq_line_raise(struct irq_line* line) {
@@ -85,17 +103,21 @@ void irq_line_raise(struct irq_line* line) {
     line->is_busy = true;
 
     bool claimed = false;
+    bool wake = false;
     struct irq_handler* handler;
     list_foreach(handler, &line->handlers, next) {
         enum irq_status status = handler->raise ? handler->raise(handler) : IRQ_STATUS_WAKE_THREAD;
-        if (status == IRQ_STATUS_WAKE_THREAD && handler->wq) {
-            wake_thread(line, handler);
+        if (status == IRQ_STATUS_WAKE_THREAD && handler->thread && line->task) {
+            atomic_store(&handler->need, true, ATOMIC_RELEASE);
+            wake = true;
             claimed = true;
         } else if (status != IRQ_STATUS_IGNORED) {
             claimed = true;
         }
     }
 
+    if (wake)
+        irq_task_schedule(line->task);
     if (!claimed)
         log("irq: Spurious interrupt\n");
 
@@ -107,6 +129,7 @@ void irq_line_raise(struct irq_line* line) {
 }
 
 void irq_line_attach(struct irq_line* line, struct irq_handler* handler) {
+    handler->line = line;
     irq_lock();
     list_insert_head(&line->handlers, handler, next);
     irq_unlock();
@@ -123,21 +146,40 @@ void irq_line_detach(struct irq_line* line, struct irq_handler* handler) {
 
     while (__atomic_load_n(&line->is_busy, __ATOMIC_ACQUIRE))
         arch_cpu_pause();
+
+    atomic_exchange(&handler->need, false, ATOMIC_SEQ_CST);
+    if (line->task) {
+        while (atomic_load(&line->task->running, ATOMIC_SEQ_CST))
+            sched_reschedule(&percpu_get()->sched);
+    }
 }
 
-errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name) {
+errno_t irq_line_attach_task(struct irq_line* line, struct irq_handler* handler, const char* name) {
     if (!handler->thread)
         return EINVAL;
 
-    char task_name[TASK_NAME_MAX];
-    ksnprintf(task_name, sizeof(task_name), "irq/%s", name);
+    mutex_lock(&irq_task_lock);
+    if (!line->task) {
+        struct irq_task* it = mem_alloc(sizeof(*it), 0);
+        if (!it) {
+            mutex_unlock(&irq_task_lock);
+            return ENOMEM;
+        }
+        it->line = line;
 
-    handler->line = line;
-    work_init(&handler->work, irq_thread_work);
-    handler->holds_mask = false;
-    errno_t status = workqueue_create(task_name, SCHED_PRIO_IRQ, &handler->wq);
-    if (status)
-        return status;
+        char task_name[TASK_NAME_MAX];
+        ksnprintf(task_name, sizeof(task_name), "irq/%s", name);
+        errno_t status = task_create(task_name, process_kernel(), irq_task_fn, (uintptr_t)it, &it->task);
+        if (status) {
+            mem_free(it);
+            mutex_unlock(&irq_task_lock);
+            return status;
+        }
+        sched_set_priority(it->task, SCHED_PRIO_IRQ);
+        sched_add_task(&percpu_get()->sched, it->task);
+        line->task = it;
+    }
+    mutex_unlock(&irq_task_lock);
 
     irq_line_attach(line, handler);
     return 0;

sched/sched_dumb.c

@@ -84,6 +84,12 @@ void sched_init(struct scheduler* sched) {
 
 // Asks the CPU of `sched` to reschedule as soon as it has interrupts enabled.
 static void kick(struct scheduler* sched) {
+    if (sched == &percpu_get()->sched) {
+        sched->owe_preempt = true;
+        if (percpu_ref(irq_data)->depth == 0 && irq_get_state() && !sched->preempt_level && sched->current)
+            sched_reschedule(sched);
+        return;
+    }
     if (!atomic_exchange(&sched->kick_pending, true, ATOMIC_RELAXED))
         arch_sched_kick(cpu_of(sched));
 }
@@ -217,6 +223,7 @@ static void finish_switch(struct task* prev) {
 
 static void do_reschedule(struct scheduler* sched, uint64_t now) {
     atomic_store(&sched->kick_pending, false, ATOMIC_RELAXED);
+    sched->owe_preempt = false;
     struct task* from = sched->current;
     struct task* to = next_task(sched, now);