From 829490f8c85eb6e35d30ea1a8e8e433b7ed77562 Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Fri, 30 Oct 2026 00:21:36 +0100 Subject: [PATCH] irq: Add irq_tasks, move NVMe to it --- arch/x86_64/idt.c | 11 ++-- dev/acpi/uacpi_glue.c | 1 - dev/block/nvme/pci.c | 14 +++-- include/zinnia/irq.h | 12 ++--- include/zinnia/sched.h | 1 + lib/irq.c | 116 ++++++++++++++++++++++++++++------------- sched/sched_dumb.c | 7 +++ 7 files changed, 107 insertions(+), 55 deletions(-) diff --git a/arch/x86_64/idt.c b/arch/x86_64/idt.c index f692f52..185a197 100644 --- a/arch/x86_64/idt.c +++ b/arch/x86_64/idt.c @@ -103,12 +103,10 @@ void interrupt_handler(struct arch_context* ctx) { break; case IDT_TIMER: lapic_eoi(&arch_cpu->lapic); + arch_sched_preempt_disable(); timer_interrupt(); - if (cpu->sched.need_resched) { - arch_sched_preempt_disable(); - if (arch_sched_preempt_enable()) - sched_reschedule(&cpu->sched); - } + if (arch_sched_preempt_enable() && (cpu->sched.need_resched || cpu->sched.owe_preempt)) + sched_reschedule(&cpu->sched); break; default: if (isr < 0x20) @@ -116,7 +114,10 @@ void interrupt_handler(struct arch_context* ctx) { assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr); struct irq_line* line = arch_cpu->irq_lines[isr - 0x20]; assert(line, "Unhandled interrupt on ISR %lu", isr); + arch_sched_preempt_disable(); irq_line_raise(line); + if (arch_sched_preempt_enable() && cpu->sched.owe_preempt) + sched_reschedule(&cpu->sched); } } diff --git a/dev/acpi/uacpi_glue.c b/dev/acpi/uacpi_glue.c index 38f7bf0..6451e75 100644 --- a/dev/acpi/uacpi_glue.c +++ b/dev/acpi/uacpi_glue.c @@ -253,7 +253,6 @@ uacpi_status uacpi_kernel_install_interrupt_handler( if (!acpi_irq) return UACPI_STATUS_OUT_OF_MEMORY; acpi_irq->handler.raise = acpi_irq_raise; - acpi_irq->handler.line = line; acpi_irq->func = handler; acpi_irq->ctx = ctx; diff --git a/dev/block/nvme/pci.c b/dev/block/nvme/pci.c index 7edbc07..00554a7 100644 --- a/dev/block/nvme/pci.c +++ b/dev/block/nvme/pci.c @@ -23,12 +23,11 @@ struct nvme_irq_handler { struct nvme_ctrl* controller; }; -static enum irq_status nvme_irq_raise(struct irq_handler* self) { +static void nvme_irq_thread(struct irq_handler* self) { struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler); struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller); if (queue) nvme_io_queue_drain(queue); - return IRQ_STATUS_HANDLED; } static errno_t map_bar(struct pci_bar* bar, void** out) { @@ -84,9 +83,16 @@ static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* ma struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0); if (!handler) return ENOMEM; - handler->handler.raise = nvme_irq_raise; + handler->handler.thread = nvme_irq_thread; handler->controller = controller; - irq_line_attach(irq_line, &handler->handler); + + char name[32]; + ksnprintf(name, sizeof(name), "nvme-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address)); + status = irq_line_attach_task(irq_line, &handler->handler, name); + if (status) { + mem_free(handler); + return status; + } if (irq_line->unmask) irq_line->unmask(irq_line); } diff --git a/include/zinnia/irq.h b/include/zinnia/irq.h index c4288ee..094fb0d 100644 --- a/include/zinnia/irq.h +++ b/include/zinnia/irq.h @@ -5,9 +5,7 @@ #include #include #include -#include #include -#include #include #include @@ -33,6 +31,7 @@ enum irq_mode : uint8_t { }; struct irq_line; +struct irq_task; struct irq_handler { enum irq_status (*raise)(struct irq_handler* self); @@ -40,9 +39,7 @@ struct irq_handler { list_link(struct irq_handler*) next; struct irq_line* line; - struct workqueue* wq; - struct work work; - bool holds_mask; + _Atomic(bool) need; }; struct irq_line { @@ -55,8 +52,7 @@ struct irq_line { list_head(struct irq_handler*) handlers; enum irq_mode mode; enum irq_trigger_mode trigger; - struct spinlock thread_lock; - size_t threads_pending; + struct irq_task* task; }; struct msi_line { @@ -76,7 +72,7 @@ void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum void irq_line_raise(struct irq_line* line); void irq_line_attach(struct irq_line* line, struct irq_handler* handler); void irq_line_detach(struct irq_line* line, struct irq_handler* handler); -errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name); +errno_t irq_line_attach_task(struct irq_line* line, struct irq_handler* handler, const char* name); void irq_lock(); void irq_unlock(); diff --git a/include/zinnia/sched.h b/include/zinnia/sched.h index 5bd43a7..ac8c10a 100644 --- a/include/zinnia/sched.h +++ b/include/zinnia/sched.h @@ -102,6 +102,7 @@ struct scheduler { struct timer preempt_timer; bool need_resched; + bool owe_preempt; uint64_t slice_start; uint64_t boost_epoch; }; diff --git a/lib/irq.c b/lib/irq.c index 649593a..c318545 100644 --- a/lib/irq.c +++ b/lib/irq.c @@ -1,10 +1,14 @@ // SPDX-License-Identifier: ISC +#include #include +#include #include #include +#include #include #include +#include #include #include #include @@ -34,8 +38,12 @@ void irq_lock() { void irq_unlock() { struct irq_percpu* state = percpu_ref(irq_data); - if (--state->depth == 0 && state->saved_state) + if (--state->depth == 0 && state->saved_state) { + struct scheduler* sched = &percpu_get()->sched; + if (sched->owe_preempt && !sched->preempt_level && sched->current) + sched_reschedule(sched); arch_irq_set_state(true); + } } void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) { @@ -45,37 +53,47 @@ void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_unlock(); } -static bool holds_mask_while_threaded(struct irq_line* line) { +struct irq_task { + struct irq_line* line; + struct task* task; + _Atomic(bool) need; + _Atomic(bool) masked; + _Atomic(bool) running; +}; + +static mutex_t irq_task_lock; + +static bool holds_mask(struct irq_line* line) { return line->trigger == IRQ_TRIGGER_LEVEL && (line->mode & IRQ_MODE_MASKABLE) && line->mask && line->unmask; } -static void wake_thread(struct irq_line* line, struct irq_handler* handler) { - if (holds_mask_while_threaded(line)) { - spin_lock(&line->thread_lock); - if (!handler->holds_mask) { - handler->holds_mask = true; - if (line->threads_pending++ == 0) - line->mask(line); - } - spin_unlock(&line->thread_lock); - } +static void irq_task_fn(uintptr_t arg) { + struct irq_task* it = (struct irq_task*)arg; + struct irq_line* line = it->line; - work_queue(handler->wq, &handler->work); + while (true) { + atomic_store(&it->running, true, ATOMIC_SEQ_CST); + while (atomic_exchange(&it->need, false, ATOMIC_ACQ_REL)) { + struct irq_handler* handler; + list_foreach(handler, &line->handlers, next) { + if (atomic_exchange(&handler->need, false, ATOMIC_ACQ_REL)) + handler->thread(handler); + } + if (atomic_exchange(&it->masked, false, ATOMIC_ACQ_REL)) + line->unmask(line); + } + atomic_store(&it->running, false, ATOMIC_SEQ_CST); + + if (!atomic_load(&it->need, ATOMIC_ACQUIRE)) + sched_block(&percpu_get()->sched); + } } -static void irq_thread_work(struct work* work) { - struct irq_handler* handler = containerof(work, struct irq_handler, work); - struct irq_line* line = handler->line; - - handler->thread(handler); - - spin_lock(&line->thread_lock); - if (handler->holds_mask) { - handler->holds_mask = false; - if (--line->threads_pending == 0) - line->unmask(line); - } - spin_unlock(&line->thread_lock); +static void irq_task_schedule(struct irq_task* it) { + if (holds_mask(it->line) && !atomic_exchange(&it->masked, true, ATOMIC_ACQ_REL)) + it->line->mask(it->line); + atomic_store(&it->need, true, ATOMIC_RELEASE); + sched_wake(it->task); } void irq_line_raise(struct irq_line* line) { @@ -85,17 +103,21 @@ void irq_line_raise(struct irq_line* line) { line->is_busy = true; bool claimed = false; + bool wake = false; struct irq_handler* handler; list_foreach(handler, &line->handlers, next) { enum irq_status status = handler->raise ? handler->raise(handler) : IRQ_STATUS_WAKE_THREAD; - if (status == IRQ_STATUS_WAKE_THREAD && handler->wq) { - wake_thread(line, handler); + if (status == IRQ_STATUS_WAKE_THREAD && handler->thread && line->task) { + atomic_store(&handler->need, true, ATOMIC_RELEASE); + wake = true; claimed = true; } else if (status != IRQ_STATUS_IGNORED) { claimed = true; } } + if (wake) + irq_task_schedule(line->task); if (!claimed) log("irq: Spurious interrupt\n"); @@ -107,6 +129,7 @@ void irq_line_raise(struct irq_line* line) { } void irq_line_attach(struct irq_line* line, struct irq_handler* handler) { + handler->line = line; irq_lock(); list_insert_head(&line->handlers, handler, next); irq_unlock(); @@ -123,21 +146,40 @@ void irq_line_detach(struct irq_line* line, struct irq_handler* handler) { while (__atomic_load_n(&line->is_busy, __ATOMIC_ACQUIRE)) arch_cpu_pause(); + + atomic_exchange(&handler->need, false, ATOMIC_SEQ_CST); + if (line->task) { + while (atomic_load(&line->task->running, ATOMIC_SEQ_CST)) + sched_reschedule(&percpu_get()->sched); + } } -errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name) { +errno_t irq_line_attach_task(struct irq_line* line, struct irq_handler* handler, const char* name) { if (!handler->thread) return EINVAL; - char task_name[TASK_NAME_MAX]; - ksnprintf(task_name, sizeof(task_name), "irq/%s", name); + mutex_lock(&irq_task_lock); + if (!line->task) { + struct irq_task* it = mem_alloc(sizeof(*it), 0); + if (!it) { + mutex_unlock(&irq_task_lock); + return ENOMEM; + } + it->line = line; - handler->line = line; - work_init(&handler->work, irq_thread_work); - handler->holds_mask = false; - errno_t status = workqueue_create(task_name, SCHED_PRIO_IRQ, &handler->wq); - if (status) - return status; + char task_name[TASK_NAME_MAX]; + ksnprintf(task_name, sizeof(task_name), "irq/%s", name); + errno_t status = task_create(task_name, process_kernel(), irq_task_fn, (uintptr_t)it, &it->task); + if (status) { + mem_free(it); + mutex_unlock(&irq_task_lock); + return status; + } + sched_set_priority(it->task, SCHED_PRIO_IRQ); + sched_add_task(&percpu_get()->sched, it->task); + line->task = it; + } + mutex_unlock(&irq_task_lock); irq_line_attach(line, handler); return 0; diff --git a/sched/sched_dumb.c b/sched/sched_dumb.c index 4844175..c2019fe 100644 --- a/sched/sched_dumb.c +++ b/sched/sched_dumb.c @@ -84,6 +84,12 @@ void sched_init(struct scheduler* sched) { // Asks the CPU of `sched` to reschedule as soon as it has interrupts enabled. static void kick(struct scheduler* sched) { + if (sched == &percpu_get()->sched) { + sched->owe_preempt = true; + if (percpu_ref(irq_data)->depth == 0 && irq_get_state() && !sched->preempt_level && sched->current) + sched_reschedule(sched); + return; + } if (!atomic_exchange(&sched->kick_pending, true, ATOMIC_RELAXED)) arch_sched_kick(cpu_of(sched)); } @@ -217,6 +223,7 @@ static void finish_switch(struct task* prev) { static void do_reschedule(struct scheduler* sched, uint64_t now) { atomic_store(&sched->kick_pending, false, ATOMIC_RELAXED); + sched->owe_preempt = false; struct task* from = sched->current; struct task* to = next_task(sched, now);