From 49c454dd97d62bf4a0f4badbb95d17499e983259 Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Thu, 29 Oct 2026 01:50:52 +0100 Subject: [PATCH] dev/block: Add NVMe driver --- dev/acpi/uacpi_glue.c | 5 +- dev/block/meson.build | 2 +- dev/block/nvme/Kconfig | 13 +- dev/block/nvme/command.c | 82 ++++++++ dev/block/nvme/controller.c | 388 ++++++++++++++++++++++++++++++++++++ dev/block/nvme/core.c | 41 ---- dev/block/nvme/meson.build | 17 +- dev/block/nvme/namespace.c | 117 +++++++++++ dev/block/nvme/nvme.h | 221 ++++++++++++++++++++ dev/block/nvme/nvme_pci.h | 5 - dev/block/nvme/pci.c | 140 ++++++++++++- dev/block/nvme/prp.c | 118 +++++++++++ dev/block/nvme/queue.c | 346 ++++++++++++++++++++++++++++++++ dev/block/nvme/spec.h | 149 ++++++++++++++ dev/pci/core.c | 121 +++++++++++ include/zinnia/dma.h | 84 ++++++++ include/zinnia/irq.h | 15 ++ include/zinnia/pci.h | 8 + lib/irq.c | 61 +++++- mem/dma.c | 272 +++++++++++++++++++++++++ mem/meson.build | 1 + 21 files changed, 2132 insertions(+), 74 deletions(-) create mode 100644 dev/block/nvme/command.c create mode 100644 dev/block/nvme/controller.c delete mode 100644 dev/block/nvme/core.c create mode 100644 dev/block/nvme/namespace.c create mode 100644 dev/block/nvme/nvme.h delete mode 100644 dev/block/nvme/nvme_pci.h create mode 100644 dev/block/nvme/prp.c create mode 100644 dev/block/nvme/queue.c create mode 100644 include/zinnia/dma.h create mode 100644 mem/dma.c diff --git a/dev/acpi/uacpi_glue.c b/dev/acpi/uacpi_glue.c index fd7e36d..38f7bf0 100644 --- a/dev/acpi/uacpi_glue.c +++ b/dev/acpi/uacpi_glue.c @@ -221,7 +221,6 @@ uacpi_status uacpi_kernel_handle_firmware_request(uacpi_firmware_request* req) { struct acpi_irq { struct irq_handler handler; - struct irq_line* line; uacpi_interrupt_handler func; uacpi_handle ctx; }; @@ -254,7 +253,7 @@ uacpi_status uacpi_kernel_install_interrupt_handler( if (!acpi_irq) return UACPI_STATUS_OUT_OF_MEMORY; acpi_irq->handler.raise = acpi_irq_raise; - acpi_irq->line = line; + acpi_irq->handler.line = line; acpi_irq->func = handler; acpi_irq->ctx = ctx; @@ -268,7 +267,7 @@ uacpi_status uacpi_kernel_install_interrupt_handler( uacpi_status uacpi_kernel_uninstall_interrupt_handler(uacpi_interrupt_handler handler, uacpi_handle irq_handle) { struct acpi_irq* acpi_irq = irq_handle; - struct irq_line* line = acpi_irq->line; + struct irq_line* line = acpi_irq->handler.line; irq_line_detach(line, &acpi_irq->handler); if (list_is_empty(&line->handlers) && line->mask) line->mask(line); diff --git a/dev/block/meson.build b/dev/block/meson.build index 636b21b..7fe0894 100644 --- a/dev/block/meson.build +++ b/dev/block/meson.build @@ -6,6 +6,6 @@ kernel_sources += files( 'ram.c', ) -if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n' +if kconfig.get('CONFIG_NVME', 'n') != 'n' subdir('nvme') endif diff --git a/dev/block/nvme/Kconfig b/dev/block/nvme/Kconfig index ec31431..45ab51d 100644 --- a/dev/block/nvme/Kconfig +++ b/dev/block/nvme/Kconfig @@ -1,13 +1,4 @@ -menu "NVMe Support" - -config NVME_CORE - bool - default y - -config NVME_PCI - bool "NVMe via PCI transport" +config NVME + bool "NVMe block devices" default y depends on PCI - depends on NVME_CORE - -endmenu diff --git a/dev/block/nvme/command.c b/dev/block/nvme/command.c new file mode 100644 index 0000000..668a9d4 --- /dev/null +++ b/dev/block/nvme/command.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: ISC + +#include +#include "nvme.h" + +void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe) { + uint32_t cdw0 = mmio_field(NVME_SQE_OPC, cmd->do_write ? NVME_CMD_WRITE : NVME_CMD_READ) | + mmio_field(NVME_SQE_CID, cmd->cid) | mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this. + + uint32_t cdw10 = mmio_field(NVME_RW_SLBA_LOW, (uint32_t)cmd->start_lba); + + uint32_t cdw11 = mmio_field(NVME_RW_SLBA_HIGH, (uint32_t)(cmd->start_lba >> 32)); + + uint32_t cdw12 = mmio_field(NVME_RW_NLB, (uint16_t)(cmd->num_lbas - 1)); + uint32_t cdw13 = 0; + uint32_t cdw14 = 0; + uint32_t cdw15 = 0; + + mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid); + mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->prp1); + mmio_reg_write(sqe, NVME_SQE_DPTR1, cmd->prp2); + mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0); + mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10); + mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11); + mmio_reg_write(sqe, NVME_SQE_CDW12, cdw12); + mmio_reg_write(sqe, NVME_SQE_CDW13, cdw13); + mmio_reg_write(sqe, NVME_SQE_CDW14, cdw14); + mmio_reg_write(sqe, NVME_SQE_CDW15, cdw15); +} + +void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe) { + uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_CMD_FLUSH) | mmio_field(NVME_SQE_CID, cmd->cid); + + mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid); + mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0); +} + +void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe) { + uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_CQ) | + mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this. + + uint32_t cdw10 = mmio_field(NVME_CREATE_CQ_QSIZE, (uint16_t)(cmd->depth - 1)) | + mmio_field(NVME_CREATE_CQ_QID, (uint16_t)cmd->qid); + + uint32_t cdw11 = mmio_field(NVME_CREATE_CQ_IV, cmd->irq_vector) | + mmio_field(NVME_CREATE_CQ_IEN, cmd->irqs_enabled ? 1 : 0) | + mmio_field(NVME_CREATE_CQ_PC, 1); // Our buffer is physically contiguous. + + mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->cq_addr); + mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0); + mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10); + mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11); +} + +void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe) { + uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_SQ) | + mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this. + + uint32_t cdw10 = mmio_field(NVME_CREATE_SQ_QSIZE, (uint16_t)(cmd->depth - 1)) | + mmio_field(NVME_CREATE_SQ_QID, (uint16_t)cmd->qid); + + uint32_t cdw11 = mmio_field(NVME_CREATE_SQ_CQID, (uint16_t)cmd->cqid) | + mmio_field(NVME_CREATE_SQ_QPRIO, 0) | // Priority is the highest. + mmio_field(NVME_CREATE_SQ_PC, 1); // Our buffer is physically contiguous. + + mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->sq_addr); + mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0); + mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10); + mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11); +} + +void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe) { + uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_IDENTIFY) | + mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this. + + uint32_t cdw10 = mmio_field(NVME_IDENTIFY_CNTID, cmd->controller_id) | mmio_field(NVME_IDENTIFY_CNS, cmd->cns); + + mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->buffer); + mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0); + mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid); + mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10); +} diff --git a/dev/block/nvme/controller.c b/dev/block/nvme/controller.c new file mode 100644 index 0000000..b26b5fc --- /dev/null +++ b/dev/block/nvme/controller.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: ISC + +#define pr_fmt(fmt) "nvme: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nvme.h" + +constexpr size_t ADMIN_QUEUE_SIZE = 32; +constexpr size_t IO_QUEUE_SIZE_MAX = 1024; +constexpr size_t IDENTIFY_SIZE = 4096; +constexpr size_t NS_LIST_LEN = 1024; + +constexpr mmio_reg8 ID_MDTS = {77, MMIO_LE}; +constexpr mmio_reg32 ID_NN = {516, MMIO_LE}; +constexpr mmio_reg64 ID_NSZE = {0, MMIO_LE}; +constexpr mmio_reg8 ID_FLBAS = {26, MMIO_LE}; +constexpr mmio_field8 ID_LBAF_IDX = {ID_FLBAS, 0, 4}; +constexpr mmio_reg32 ID_LBAF0 = {128, MMIO_LE}; +constexpr mmio_field32 ID_LBADS = {ID_LBAF0, 16, 8}; + +size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl) { + const size_t cap = 512 * arch_mem_page_size(); + if (ctrl->mdts != 0) + return MIN(ctrl->mdts, cap); + return cap; +} + +errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out) { + // Read controller version. + uint32_t vs = mmio_reg_read(regs, NVME_REG_VS); + log("NVMe controller version %u.%u.%u\n", + mmio_field_get(NVME_VS_MJR, vs), + mmio_field_get(NVME_VS_MNR, vs), + mmio_field_get(NVME_VS_TER, vs)); + + uint64_t cap = mmio_reg_read(regs, NVME_REG_CAP); + + // Check if our host page size is supported. + const size_t mps_max = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMAX, cap) + 12); + const size_t mps_min = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMIN, cap) + 12); + const size_t page_size = arch_mem_page_size(); + if (page_size < mps_min || page_size > mps_max) { + log_err("Host page size is not supported on this NVMe!\n"); + return ENOTSUP; + } + + struct nvme_ctrl* ctrl = mem_alloc(sizeof(*ctrl), 0); + if (!ctrl) + return ENOMEM; + + ctrl->regs = regs; + ctrl->dma_tag = dma_tag; + ctrl->doorbell_stride = (size_t)4 << mmio_field_get(NVME_CAP_DSTRD, cap); + ctrl->mps_min = mps_min; + ctrl->mps = __builtin_ctzl(page_size) - 12; + mutex_init(&ctrl->admin_lock); + spin_init(&ctrl->io_lock); + atomic_init(&ctrl->ns_count, 0); + + *out = ctrl; + return 0; +} + +// Sets the CC.EN flag. +static errno_t set_status(struct nvme_ctrl* ctrl, bool enable) { + uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP); + // CAP.TO is in 500ms units. + const uint64_t timeout = MAX((uint64_t)mmio_field_get(NVME_CAP_TO, cap), 1) * 500'000'000; + const uint64_t deadline = clock_get_elapsed_ns() + timeout; + + uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC); + cc &= ~mmio_field(NVME_CC_EN, UINT32_MAX); + cc |= mmio_field(NVME_CC_EN, enable ? 1 : 0); + mmio_reg_write(ctrl->regs, NVME_REG_CC, cc); + + while (true) { + uint32_t csts = mmio_reg_read(ctrl->regs, NVME_REG_CSTS); + + if (mmio_field_get(NVME_CSTS_CFS, csts) != 0) + return EIO; + + bool ready = mmio_field_get(NVME_CSTS_RDY, csts) != 0; + if (ready == enable) + break; + + if (clock_get_elapsed_ns() >= deadline) + return ETIMEDOUT; + } + + return 0; +} + +static errno_t admin_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe, struct nvme_completion* out) { + errno_t status = nvme_queue_submit_cmd(queue, sqe); + if (status) + return status; + return nvme_queue_next_completion(queue, out); +} + +static errno_t identify(struct nvme_queue* queue, const struct nvme_identify_cmd* cmd) { + struct nvme_sqe sqe = {}; + nvme_identify_cmd_write(cmd, &sqe); + struct nvme_completion comp; + errno_t status = admin_cmd(queue, &sqe, &comp); + if (status) + return status; + if (!nvme_status_is_success(comp.status)) { + log_err("Identification failed with status %x\n", comp.status); + return EIO; + } + return 0; +} + +errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector) { + // Disable the controller. + errno_t status = set_status(ctrl, false); + if (status) + return status; + + uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP); + + const size_t queue_depth = MIN((size_t)mmio_field_get(NVME_CAP_MQES, cap) + 1, IO_QUEUE_SIZE_MAX); + + // Create an admin queue first so we can create more queues. + struct nvme_queue* admin_queue; + status = nvme_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 0, ADMIN_QUEUE_SIZE, &admin_queue); + if (status) + return status; + + // Set the admin queue sizes. + uint32_t aqa = 0; + aqa |= mmio_field(NVME_AQA_ACQS, admin_queue->depth - 1); + aqa |= mmio_field(NVME_AQA_ASQS, admin_queue->depth - 1); + mmio_reg_write(ctrl->regs, NVME_REG_AQA, aqa); + + // Set the addresses to the admin submission and completion queue. + mmio_reg_write(ctrl->regs, NVME_REG_ASQ, admin_queue->sq.addr); + mmio_reg_write(ctrl->regs, NVME_REG_ACQ, admin_queue->cq.addr); + + uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC); + cc &= + ~(mmio_field(NVME_CC_MPS, UINT32_MAX) | mmio_field(NVME_CC_IOCQES, UINT32_MAX) | + mmio_field(NVME_CC_IOSQES, UINT32_MAX)); + + cc |= mmio_field(NVME_CC_MPS, ctrl->mps); + cc |= mmio_field(NVME_CC_IOCQES, __builtin_ctzl(sizeof(struct nvme_cqe))); + cc |= mmio_field(NVME_CC_IOSQES, __builtin_ctzl(sizeof(struct nvme_sqe))); + + mmio_reg_write(ctrl->regs, NVME_REG_CC, cc); + + // Re-enable the controller. + status = set_status(ctrl, true); + if (status) + goto fail_admin; + + struct nvme_io_queue* io_queue; + status = + nvme_io_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 1, queue_depth, !irqs_enabled, &io_queue); + if (status) + goto fail_admin; + + struct nvme_sqe sqe = {}; + struct nvme_completion comp; + nvme_create_cq_cmd_write( + &(struct nvme_create_cq_cmd){ + .cq_addr = io_queue->cq.addr, + .depth = io_queue->depth, + .qid = io_queue->queue_id, + .irqs_enabled = irqs_enabled, + .irq_vector = irqs_enabled ? io_irq_vector : 0, + }, + &sqe + ); + status = admin_cmd(admin_queue, &sqe, &comp); + if (!status && !nvme_status_is_success(comp.status)) + status = EIO; + if (status) + goto fail_io; + + sqe = (struct nvme_sqe){}; + nvme_create_sq_cmd_write( + &(struct nvme_create_sq_cmd){ + .sq_addr = io_queue->sq.addr, + .depth = io_queue->depth, + .qid = io_queue->queue_id, + .cqid = io_queue->queue_id, + }, + &sqe + ); + status = admin_cmd(admin_queue, &sqe, &comp); + if (!status && !nvme_status_is_success(comp.status)) + status = EIO; + if (status) + goto fail_io; + + mutex_lock(&ctrl->admin_lock); + struct nvme_queue* old_admin = ctrl->admin_queue; + ctrl->admin_queue = admin_queue; + mutex_unlock(&ctrl->admin_lock); + if (old_admin) + nvme_queue_free(old_admin); + + spin_lock(&ctrl->io_lock); + ctrl->io_queue = io_queue; + spin_unlock(&ctrl->io_lock); + + return 0; + +fail_io: + nvme_io_queue_free(io_queue); +fail_admin: + nvme_queue_free(admin_queue); + return status; +} + +static void trim_ascii_end(char* str, size_t len) { + while (len > 0 && + (str[len - 1] == ' ' || str[len - 1] == '\t' || str[len - 1] == '\n' || str[len - 1] == '\r' || + str[len - 1] == '\f')) + len--; + str[len] = '\0'; +} + +errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl) { + struct dma_buffer identify_buffer; + errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer); + if (status) + return ENOMEM; + + mutex_lock(&ctrl->admin_lock); + if (!ctrl->admin_queue) { + status = ENXIO; + goto out; + } + + status = identify( + ctrl->admin_queue, + &(struct nvme_identify_cmd){ + .buffer = identify_buffer.addr, + .controller_id = 0, // TODO: Use a real number here + .cns = 1, + .nsid = 0, + } + ); + // Check if the identification was successful. + if (status) + goto out; + + dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD); + + // Read the model strings. + const uint8_t* view = identify_buffer.virt; + memcpy(ctrl->serial, view + 4, sizeof(ctrl->serial) - 1); + memcpy(ctrl->model, view + 24, sizeof(ctrl->model) - 1); + memcpy(ctrl->revision, view + 64, sizeof(ctrl->revision) - 1); + trim_ascii_end(ctrl->serial, sizeof(ctrl->serial) - 1); + trim_ascii_end(ctrl->model, sizeof(ctrl->model) - 1); + trim_ascii_end(ctrl->revision, sizeof(ctrl->revision) - 1); + log("Model \"%s\"\n", ctrl->model); + + atomic_store(&ctrl->ns_count, mmio_reg_read(identify_buffer.virt, ID_NN), ATOMIC_RELEASE); + + uint8_t mdts = mmio_reg_read(identify_buffer.virt, ID_MDTS); + if (mdts != 0 && mdts < sizeof(size_t) * 8) + ctrl->mdts = ((size_t)1 << mdts) * ctrl->mps_min; + +out: + mutex_unlock(&ctrl->admin_lock); + dma_buffer_free(&identify_buffer); + return status; +} + +// Scans all existing namespaces. +errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out) { + vec_init(out); + + uint32_t* ns_list = mem_alloc(NS_LIST_LEN * sizeof(uint32_t), 0); + if (!ns_list) + return ENOMEM; + + struct dma_buffer identify_buffer; + errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer); + if (status) { + mem_free(ns_list); + return ENOMEM; + } + + mutex_lock(&ctrl->admin_lock); + struct nvme_queue* aq = ctrl->admin_queue; + if (!aq) { + status = ENXIO; + goto out; + } + + uint32_t nsid = 0; + while (true) { + const struct nvme_identify_cmd admin_ident_cmd = { + .buffer = identify_buffer.addr, + .controller_id = 0, + .cns = 2, + .nsid = nsid, + }; + status = identify(aq, &admin_ident_cmd); + + if (status) + goto out; + + dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD); + + // Parse the identifier list until we hit an namespace value of 0. + size_t ns_list_len = 0; + const uint32_t* raw_list = identify_buffer.virt; + while (ns_list_len < NS_LIST_LEN && raw_list[ns_list_len] != 0) { + ns_list[ns_list_len] = raw_list[ns_list_len]; + ns_list_len++; + } + + for (size_t i = 0; i < ns_list_len; i++) { + const uint32_t id = ns_list[i]; + const struct nvme_identify_cmd ns_ident_cmd = { + .buffer = identify_buffer.addr, + .controller_id = 0, + .cns = 0, + .nsid = id, + }; + status = identify(aq, &ns_ident_cmd); + + if (status) + goto out; + + dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD); + + const uint64_t lba_count = mmio_reg_read(identify_buffer.virt, ID_NSZE); + + // To get the block size of a namespace, we need to read a descriptor (see LBAF0-15). + // Get the format descriptor index used by this namespace. + const uint8_t lba_format_idx = mmio_field_get(ID_LBAF_IDX, mmio_reg_read(identify_buffer.virt, ID_FLBAS)); + + const mmio_reg32 lbaf = {ID_LBAF0.offset + lba_format_idx * sizeof(uint32_t), MMIO_LE}; + const uint32_t lba_format = mmio_reg_read(identify_buffer.virt, lbaf); + + uint8_t lba_shift = mmio_field_get(ID_LBADS, lba_format); + // If LBADS doesn't contain a non-zero value, we need to assume a default. Use 512 bytes here. + if (lba_shift == 0) + lba_shift = 9; + + struct nvme_ns* ns; + status = nvme_ns_new(ctrl, id, lba_shift, lba_count, nvme_ctrl_max_transfer_bytes(ctrl), &ns); + if (status) + goto out; + status = vec_push(out, ns); + if (status) { + blockdev_unref(&ns->blk); + goto out; + } + } + + // If there is no 0 in the buffer, there's more entries than can fit in `identify_buffer`. + if (ns_list_len < NS_LIST_LEN) + break; + nsid = ns_list[ns_list_len - 1]; + } + +out: + mutex_unlock(&ctrl->admin_lock); + dma_buffer_free(&identify_buffer); + mem_free(ns_list); + if (status) { + vec_foreach(out, i) { + blockdev_unref(&vec_elem_get(out, i)->blk); + } + vec_destroy(out); + } + return status; +} + +struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl) { + spin_lock(&ctrl->io_lock); + struct nvme_io_queue* queue = ctrl->io_queue; + spin_unlock(&ctrl->io_lock); + return queue; +} diff --git a/dev/block/nvme/core.c b/dev/block/nvme/core.c deleted file mode 100644 index 0423964..0000000 --- a/dev/block/nvme/core.c +++ /dev/null @@ -1,41 +0,0 @@ -#define pr_fmt(fmt) "nvme: " fmt - -#include -#include -#include -#include -#include -#include - -#if CONFIG_NVME_PCI -#include "nvme_pci.h" -#endif - -struct nvme_dev { -#if CONFIG_NVME_PCI - struct pci_device pci_dev; -#endif -}; - -errno_t nvme_init(struct module* mod) { - errno_t status = 0; - -#if CONFIG_NVME_PCI - status = pci_driver_register(&nvme_driver); -#endif - - return status; -} - -errno_t nvme_exit(struct module* mod) { - // TODO: Unregister - return 0; -} - -MODULE_ENTRY(nvme_init); -MODULE_EXIT(nvme_exit); - -MODULE_DESC("NVMe Block Devices"); -MODULE_AUTHOR("Marvin Friedrich"); -MODULE_LICENSE("ISC"); -MODULE_VERSION("1.0"); diff --git a/dev/block/nvme/meson.build b/dev/block/nvme/meson.build index 624df93..e00f6e2 100644 --- a/dev/block/nvme/meson.build +++ b/dev/block/nvme/meson.build @@ -1,12 +1,13 @@ -nvme_sources = files('core.c') - -if kconfig.get('CONFIG_NVME_PCI', 'n') == 'y' - nvme_sources += files('pci.c') -endif - modules += { 'nvme': { - 'modular': kconfig.get('CONFIG_NVME_CORE') == 'm', - 'sources': nvme_sources, + 'modular': kconfig.get('CONFIG_NVME') == 'm', + 'sources': files( + 'command.c', + 'controller.c', + 'namespace.c', + 'pci.c', + 'prp.c', + 'queue.c', + ), }, } diff --git a/dev/block/nvme/namespace.c b/dev/block/nvme/namespace.c new file mode 100644 index 0000000..2c9bb39 --- /dev/null +++ b/dev/block/nvme/namespace.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: ISC + +#define pr_fmt(fmt) "nvme: " fmt + +#include +#include +#include +#include +#include +#include "nvme.h" + +static errno_t nvme_ns_submit(struct blockdev* blk, struct bio* bio) { + struct nvme_ns* ns = containerof(blk, struct nvme_ns, blk); + + if (bio->op != BIO_FLUSH) { + uint64_t end_lba; + if (ckd_add(&end_lba, bio->lba, (uint64_t)bio->num_lbas)) { + bio_complete(bio, EOVERFLOW, 0); + return 0; + } + if (end_lba > ns->lba_count) { + bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0); + return 0; + } + } + + struct nvme_io_queue* queue = nvme_ctrl_io_queue(ns->ctrl); + if (!queue) { + bio_complete(bio, EIO, 0); + return 0; + } + + size_t bytes = 0; + for (size_t i = 0; i < bio->num_segs; i++) + bytes += bio->segs[i].len; + + _Atomic(bool) done = false; + const bool poll = queue->poll; + + uint16_t cid = nvme_io_queue_submit( + queue, + bio, + (struct nvme_rw_cmd){ + .prp1 = 0, + .prp2 = 0, + .cid = 0, + .do_write = bio->op == BIO_WRITE, + .start_lba = bio->lba, + .num_lbas = bio->num_lbas, + .bytes = bytes, + .control = 0, + .ds_mgmt = 0, + .ref_tag = 0, + .app_tag = 0, + .app_mask = 0, + .nsid = ns->nsid, + }, + poll ? &done : nullptr + ); + + // If we're not interrupt-driven, poll the IO completion queue. + if (poll) { + const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS; + while (!atomic_load(&done, ATOMIC_ACQUIRE)) { + nvme_io_queue_drain(queue); + if (atomic_load(&done, ATOMIC_ACQUIRE)) + break; + if (clock_get_elapsed_ns() >= deadline && nvme_io_queue_abandon(queue, cid, bio)) { + bio_complete(bio, EIO, 0); + break; + } + } + } + + return 0; +} + +static void nvme_ns_destroy(struct blockdev* blk) { + mem_free(containerof(blk, struct nvme_ns, blk)); +} + +static const struct blockdev_ops nvme_ns_ops = { + .submit = nvme_ns_submit, + .destroy = nvme_ns_destroy, +}; + +errno_t nvme_ns_new( + struct nvme_ctrl* ctrl, + uint32_t nsid, + uint8_t lba_shift, + uint64_t lba_count, + size_t max_transfer_bytes, + struct nvme_ns** out +) { + log("Namespace %u: LBA size %zu bytes, %zu MBs total\n", + nsid, + (size_t)1 << lba_shift, + (size_t)(lba_count << lba_shift) / 1024 / 1024); + + struct nvme_ns* ns = mem_alloc(sizeof(*ns), 0); + if (!ns) + return ENOMEM; + + blockdev_init(&ns->blk, &nvme_ns_ops, NVME_MAJOR, nsid, (size_t)1 << lba_shift, lba_count); + ns->blk.limits = (struct blockdev_limits){ + .max_lbas = MAX(max_transfer_bytes >> lba_shift, 1), + .max_segs = 512, + }; + ns->ctrl = ctrl; + ns->nsid = nsid; + ns->lba_shift = lba_shift; + ns->lba_count = lba_count; + ns->max_transfer_bytes = max_transfer_bytes; + + *out = ns; + return 0; +} diff --git a/dev/block/nvme/nvme.h b/dev/block/nvme/nvme.h new file mode 100644 index 0000000..0f8b197 --- /dev/null +++ b/dev/block/nvme/nvme.h @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "spec.h" + +constexpr uint32_t NVME_MAJOR = 159; +constexpr uint64_t NVME_COMPLETION_TIMEOUT_NS = 30'000'000'000; + +// An entry in the completion queue. +struct nvme_completion { + uint32_t result; + uint16_t sq_head; + uint16_t sq_id; + uint16_t cmd_id; + bool phase_tag; + uint16_t status; +}; + +static inline bool nvme_status_is_success(uint16_t status) { + return status == 0; +} + +struct nvme_rw_cmd { + uint64_t prp1; + uint64_t prp2; + uint16_t cid; + bool do_write; + uint64_t start_lba; + size_t num_lbas; + size_t bytes; + uint16_t control; + uint32_t ds_mgmt; + uint32_t ref_tag; + uint16_t app_tag; + uint16_t app_mask; + uint32_t nsid; +}; + +struct nvme_flush_cmd { + uint16_t cid; + uint32_t nsid; +}; + +struct nvme_create_cq_cmd { + dma_addr_t cq_addr; + size_t depth; + size_t qid; + bool irqs_enabled; + uint16_t irq_vector; +}; + +struct nvme_create_sq_cmd { + dma_addr_t sq_addr; + size_t depth; + size_t qid; + size_t cqid; +}; + +struct nvme_identify_cmd { + dma_addr_t buffer; + uint16_t controller_id; + uint8_t cns; + uint32_t nsid; +}; + +void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe); +void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe); +void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe); +void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe); +void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe); + +struct nvme_queue { + // Amount of queue entries. + size_t depth; + size_t doorbells_offset; + size_t doorbell_stride; + void* regs; + // Physical buffer for the completion queue. + struct dma_buffer cq; + // The index of the current completion queue entry. + size_t cq_head; + // Determines whether a completion queue entry is new. + uint8_t cq_phase; + // Physical buffer for the submission queue. + struct dma_buffer sq; + // The index of the current submission queue entry. + size_t sq_tail; +}; + +errno_t nvme_queue_new( + struct dma_tag* tag, + void* regs, + size_t doorbell_stride, + size_t queue_id, + size_t depth, + struct nvme_queue** out +); +void nvme_queue_free(struct nvme_queue* queue); +errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe); +errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out); + +struct nvme_inflight { + bool active; + struct bio* bio; + size_t lbas; + struct dma_map* map; + struct dma_buffer prps; + _Atomic(bool)* done; +}; + +struct nvme_io_queue { + size_t queue_id; + size_t depth; + size_t doorbells_offset; + size_t doorbell_stride; + void* regs; + bool poll; + struct dma_tag* tag; + struct dma_buffer cq; + struct dma_buffer sq; + struct spinlock sq_lock; + size_t sq_tail; + struct spinlock cq_lock; + size_t cq_head; + uint8_t cq_phase; + struct spinlock cmds_lock; + struct nvme_inflight* slots; + uint16_t* free; + size_t num_free; + struct event slots_free; +}; + +errno_t nvme_io_queue_new( + struct dma_tag* tag, + void* regs, + size_t doorbell_stride, + size_t queue_id, + size_t depth, + bool poll, + struct nvme_io_queue** out +); +void nvme_io_queue_free(struct nvme_io_queue* queue); +uint16_t nvme_io_queue_submit( + struct nvme_io_queue* queue, + struct bio* bio, + struct nvme_rw_cmd cmd, + _Atomic(bool)* done +); +bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio); +size_t nvme_io_queue_drain(struct nvme_io_queue* queue); + +errno_t nvme_build_prps( + struct dma_tag* tag, + struct bio* bio, + struct nvme_inflight* slot, + uint64_t* prp1, + uint64_t* prp2 +); +void nvme_release_prps(struct nvme_inflight* slot); + +struct nvme_ns; + +struct nvme_ctrl { + void* regs; + struct dma_tag* dma_tag; + size_t doorbell_stride; // The amount of bytes between two doorbells. + size_t mps_min; + uint8_t mps; + + char serial[20 + 1]; // Serial number of the controller. + char model[40 + 1]; // Model name of the controller. + char revision[8 + 1]; // Revision of the controller's firmware. + + struct nvme_queue* admin_queue; // Queue to submit admin commands to. + mutex_t admin_lock; + + struct nvme_io_queue* io_queue; // Queue to submit IO commands to. + struct spinlock io_lock; + + _Atomic(uint32_t) ns_count; // Amount of namespaces. + size_t mdts; // Maximum data transfer size. +}; + +typedef vec(struct nvme_ns*) nvme_ns_vec_t; + +errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out); +size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl); +errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector); +errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl); +errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out); +struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl); + +struct nvme_ns { + struct blockdev blk; + struct nvme_ctrl* ctrl; + uint32_t nsid; + uint8_t lba_shift; + uint64_t lba_count; + size_t max_transfer_bytes; +}; + +errno_t nvme_ns_new( + struct nvme_ctrl* ctrl, + uint32_t nsid, + uint8_t lba_shift, + uint64_t lba_count, + size_t max_transfer_bytes, + struct nvme_ns** out +); diff --git a/dev/block/nvme/nvme_pci.h b/dev/block/nvme/nvme_pci.h deleted file mode 100644 index 653c967..0000000 --- a/dev/block/nvme/nvme_pci.h +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: ISC - -#pragma once - -extern struct pci_driver nvme_driver; diff --git a/dev/block/nvme/pci.c b/dev/block/nvme/pci.c index 1efd5d1..7edbc07 100644 --- a/dev/block/nvme/pci.c +++ b/dev/block/nvme/pci.c @@ -1,13 +1,128 @@ // SPDX-License-Identifier: ISC +#define pr_fmt(fmt) "nvme: " fmt + +#include +#include #include +#include #include #include +#include +#include +#include +#include +#include +#include +#include "nvme.h" + +static _Atomic(size_t) nvme_counter; + +struct nvme_irq_handler { + struct irq_handler handler; + struct nvme_ctrl* controller; +}; + +static enum irq_status nvme_irq_raise(struct irq_handler* self) { + struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler); + struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller); + if (queue) + nvme_io_queue_drain(queue); + return IRQ_STATUS_HANDLED; +} + +static errno_t map_bar(struct pci_bar* bar, void** out) { + const size_t page_size = arch_mem_page_size(); + const size_t len = ALIGN_UP(bar->size, page_size); + uintptr_t vaddr = vm_alloc(len); + if (!vaddr) + return ENOMEM; + + for (size_t i = 0; i < len; i += page_size) { + errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, bar->address + i, PTE_READ | PTE_WRITE, CACHE_MMIO); + if (status) + return status; + } + + *out = (void*)vaddr; + return 0; +} static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) { - pci_log(dev, "Detected controller \n"); + pci_log(dev, "Probing NVMe device\n"); - return 0; + // Enable MMIO decoding and DMA. We only support MSI-X or polling, so keep legacy INTx disabled. + uint16_t cmd = pci_read16(dev, PCI_COMMAND); + pci_write16(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEM | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE); + + struct irq_line* irq_line = nullptr; + if (pci_setup_msix(dev, &irq_line)) { + pci_log_warn(dev, "NVMe MSI-X setup failed, falling back to polling completions\n"); + irq_line = nullptr; + } + + struct pci_bar bar; + errno_t status = pci_bar_get(dev, 0, &bar); + if (status) + return ENXIO; + if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64) + return ENXIO; + + void* regs; + status = map_bar(&bar, ®s); + if (status) + return status; + + struct nvme_ctrl* controller; + status = nvme_ctrl_new_pci(regs, dev->dma_tag, &controller); + if (status) { + pci_log_err(dev, "Failed to probe controller: %i\n", status); + return ENODEV; + } + + if (irq_line) { + struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0); + if (!handler) + return ENOMEM; + handler->handler.raise = nvme_irq_raise; + handler->controller = controller; + irq_line_attach(irq_line, &handler->handler); + if (irq_line->unmask) + irq_line->unmask(irq_line); + } + + // Reset the controller to initialize all queues and other structures. + status = nvme_ctrl_reset(controller, irq_line != nullptr, 0); + if (status) { + pci_log_err(dev, "Failed to reset controller: %i\n", status); + return ENODEV; + } + + status = nvme_ctrl_identify(controller); + if (status) { + pci_log_err(dev, "Failed to identify controller: %i\n", status); + return ENODEV; + } + + nvme_ns_vec_t namespaces; + status = nvme_ctrl_scan_namespaces(controller, &namespaces); + if (status) { + pci_log_err(dev, "Failed to identify controller: %i\n", status); + return ENODEV; + } + + const size_t nvme_id = atomic_fetch_add(&nvme_counter, 1, ATOMIC_SEQ_CST); + vec_foreach(&namespaces, i) { + struct nvme_ns* ns = vec_elem_get(&namespaces, i); + char path[32]; + ksnprintf(path, sizeof(path), "nvme%zun%u", nvme_id, ns->nsid); + status = blockdev_register(path, &ns->blk); + if (status) + break; + } + vec_destroy(&namespaces); + + return status; } static const struct pci_match nvme_match[] = { @@ -17,9 +132,26 @@ static const struct pci_match nvme_match[] = { MATCH_TABLE("pci", nvme_match); -struct pci_driver nvme_driver = { - .name = "nvme_pci", +static struct pci_driver nvme_driver = { + .name = "nvme", .match = nvme_match, .num_matches = ARRAY_SIZE(nvme_match), .probe = nvme_pci_probe, }; + +errno_t nvme_init(struct module* mod) { + return pci_driver_register(&nvme_driver); +} + +errno_t nvme_exit(struct module* mod) { + // TODO: Unregister + return 0; +} + +MODULE_ENTRY(nvme_init); +MODULE_EXIT(nvme_exit); + +MODULE_DESC("NVMe Block Devices"); +MODULE_AUTHOR("Marvin Friedrich"); +MODULE_LICENSE("ISC"); +MODULE_VERSION("1.0"); diff --git a/dev/block/nvme/prp.c b/dev/block/nvme/prp.c new file mode 100644 index 0000000..5c74ee0 --- /dev/null +++ b/dev/block/nvme/prp.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include "nvme.h" + +static size_t count_pages(struct dma_map* map, size_t page_size) { + size_t count = 0; + for (size_t i = 0; i < map->num_segs; i++) { + const struct dma_seg* seg = &map->segs[i]; + if (seg->len == 0) + continue; + const dma_addr_t end = seg->addr + seg->len; + for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size) + count++; + } + return count; +} + +static void fill_pages(struct dma_map* map, size_t page_size, uint64_t* out) { + bool first_page = true; + size_t idx = 0; + for (size_t i = 0; i < map->num_segs; i++) { + const struct dma_seg* seg = &map->segs[i]; + if (seg->len == 0) + continue; + const dma_addr_t end = seg->addr + seg->len; + for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size) { + if (first_page) + first_page = false; + else + out[idx++] = page; + } + } +} + +errno_t nvme_build_prps( + struct dma_tag* tag, + struct bio* bio, + struct nvme_inflight* slot, + uint64_t* prp1, + uint64_t* prp2 +) { + const size_t page_size = arch_mem_page_size(); + slot->map = nullptr; + slot->prps = (struct dma_buffer){}; + + if (bio->num_segs == 0) + return EIO; + + errno_t status = dma_map_create(tag, SIZE_MAX, bio->num_segs, SIZE_MAX, 0, &slot->map); + if (status) + return status; + + status = dma_map_load_bio(slot->map, bio); + if (status) + goto fail; + if (slot->map->num_segs == 0) { + status = EIO; + goto fail; + } + + dma_map_sync(slot->map, 0, slot->map->mapped_len, bio->op == BIO_WRITE ? DMA_SYNC_PREWRITE : DMA_SYNC_PREREAD); + + *prp1 = slot->map->segs[0].addr; + + // Enumerate the base of every page the transfer touches in order. + const size_t pages = count_pages(slot->map, page_size); + const size_t rest_pages = pages ? pages - 1 : 0; + + if (rest_pages == 0) { + *prp2 = 0; + return 0; + } + + if (rest_pages == 1) { + uint64_t second; + fill_pages(slot->map, page_size, &second); + *prp2 = second; + return 0; + } + + // We don't want to chain across pages. + const size_t entries_per_page = page_size / sizeof(uint64_t); + if (rest_pages > entries_per_page) { + status = ENOMEM; + goto fail; + } + + status = dma_buffer_alloc(tag, page_size, &slot->prps); + if (status) + goto fail; + + fill_pages(slot->map, page_size, slot->prps.virt); + dma_buffer_sync(&slot->prps, 0, rest_pages * sizeof(uint64_t), DMA_SYNC_PREWRITE); + + *prp2 = slot->prps.addr; + return 0; + +fail: + nvme_release_prps(slot); + return status; +} + +void nvme_release_prps(struct nvme_inflight* slot) { + if (slot->map) { + dma_map_sync( + slot->map, + 0, + slot->map->mapped_len, + slot->bio && slot->bio->op == BIO_WRITE ? DMA_SYNC_POSTWRITE : DMA_SYNC_POSTREAD + ); + dma_map_unload(slot->map); + dma_map_destroy(slot->map); + slot->map = nullptr; + } + dma_buffer_free(&slot->prps); +} diff --git a/dev/block/nvme/queue.c b/dev/block/nvme/queue.c new file mode 100644 index 0000000..08e27fc --- /dev/null +++ b/dev/block/nvme/queue.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: ISC + +#define pr_fmt(fmt) "nvme: " fmt + +#include +#include +#include +#include +#include +#include +#include "nvme.h" + +constexpr size_t DOORBELL_OFFSET = 0x1000; + +static void ring_tail(void* regs, size_t doorbells_offset, size_t tail) { + mmio_reg_write(regs, ((mmio_reg32){doorbells_offset, MMIO_LE}), (uint32_t)tail); +} + +static void ring_head(void* regs, size_t doorbells_offset, size_t doorbell_stride, size_t head) { + mmio_reg_write(regs, ((mmio_reg32){doorbells_offset + doorbell_stride, MMIO_LE}), (uint32_t)head); +} + +// Creates a new submission and completion queue pair. +errno_t nvme_queue_new( + struct dma_tag* tag, + void* regs, + size_t doorbell_stride, + size_t queue_id, + size_t depth, + struct nvme_queue** out +) { + const size_t align = 0x1000; + const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align); + const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align); + + struct nvme_queue* queue = mem_alloc(sizeof(*queue), 0); + if (!queue) + return ENOMEM; + + // Allocate memory the completion queue. + errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq); + if (status) + goto fail; + + // Allocate memory for the submission queue. + status = dma_buffer_alloc(tag, sq_size, &queue->sq); + if (status) + goto fail; + + // Calculate the offset of the doorbell registers. The stride is already precomputed here. + queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride); + queue->doorbell_stride = doorbell_stride; + queue->depth = depth; + queue->regs = regs; + queue->cq_head = 0; + queue->cq_phase = 1; // When the controller is enabled, the first phase is 1. + queue->sq_tail = 0; + + *out = queue; + return 0; + +fail: + nvme_queue_free(queue); + return status; +} + +void nvme_queue_free(struct nvme_queue* queue) { + dma_buffer_free(&queue->cq); + dma_buffer_free(&queue->sq); + mem_free(queue); +} + +// Submits a command to this queue. +errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe) { + struct nvme_sqe* entries = queue->sq.virt; + memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe)); + dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE); + + queue->sq_tail++; + if (queue->sq_tail == queue->depth) + queue->sq_tail = 0; + + // Notify the controller of the new tail index. + ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail); + + return 0; +} + +// Reads the next completion entry from the queue. +errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out) { + struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head]; + + // Wait until the phase for this entry has changed. + uint32_t dw3; + const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS; + while (true) { + dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD); + dw3 = mmio_reg_read(entry, NVME_CQE_DW3); + + // The controller will flip the phase bit of the current entry when writing. + if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) == queue->cq_phase) + break; + + if (clock_get_elapsed_ns() >= deadline) + return ETIMEDOUT; + } + + // Then, read the rest of the completion queue entry. + uint32_t dw0 = mmio_reg_read(entry, NVME_CQE_DW0); + uint32_t dw2 = mmio_reg_read(entry, NVME_CQE_DW2); + + *out = (struct nvme_completion){ + .result = dw0, + .sq_head = mmio_field_get(NVME_CQE_SQ_HEAD, dw2), + .sq_id = mmio_field_get(NVME_CQE_SQ_IDENT, dw2), + .cmd_id = mmio_field_get(NVME_CQE_CID, dw3), + .status = mmio_field_get(NVME_CQE_STATUS, dw3), + .phase_tag = mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != 0, + }; + + queue->cq_head++; + if (queue->cq_head == queue->depth) { + queue->cq_head = 0; + queue->cq_phase ^= 1; // Flip the phase bit. + } + + // Notify the controller of the new head index. + ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head); + + return 0; +} + +errno_t nvme_io_queue_new( + struct dma_tag* tag, + void* regs, + size_t doorbell_stride, + size_t queue_id, + size_t depth, + bool poll, + struct nvme_io_queue** out +) { + const size_t align = 0x1000; + const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align); + const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align); + + struct nvme_io_queue* queue = mem_alloc(sizeof(*queue), 0); + if (!queue) + return ENOMEM; + + errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq); + if (status) + goto fail; + + status = dma_buffer_alloc(tag, sq_size, &queue->sq); + if (status) + goto fail; + + const size_t slots = depth - 1; + queue->slots = mem_alloc(slots * sizeof(*queue->slots), 0); + queue->free = mem_alloc(slots * sizeof(*queue->free), 0); + if (!queue->slots || !queue->free) { + status = ENOMEM; + goto fail; + } + for (size_t i = 0; i < slots; i++) + queue->free[i] = slots - 1 - i; + queue->num_free = slots; + + queue->queue_id = queue_id; + queue->depth = depth; + queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride); + queue->doorbell_stride = doorbell_stride; + queue->regs = regs; + queue->poll = poll; + queue->tag = tag; + queue->sq_tail = 0; + queue->cq_head = 0; + queue->cq_phase = 1; + spin_init(&queue->sq_lock); + spin_init(&queue->cq_lock); + spin_init(&queue->cmds_lock); + event_init(&queue->slots_free); + + *out = queue; + return 0; + +fail: + nvme_io_queue_free(queue); + return status; +} + +void nvme_io_queue_free(struct nvme_io_queue* queue) { + dma_buffer_free(&queue->cq); + dma_buffer_free(&queue->sq); + mem_free(queue->slots); + mem_free(queue->free); + mem_free(queue); +} + +static uint16_t acquire_slot(struct nvme_io_queue* queue) { + while (true) { + spin_lock(&queue->cmds_lock); + if (queue->num_free) { + uint16_t cid = queue->free[--queue->num_free]; + spin_unlock(&queue->cmds_lock); + return cid; + } + spin_unlock(&queue->cmds_lock); + + if (queue->poll) { + nvme_io_queue_drain(queue); + continue; + } + + nvme_io_queue_drain(queue); + event_reset(&queue->slots_free); + spin_lock(&queue->cmds_lock); + bool empty = queue->num_free == 0; + spin_unlock(&queue->cmds_lock); + if (empty) + event_wait(&queue->slots_free); + } +} + +static void release_slot(struct nvme_io_queue* queue, uint16_t cid) { + spin_lock(&queue->cmds_lock); + queue->free[queue->num_free++] = cid; + spin_unlock(&queue->cmds_lock); +} + +static void write_sqe(struct nvme_io_queue* queue, const struct nvme_sqe* sqe) { + spin_lock(&queue->sq_lock); + struct nvme_sqe* entries = queue->sq.virt; + memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe)); + dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE); + queue->sq_tail++; + if (queue->sq_tail == queue->depth) + queue->sq_tail = 0; + ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail); + spin_unlock(&queue->sq_lock); +} + +uint16_t nvme_io_queue_submit( + struct nvme_io_queue* queue, + struct bio* bio, + struct nvme_rw_cmd cmd, + _Atomic(bool)* done +) { + const uint16_t cid = acquire_slot(queue); + struct nvme_inflight* slot = &queue->slots[cid]; + + struct nvme_sqe sqe = {}; + if (bio->op == BIO_FLUSH) { + slot->map = nullptr; + slot->prps = (struct dma_buffer){}; + nvme_flush_cmd_write(&(struct nvme_flush_cmd){.cid = cid, .nsid = cmd.nsid}, &sqe); + } else { + errno_t status = nvme_build_prps(queue->tag, bio, slot, &cmd.prp1, &cmd.prp2); + if (status) { + release_slot(queue, cid); + bio_complete(bio, EIO, 0); + if (done) + atomic_store(done, true, ATOMIC_RELEASE); + return cid; + } + cmd.cid = cid; + nvme_rw_cmd_write(&cmd, &sqe); + } + + spin_lock(&queue->cmds_lock); + slot->active = true; + slot->bio = bio; + slot->lbas = bio->num_lbas; + slot->done = done; + spin_unlock(&queue->cmds_lock); + + write_sqe(queue, &sqe); + return cid; +} + +bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio) { + spin_lock(&queue->cmds_lock); + struct nvme_inflight* slot = &queue->slots[cid]; + bool abandoned = slot->active && slot->bio == bio; + if (abandoned) { + slot->bio = nullptr; + slot->done = nullptr; + } + spin_unlock(&queue->cmds_lock); + return abandoned; +} + +size_t nvme_io_queue_drain(struct nvme_io_queue* queue) { + spin_lock(&queue->cq_lock); + size_t completed = 0; + + while (true) { + struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head]; + dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD); + uint32_t dw3 = mmio_reg_read(entry, NVME_CQE_DW3); + if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != queue->cq_phase) + break; + + uint16_t cid = mmio_field_get(NVME_CQE_CID, dw3); + uint16_t status = mmio_field_get(NVME_CQE_STATUS, dw3); + + queue->cq_head++; + if (queue->cq_head == queue->depth) { + queue->cq_head = 0; + queue->cq_phase ^= 1; + } + + struct nvme_inflight inflight = {}; + spin_lock(&queue->cmds_lock); + if (cid < queue->depth - 1 && queue->slots[cid].active) { + inflight = queue->slots[cid]; + queue->slots[cid].active = false; + queue->free[queue->num_free++] = cid; + } + spin_unlock(&queue->cmds_lock); + + if (inflight.active) { + nvme_release_prps(&inflight); + if (inflight.bio) { + if (nvme_status_is_success(status)) + bio_complete(inflight.bio, 0, inflight.lbas); + else + bio_complete(inflight.bio, EIO, 0); + } + if (inflight.done) + atomic_store(inflight.done, true, ATOMIC_RELEASE); + } + + completed++; + } + + if (completed > 0) + ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head); + + spin_unlock(&queue->cq_lock); + + if (completed > 0) + event_set(&queue->slots_free); + + return completed; +} diff --git a/dev/block/nvme/spec.h b/dev/block/nvme/spec.h index baecd22..6c06f86 100644 --- a/dev/block/nvme/spec.h +++ b/dev/block/nvme/spec.h @@ -1,3 +1,152 @@ // SPDX-License-Identifier: ISC #pragma once + +#include +#include +#include +#include + +// Submission queue entry. +struct [[__packed]] nvme_sqe { + uint32_t cdw0; + + uint32_t nsid; // Namespace Identifier + uint32_t cdw2; + uint32_t cdw3; + + uint64_t mptr; // Metadata Pointer + + uint64_t dptr0; // Data Pointer 0 + uint64_t dptr1; // Data Pointer 1 + + uint32_t cdw10; + uint32_t cdw11; + uint32_t cdw12; + uint32_t cdw13; + uint32_t cdw14; + uint32_t cdw15; +}; +static_assert(sizeof(struct nvme_sqe) == 0x40); + +constexpr mmio_reg32 NVME_SQE_CDW0 = {offsetof(struct nvme_sqe, cdw0), MMIO_LE}; +constexpr mmio_field32 NVME_SQE_CID = {NVME_SQE_CDW0, 16, 16}; // Command Identifier +constexpr mmio_field32 NVME_SQE_PSDT = {NVME_SQE_CDW0, 14, 2}; // PRP or SGL for Data Transfer +constexpr mmio_field32 NVME_SQE_FUSE = {NVME_SQE_CDW0, 8, 2}; // Fused Operation +constexpr mmio_field32 NVME_SQE_OPC = {NVME_SQE_CDW0, 0, 8}; // Opcode + +constexpr mmio_reg32 NVME_SQE_NSID = {offsetof(struct nvme_sqe, nsid), MMIO_LE}; // Namespace Identifier +constexpr mmio_reg64 NVME_SQE_MPTR = {offsetof(struct nvme_sqe, mptr), MMIO_LE}; // Metadata Pointer +constexpr mmio_reg64 NVME_SQE_DPTR0 = {offsetof(struct nvme_sqe, dptr0), MMIO_LE}; // Data Pointer 0 +constexpr mmio_reg64 NVME_SQE_DPTR1 = {offsetof(struct nvme_sqe, dptr1), MMIO_LE}; // Data Pointer 1 + +constexpr mmio_reg32 NVME_SQE_CDW10 = {offsetof(struct nvme_sqe, cdw10), MMIO_LE}; +constexpr mmio_reg32 NVME_SQE_CDW11 = {offsetof(struct nvme_sqe, cdw11), MMIO_LE}; +constexpr mmio_reg32 NVME_SQE_CDW12 = {offsetof(struct nvme_sqe, cdw12), MMIO_LE}; +constexpr mmio_reg32 NVME_SQE_CDW13 = {offsetof(struct nvme_sqe, cdw13), MMIO_LE}; +constexpr mmio_reg32 NVME_SQE_CDW14 = {offsetof(struct nvme_sqe, cdw14), MMIO_LE}; +constexpr mmio_reg32 NVME_SQE_CDW15 = {offsetof(struct nvme_sqe, cdw15), MMIO_LE}; + +constexpr uint8_t NVME_CREATE_CQ_OPCODE = 0x05; +constexpr mmio_field32 NVME_CREATE_CQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size +constexpr mmio_field32 NVME_CREATE_CQ_QID = {NVME_SQE_CDW10, 0, 16}; // Queue Identifier +constexpr mmio_field32 NVME_CREATE_CQ_IV = {NVME_SQE_CDW11, 16, 16}; // Interrupt Vector +constexpr mmio_field32 NVME_CREATE_CQ_IEN = {NVME_SQE_CDW11, 1, 1}; // Interrupts Enabled +constexpr mmio_field32 NVME_CREATE_CQ_PC = {NVME_SQE_CDW11, 0, 1}; // Physically Contiguous + +constexpr uint8_t NVME_CREATE_SQ_OPCODE = 0x01; +constexpr mmio_field32 NVME_CREATE_SQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size +constexpr mmio_field32 NVME_CREATE_SQ_QID = {NVME_SQE_CDW10, 0, 16}; // Queue Identifier +constexpr mmio_field32 NVME_CREATE_SQ_CQID = {NVME_SQE_CDW11, 16, 16}; // Completion Queue Identifier +constexpr mmio_field32 NVME_CREATE_SQ_QPRIO = {NVME_SQE_CDW11, 1, 2}; // Queue Priority +constexpr mmio_field32 NVME_CREATE_SQ_PC = {NVME_SQE_CDW11, 0, 1}; // Physically Contiguous + +constexpr mmio_field32 NVME_IDENTIFY_CNTID = {NVME_SQE_CDW10, 16, 16}; // Controller Identifier +constexpr mmio_field32 NVME_IDENTIFY_CNS = {NVME_SQE_CDW10, 0, 8}; // Controller or Namespace Structure + +constexpr mmio_field32 NVME_RW_SLBA_LOW = {NVME_SQE_CDW10, 0, 32}; // Starting LBA Lower Bits +constexpr mmio_field32 NVME_RW_SLBA_HIGH = {NVME_SQE_CDW11, 0, 32}; // Starting LBA Higher Bits +constexpr mmio_field32 NVME_RW_LR = {NVME_SQE_CDW12, 31, 1}; // Limited Retry +constexpr mmio_field32 NVME_RW_FUA = {NVME_SQE_CDW12, 30, 1}; // Force Unit Access +constexpr mmio_field32 NVME_RW_PRINFO = {NVME_SQE_CDW12, 26, 4}; // Protection Information Field +constexpr mmio_field32 NVME_RW_NLB = {NVME_SQE_CDW12, 0, 16}; // Number of Logical Blocks + +struct [[__packed]] nvme_cqe { + uint32_t dw0; + uint32_t dw1; + uint32_t dw2; + uint32_t dw3; +}; +static_assert(sizeof(struct nvme_cqe) == 0x10); + +constexpr mmio_reg32 NVME_CQE_DW0 = {offsetof(struct nvme_cqe, dw0), MMIO_LE}; +constexpr mmio_reg32 NVME_CQE_DW2 = {offsetof(struct nvme_cqe, dw2), MMIO_LE}; +constexpr mmio_field32 NVME_CQE_SQ_IDENT = {NVME_CQE_DW2, 16, 16}; +constexpr mmio_field32 NVME_CQE_SQ_HEAD = {NVME_CQE_DW2, 0, 16}; +constexpr mmio_reg32 NVME_CQE_DW3 = {offsetof(struct nvme_cqe, dw3), MMIO_LE}; +constexpr mmio_field32 NVME_CQE_STATUS = {NVME_CQE_DW3, 17, 15}; +constexpr mmio_field32 NVME_CQE_PHASE_TAG = {NVME_CQE_DW3, 16, 1}; +constexpr mmio_field32 NVME_CQE_CID = {NVME_CQE_DW3, 0, 16}; + +constexpr mmio_reg64 NVME_REG_CAP = {0x00, MMIO_LE}; // Controller Capabilities +constexpr mmio_field64 NVME_CAP_MPSMAX = {NVME_REG_CAP, 52, 4}; // Memory Page Size Maximum +constexpr mmio_field64 NVME_CAP_MPSMIN = {NVME_REG_CAP, 48, 4}; // Memory Page Size Minimum +constexpr mmio_field64 NVME_CAP_BPS = {NVME_REG_CAP, 45, 1}; // Boot Partition Support +constexpr mmio_field64 NVME_CAP_CSS = {NVME_REG_CAP, 37, 8}; // Command Sets Supported +constexpr mmio_field64 NVME_CAP_NSSRS = {NVME_REG_CAP, 36, 1}; // NVM Subsystem Reset Supported +constexpr mmio_field64 NVME_CAP_DSTRD = {NVME_REG_CAP, 32, 4}; // Doorbell Stride +constexpr mmio_field64 NVME_CAP_TO = {NVME_REG_CAP, 24, 8}; // Timeout +constexpr mmio_field64 NVME_CAP_AMS = {NVME_REG_CAP, 17, 2}; // Arbitration Mechanism Supported +constexpr mmio_field64 NVME_CAP_CQR = {NVME_REG_CAP, 16, 1}; // Contiguous Queues Required +constexpr mmio_field64 NVME_CAP_MQES = {NVME_REG_CAP, 0, 16}; // Maximum Queue Entries Supported + +constexpr mmio_reg32 NVME_REG_VS = {0x08, MMIO_LE}; // Version +constexpr mmio_field32 NVME_VS_MJR = {NVME_REG_VS, 16, 16}; // Major Version Number +constexpr mmio_field32 NVME_VS_MNR = {NVME_REG_VS, 8, 8}; // Minor Version Number +constexpr mmio_field32 NVME_VS_TER = {NVME_REG_VS, 0, 8}; // Tertiary Version Number + +constexpr mmio_reg32 NVME_REG_INTMS = {0x0C, MMIO_LE}; // Interrupt Mask Set +constexpr mmio_reg32 NVME_REG_INTMC = {0x10, MMIO_LE}; // Interrupt Mask Clear + +constexpr mmio_reg32 NVME_REG_CC = {0x14, MMIO_LE}; // Controller Configuration +constexpr mmio_field32 NVME_CC_IOCQES = {NVME_REG_CC, 20, 4}; // I/O Completion Queue Entry Size +constexpr mmio_field32 NVME_CC_IOSQES = {NVME_REG_CC, 16, 4}; // I/O Submission Queue Entry Size +constexpr mmio_field32 NVME_CC_SHN = {NVME_REG_CC, 14, 2}; // Shutdown Notification +constexpr mmio_field32 NVME_CC_AMS = {NVME_REG_CC, 11, 3}; // Arbitration Mechanism Selected +constexpr mmio_field32 NVME_CC_MPS = {NVME_REG_CC, 7, 4}; // Memory Page Size +constexpr mmio_field32 NVME_CC_CSS = {NVME_REG_CC, 4, 3}; // I/O Command Set Selected +constexpr mmio_field32 NVME_CC_EN = {NVME_REG_CC, 0, 1}; // Enable + +constexpr mmio_reg32 NVME_REG_CSTS = {0x1C, MMIO_LE}; // Controller Status +constexpr mmio_field32 NVME_CSTS_PP = {NVME_REG_CSTS, 5, 1}; // Processing Paused +constexpr mmio_field32 NVME_CSTS_NSSRO = {NVME_REG_CSTS, 4, 1}; // NVM Subsystem Reset Occurred +constexpr mmio_field32 NVME_CSTS_SHST = {NVME_REG_CSTS, 2, 2}; // Shutdown Status +constexpr mmio_field32 NVME_CSTS_CFS = {NVME_REG_CSTS, 1, 1}; // Controller Fatal Status +constexpr mmio_field32 NVME_CSTS_RDY = {NVME_REG_CSTS, 0, 1}; // Ready + +constexpr mmio_reg32 NVME_REG_AQA = {0x24, MMIO_LE}; // Admin Queue Attributes + +constexpr mmio_field32 NVME_AQA_ACQS = {NVME_REG_AQA, 0, 12}; // Admin Completion Queue Size +constexpr mmio_field32 NVME_AQA_ASQS = {NVME_REG_AQA, 16, 12}; // Admin Submission Queue Size + +constexpr mmio_reg64 NVME_REG_ASQ = {0x28, MMIO_LE}; // Admin Submission Queue +constexpr mmio_reg64 NVME_REG_ACQ = {0x30, MMIO_LE}; // Admin Completion Queue + +// Generic Command Set +enum nvme_cmd : uint8_t { + NVME_CMD_FLUSH = 0x00, + NVME_CMD_WRITE = 0x01, + NVME_CMD_READ = 0x02, +}; + +// Admin Command Set +enum nvme_admin_cmd : uint8_t { + NVME_ADMIN_DELETE_SQ = 0x00, + NVME_ADMIN_CREATE_SQ = 0x01, + NVME_ADMIN_DELETE_CQ = 0x04, + NVME_ADMIN_CREATE_CQ = 0x05, + NVME_ADMIN_IDENTIFY = 0x06, + NVME_ADMIN_ABORT = 0x08, + NVME_ADMIN_SET_FEATURES = 0x09, + NVME_ADMIN_GET_FEATURES = 0x0A, +}; diff --git a/dev/pci/core.c b/dev/pci/core.c index dadc76d..730f01b 100644 --- a/dev/pci/core.c +++ b/dev/pci/core.c @@ -4,11 +4,18 @@ #include #include +#include #include #include +#include #include #include +#include #include +#include +#include +#include +#include #include static struct spinlock access_lock = {}; @@ -170,6 +177,119 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) { return 0; } +constexpr uint32_t PCI_MSI_CTRL_ENABLE = 1 << 16; +constexpr uint32_t PCI_MSI_CTRL_MME = 0x7 << 20; +constexpr uint32_t PCI_MSI_CTRL_64BIT = 1 << 23; +constexpr uint32_t PCI_MSIX_CTRL_FUNCTION_MASK = 1 << 30; +constexpr uint32_t PCI_MSIX_CTRL_ENABLE = 1u << 31; + +static void msix_set_state(struct pci_device* dev, uint8_t cap, bool status) { + uint32_t ctrl = pci_read32(dev, cap); + ctrl &= ~(PCI_MSIX_CTRL_ENABLE | PCI_MSIX_CTRL_FUNCTION_MASK); + if (status) + ctrl |= PCI_MSIX_CTRL_ENABLE; + pci_write32(dev, cap, ctrl); +} + +errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out) { + // message address + data + vector control + constexpr size_t ENTRY_SIZE = 8 + 4 + 4; + + const uint8_t cap = pci_cap_find(dev, PCI_CAP_MSIX); + if (cap == 0) + return ENODEV; + + const uint32_t table = pci_read32(dev, cap + 4); + const uint8_t bir = table & 0x7; + const size_t table_offset = table & ~0x7; + + pci_log(dev, "Using MSI-X for this device, table at BAR%hhu + %#zx\n", bir, table_offset); + + // Map the first MSI-X table entry (16 bytes). + struct pci_bar bar; + errno_t status = pci_bar_get(dev, bir, &bar); + if (status) + return ENODEV; + if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64) + return EINVAL; + + const size_t page_size = arch_mem_page_size(); + const phys_t table_phys = bar.address + table_offset; + const phys_t page_base = ALIGN_DOWN(table_phys, page_size); + const size_t page_offset = table_phys - page_base; + const size_t map_len = ALIGN_UP(page_offset + ENTRY_SIZE, page_size); + + uintptr_t vaddr = vm_alloc(map_len); + if (!vaddr) + return ENOMEM; + for (size_t i = 0; i < map_len; i += page_size) { + status = pmap_map(&kernel_vma.pmap, vaddr + i, page_base + i, PTE_READ | PTE_WRITE, CACHE_MMIO); + if (status) + return status; + } + void* entry = (void*)(vaddr + page_offset); + + struct msi_line* msi; + status = arch_irq_alloc_msi(&msi); + if (status) + return status; + irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH); + + // Write entry 0. + const uint64_t msg_addr = msi->msg_addr(msi); + const uint32_t msg_data = msi->msg_data(msi); + mmio_write32(entry + 0x00, (uint32_t)msg_addr); // Address lo + mmio_write32(entry + 0x04, (uint32_t)(msg_addr >> 32)); // Address hi + mmio_write32(entry + 0x08, msg_data); // Data + mmio_write32(entry + 0x0C, 0); // Vector control + + // Enable MSI-X on the device. + msix_set_state(dev, cap, true); + + *out = &msi->line; + return 0; +} + +errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out) { + uint8_t cap = pci_cap_find(dev, PCI_CAP_MSI); + if (cap == 0) + return ENODEV; + + pci_log(dev, "Using MSI for this device\n"); + + struct msi_line* msi; + errno_t status = arch_irq_alloc_msi(&msi); + if (status) + return status; + irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH); + + const uint64_t addr = msi->msg_addr(msi); + const uint16_t data = msi->msg_data(msi); + + uint32_t ctrl = pci_read32(dev, cap); + pci_write32(dev, cap + 0x04, (uint32_t)addr); + uint16_t data_off = 0x08; + if (ctrl & PCI_MSI_CTRL_64BIT) { + pci_write32(dev, cap + 0x08, (uint32_t)(addr >> 32)); + data_off = 0x0C; + } + pci_write32(dev, cap + data_off, data); + + ctrl &= ~PCI_MSI_CTRL_MME; + ctrl |= PCI_MSI_CTRL_ENABLE; + pci_write32(dev, cap, ctrl); + + *out = &msi->line; + return 0; +} + +errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out) { + errno_t status = pci_setup_msix(dev, out); + if (status == ENODEV) + return pci_setup_msi(dev, out); + return status; +} + static void pci_device_name(const struct pci_device* dev, char buf[static 18]) { ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address)); } @@ -192,6 +312,7 @@ static bool scan_function(struct pci_access* access, struct pci_address addr) { dev->address = addr; dev->access = access; + dev->dma_tag = &dma_default_tag; dev->id.vendor_id = vendor_id; dev->id.device_id = device_id; dev->class.prog_if = pci_read8(dev, PCI_PROG_IF); diff --git a/include/zinnia/dma.h b/include/zinnia/dma.h new file mode 100644 index 0000000..f195d8c --- /dev/null +++ b/include/zinnia/dma.h @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include +#include +#include + +typedef uint64_t dma_addr_t; + +constexpr dma_addr_t DMA_ADDR_MAX = UINT64_MAX; + +struct bio; + +struct dma_tag; + +struct dma_ops { + errno_t (*map)(struct dma_tag* tag, phys_t phys, size_t len, dma_addr_t* out); + void (*unmap)(struct dma_tag* tag, dma_addr_t addr, size_t len); +}; + +struct dma_tag { + const struct dma_ops* ops; + void* priv; + dma_addr_t max_addr; +}; + +extern struct dma_tag dma_default_tag; + +void dma_tag_subregion(struct dma_tag* parent, dma_addr_t max_addr, struct dma_tag* out); + +struct dma_seg { + dma_addr_t addr; + size_t len; +}; + +struct dma_buffer { + struct dma_tag* tag; + phys_t phys; + dma_addr_t addr; + void* virt; + size_t len; +}; + +errno_t dma_buffer_alloc(struct dma_tag* tag, size_t len, struct dma_buffer* out); +void dma_buffer_free(struct dma_buffer* buf); + +struct dma_map { + struct dma_tag* tag; + size_t size; + size_t max_segs; + size_t max_seg_size; + size_t boundary; + size_t mapped_len; + size_t num_segs; + struct dma_seg segs[]; +}; + +enum dma_sync { + DMA_SYNC_PREREAD = 1 << 0, + DMA_SYNC_POSTREAD = 1 << 1, + DMA_SYNC_PREWRITE = 1 << 2, + DMA_SYNC_POSTWRITE = 1 << 3, +}; + +errno_t dma_map_create( + struct dma_tag* tag, + size_t size, + size_t max_segs, + size_t max_seg_size, + size_t boundary, + struct dma_map** out +); +void dma_map_destroy(struct dma_map* map); + +errno_t dma_map_load(struct dma_map* map, void* buf, size_t len); +errno_t dma_map_load_bio(struct dma_map* map, struct bio* bio); +errno_t dma_map_load_iter(struct dma_map* map, const iovec_iter_t* iter); +void dma_map_unload(struct dma_map* map); + +void dma_map_sync(struct dma_map* map, size_t offset, size_t len, enum dma_sync ops); +void dma_buffer_sync(struct dma_buffer* buf, size_t offset, size_t len, enum dma_sync ops); diff --git a/include/zinnia/irq.h b/include/zinnia/irq.h index e2badae..c4288ee 100644 --- a/include/zinnia/irq.h +++ b/include/zinnia/irq.h @@ -5,13 +5,16 @@ #include #include #include +#include #include +#include #include #include enum irq_status : uint8_t { IRQ_STATUS_IGNORED, IRQ_STATUS_HANDLED, + IRQ_STATUS_WAKE_THREAD, }; enum irq_polarity : uint8_t { @@ -29,9 +32,17 @@ enum irq_mode : uint8_t { IRQ_MODE_MASKABLE = 1 << 1, }; +struct irq_line; + struct irq_handler { enum irq_status (*raise)(struct irq_handler* self); + void (*thread)(struct irq_handler* self); list_link(struct irq_handler*) next; + + struct irq_line* line; + struct workqueue* wq; + struct work work; + bool holds_mask; }; struct irq_line { @@ -43,6 +54,9 @@ struct irq_line { bool is_busy; list_head(struct irq_handler*) handlers; enum irq_mode mode; + enum irq_trigger_mode trigger; + struct spinlock thread_lock; + size_t threads_pending; }; struct msi_line { @@ -62,6 +76,7 @@ void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum void irq_line_raise(struct irq_line* line); void irq_line_attach(struct irq_line* line, struct irq_handler* handler); void irq_line_detach(struct irq_line* line, struct irq_handler* handler); +errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name); void irq_lock(); void irq_unlock(); diff --git a/include/zinnia/pci.h b/include/zinnia/pci.h index 293e0e3..d39cb87 100644 --- a/include/zinnia/pci.h +++ b/include/zinnia/pci.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -106,6 +107,7 @@ struct pci_device { struct pci_access* access; uint8_t header_type; struct pci_driver* driver; + struct dma_tag* dma_tag; }; uint8_t pci_read8(struct pci_device* dev, uint16_t offset); @@ -137,6 +139,12 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id); constexpr uint16_t PCI_ANY_ID = 0xffff; constexpr uint8_t PCI_ANY_CLASS = 0xff; +struct irq_line; + +errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out); +errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out); +errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out); + // Matches a device by ID and class. struct pci_match { uint16_t vendor_id; diff --git a/lib/irq.c b/lib/irq.c index a459a0b..649593a 100644 --- a/lib/irq.c +++ b/lib/irq.c @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include @@ -39,9 +41,43 @@ void irq_unlock() { void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) { irq_lock(); line->mode = line->set_config(line, trigger, polarity); + line->trigger = trigger; irq_unlock(); } +static bool holds_mask_while_threaded(struct irq_line* line) { + return line->trigger == IRQ_TRIGGER_LEVEL && (line->mode & IRQ_MODE_MASKABLE) && line->mask && line->unmask; +} + +static void wake_thread(struct irq_line* line, struct irq_handler* handler) { + if (holds_mask_while_threaded(line)) { + spin_lock(&line->thread_lock); + if (!handler->holds_mask) { + handler->holds_mask = true; + if (line->threads_pending++ == 0) + line->mask(line); + } + spin_unlock(&line->thread_lock); + } + + work_queue(handler->wq, &handler->work); +} + +static void irq_thread_work(struct work* work) { + struct irq_handler* handler = containerof(work, struct irq_handler, work); + struct irq_line* line = handler->line; + + handler->thread(handler); + + spin_lock(&line->thread_lock); + if (handler->holds_mask) { + handler->holds_mask = false; + if (--line->threads_pending == 0) + line->unmask(line); + } + spin_unlock(&line->thread_lock); +} + void irq_line_raise(struct irq_line* line) { assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled"); @@ -51,8 +87,13 @@ void irq_line_raise(struct irq_line* line) { bool claimed = false; struct irq_handler* handler; list_foreach(handler, &line->handlers, next) { - if (handler->raise(handler) == IRQ_STATUS_HANDLED) + enum irq_status status = handler->raise ? handler->raise(handler) : IRQ_STATUS_WAKE_THREAD; + if (status == IRQ_STATUS_WAKE_THREAD && handler->wq) { + wake_thread(line, handler); claimed = true; + } else if (status != IRQ_STATUS_IGNORED) { + claimed = true; + } } if (!claimed) @@ -83,3 +124,21 @@ void irq_line_detach(struct irq_line* line, struct irq_handler* handler) { while (__atomic_load_n(&line->is_busy, __ATOMIC_ACQUIRE)) arch_cpu_pause(); } + +errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name) { + if (!handler->thread) + return EINVAL; + + char task_name[TASK_NAME_MAX]; + ksnprintf(task_name, sizeof(task_name), "irq/%s", name); + + handler->line = line; + work_init(&handler->work, irq_thread_work); + handler->holds_mask = false; + errno_t status = workqueue_create(task_name, SCHED_PRIO_IRQ, &handler->wq); + if (status) + return status; + + irq_line_attach(line, handler); + return 0; +} diff --git a/mem/dma.c b/mem/dma.c new file mode 100644 index 0000000..7b2cf71 --- /dev/null +++ b/mem/dma.c @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static errno_t dma_noop_map(struct dma_tag*, phys_t phys, size_t, dma_addr_t* out) { + *out = phys; + return 0; +} + +static void dma_noop_unmap(struct dma_tag*, dma_addr_t, size_t) { + // Does nothing. +} + +static const struct dma_ops dma_noop_ops = { + .map = dma_noop_map, + .unmap = dma_noop_unmap, +}; + +struct dma_tag dma_default_tag = { + .ops = &dma_noop_ops, + .max_addr = DMA_ADDR_MAX, +}; + +void dma_tag_subregion(struct dma_tag* parent, dma_addr_t max_addr, struct dma_tag* out) { + *out = *parent; + out->max_addr = MIN(parent->max_addr, max_addr); +} + +static errno_t tag_alloc_flags(struct dma_tag* tag, enum alloc_flags* out) { + if (tag->max_addr == DMA_ADDR_MAX) + *out = 0; + else if (tag->max_addr >= UINT32_MAX) + *out = ALLOC_MEM32; + else if (tag->max_addr >= ((dma_addr_t)1 << 20) - 1) + *out = ALLOC_MEM20; + else + return EINVAL; + return 0; +} + +errno_t dma_buffer_alloc(struct dma_tag* tag, size_t len, struct dma_buffer* out) { + if (len == 0) + return EINVAL; + + enum alloc_flags flags; + errno_t status = tag_alloc_flags(tag, &flags); + if (status) + return status; + + const size_t num_pages = ROUND_UP(len, arch_mem_page_size()); + phys_t phys; + status = mem_phys_alloc(num_pages, flags, &phys); + if (status) + return status; + + const size_t size = num_pages * arch_mem_page_size(); + dma_addr_t addr; + status = tag->ops->map(tag, phys, size, &addr); + if (status) { + mem_phys_free(phys, num_pages); + return status; + } + if (addr + size - 1 > tag->max_addr) { + tag->ops->unmap(tag, addr, size); + mem_phys_free(phys, num_pages); + return ENOMEM; + } + + *out = (struct dma_buffer){ + .tag = tag, + .phys = phys, + .addr = addr, + .virt = HHDM_PTR(phys), + .len = len, + }; + return 0; +} + +void dma_buffer_free(struct dma_buffer* buf) { + if (buf->virt == nullptr) + return; + + const size_t num_pages = ROUND_UP(buf->len, arch_mem_page_size()); + buf->tag->ops->unmap(buf->tag, buf->addr, num_pages * arch_mem_page_size()); + mem_phys_free(buf->phys, num_pages); + *buf = (struct dma_buffer){0}; +} + +errno_t dma_map_create( + struct dma_tag* tag, + size_t size, + size_t max_segs, + size_t max_seg_size, + size_t boundary, + struct dma_map** out +) { + if (max_segs == 0 || max_seg_size == 0) + return EINVAL; + if (boundary & (boundary - 1)) + return EINVAL; + + struct dma_map* map = mem_alloc(sizeof(*map) + max_segs * sizeof(struct dma_seg), 0); + if (map == nullptr) + return ENOMEM; + + map->tag = tag; + map->size = size; + map->max_segs = max_segs; + map->max_seg_size = max_seg_size; + map->boundary = boundary; + map->mapped_len = 0; + map->num_segs = 0; + *out = map; + return 0; +} + +void dma_map_destroy(struct dma_map* map) { + mem_free(map); +} + +static errno_t map_add_range(struct dma_map* map, phys_t phys, size_t len) { + if (len == 0) + return 0; + + dma_addr_t addr; + errno_t status = map->tag->ops->map(map->tag, phys, len, &addr); + if (status) + return status; + + if (addr + len - 1 > map->tag->max_addr) { + map->tag->ops->unmap(map->tag, addr, len); + return EINVAL; + } + + while (len > 0) { + size_t chunk = MIN(len, map->max_seg_size); + if (map->boundary) { + const dma_addr_t limit = ALIGN_DOWN(addr, map->boundary) + map->boundary; + chunk = MIN(chunk, limit - addr); + } + + struct dma_seg* last = map->num_segs ? &map->segs[map->num_segs - 1] : nullptr; + if (last && last->addr + last->len == addr && last->len + chunk <= map->max_seg_size && + (!map->boundary || ALIGN_DOWN(last->addr, map->boundary) == ALIGN_DOWN(addr + chunk - 1, map->boundary))) { + last->len += chunk; + } else { + if (map->num_segs == map->max_segs) { + map->tag->ops->unmap(map->tag, addr, len); + return EFBIG; + } + map->segs[map->num_segs++] = (struct dma_seg){.addr = addr, .len = chunk}; + } + + map->mapped_len += chunk; + addr += chunk; + len -= chunk; + } + return 0; +} + +static errno_t map_add_virt(struct dma_map* map, struct pmap* pmap, uintptr_t vaddr, size_t len) { + const size_t page_size = arch_mem_page_size(); + while (len > 0) { + const size_t offset = vaddr % page_size; + const size_t chunk = MIN(len, page_size - offset); + + phys_t phys; + if (pmap_extract(pmap, vaddr - offset, &phys)) + return EFAULT; + + errno_t status = map_add_range(map, phys + offset, chunk); + if (status) + return status; + + vaddr += chunk; + len -= chunk; + } + return 0; +} + +static errno_t map_load_finish(struct dma_map* map, errno_t status) { + if (status) + dma_map_unload(map); + return status; +} + +errno_t dma_map_load(struct dma_map* map, void* buf, size_t len) { + if (map->num_segs || len > map->size) + return EINVAL; + + return map_load_finish(map, map_add_virt(map, &kernel_vma.pmap, (uintptr_t)buf, len)); +} + +errno_t dma_map_load_bio(struct dma_map* map, struct bio* bio) { + if (map->num_segs) + return EINVAL; + + size_t total = 0; + for (size_t i = 0; i < bio->num_segs; i++) + total += bio->segs[i].len; + if (total > map->size) + return EINVAL; + + errno_t status = 0; + for (size_t i = 0; i < bio->num_segs && !status; i++) + status = map_add_range(map, bio->segs[i].addr, bio->segs[i].len); + + return map_load_finish(map, status); +} + +errno_t dma_map_load_iter(struct dma_map* map, const iovec_iter_t* iter) { + if (map->num_segs || iter->total_len - iter->total_offset > map->size) + return EINVAL; + + struct pmap* pmap = &kernel_vma.pmap; + if (iter->user) + pmap = &percpu_get()->sched.current->space->pmap; + + errno_t status = 0; + size_t offset = iter->current_offset; + for (size_t i = iter->current_idx; i < iter->iovecs_len && !status; i++) { + const iovec_t* iov = &iter->iovecs[i]; + if (offset >= iov->len) { + offset = 0; + continue; + } + + const uintptr_t base = iov->base + offset; + const size_t len = iov->len - offset; + offset = 0; + + if (iter->user && (!arch_mem_addr_is_user(base) || !arch_mem_addr_is_user(base + len - 1))) { + status = EFAULT; + break; + } + + status = map_add_virt(map, pmap, base, len); + } + + return map_load_finish(map, status); +} + +void dma_map_unload(struct dma_map* map) { + for (size_t i = 0; i < map->num_segs; i++) + map->tag->ops->unmap(map->tag, map->segs[i].addr, map->segs[i].len); + map->num_segs = 0; + map->mapped_len = 0; +} + +static void dma_sync(enum dma_sync ops) { + if (ops & (DMA_SYNC_PREREAD | DMA_SYNC_PREWRITE)) + atomic_fence(ATOMIC_RELEASE); + if (ops & (DMA_SYNC_POSTREAD | DMA_SYNC_POSTWRITE)) + atomic_fence(ATOMIC_ACQUIRE); +} + +void dma_map_sync(struct dma_map* map, size_t offset, size_t len, enum dma_sync ops) { + dma_sync(ops); +} + +void dma_buffer_sync(struct dma_buffer* buf, size_t offset, size_t len, enum dma_sync ops) { + dma_sync(ops); +} diff --git a/mem/meson.build b/mem/meson.build index b17bf93..2f2f816 100644 --- a/mem/meson.build +++ b/mem/meson.build @@ -1,5 +1,6 @@ kernel_sources += files( 'amap.c', + 'dma.c', 'init.c', 'page.c', 'pmm.c',