marv7000/zinnia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev/block: Add NVMe driver

Browse Source
This commit is contained in:
Marvin
2026-10-29 01:50:52 +01:00
parent b52a3949a5
commit 49c454dd97
21 changed files with 2132 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/acpi/uacpi_glue.c
+2 -3
View File
@@ -221,7 +221,6 @@ uacpi_status uacpi_kernel_handle_firmware_request(uacpi_firmware_request* req) {
struct acpi_irq {
struct irq_handler handler;
struct irq_line* line;
uacpi_interrupt_handler func;
uacpi_handle ctx;
};
@@ -254,7 +253,7 @@ uacpi_status uacpi_kernel_install_interrupt_handler(
if (!acpi_irq)
return UACPI_STATUS_OUT_OF_MEMORY;
acpi_irq->handler.raise = acpi_irq_raise;
acpi_irq->line = line;
acpi_irq->handler.line = line;
acpi_irq->func = handler;
acpi_irq->ctx = ctx;
@@ -268,7 +267,7 @@ uacpi_status uacpi_kernel_install_interrupt_handler(
uacpi_status uacpi_kernel_uninstall_interrupt_handler(uacpi_interrupt_handler handler, uacpi_handle irq_handle) {
struct acpi_irq* acpi_irq = irq_handle;
struct irq_line* line = acpi_irq->line;
struct irq_line* line = acpi_irq->handler.line;
irq_line_detach(line, &acpi_irq->handler);
if (list_is_empty(&line->handlers) && line->mask)
line->mask(line);
dev/block/meson.build
+1 -1
View File
@@ -6,6 +6,6 @@ kernel_sources += files(
'ram.c',
)
if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
if kconfig.get('CONFIG_NVME', 'n') != 'n'
subdir('nvme')
endif
dev/block/nvme/Kconfig
+2 -11
View File
@@ -1,13 +1,4 @@
menu "NVMe Support"
config NVME_CORE
bool
default y
config NVME_PCI
bool "NVMe via PCI transport"
config NVME
bool "NVMe block devices"
default y
depends on PCI
depends on NVME_CORE
endmenu
dev/block/nvme/command.c
+82
View File
@@ -0,0 +1,82 @@
// SPDX-License-Identifier: ISC
#include <zinnia/mmio.h>
#include "nvme.h"
void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe) {
uint32_t cdw0 = mmio_field(NVME_SQE_OPC, cmd->do_write ? NVME_CMD_WRITE : NVME_CMD_READ) |
mmio_field(NVME_SQE_CID, cmd->cid) | mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
uint32_t cdw10 = mmio_field(NVME_RW_SLBA_LOW, (uint32_t)cmd->start_lba);
uint32_t cdw11 = mmio_field(NVME_RW_SLBA_HIGH, (uint32_t)(cmd->start_lba >> 32));
uint32_t cdw12 = mmio_field(NVME_RW_NLB, (uint16_t)(cmd->num_lbas - 1));
uint32_t cdw13 = 0;
uint32_t cdw14 = 0;
uint32_t cdw15 = 0;
mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->prp1);
mmio_reg_write(sqe, NVME_SQE_DPTR1, cmd->prp2);
mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
mmio_reg_write(sqe, NVME_SQE_CDW12, cdw12);
mmio_reg_write(sqe, NVME_SQE_CDW13, cdw13);
mmio_reg_write(sqe, NVME_SQE_CDW14, cdw14);
mmio_reg_write(sqe, NVME_SQE_CDW15, cdw15);
}
void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe) {
uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_CMD_FLUSH) | mmio_field(NVME_SQE_CID, cmd->cid);
mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
}
void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe) {
uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_CQ) |
mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
uint32_t cdw10 = mmio_field(NVME_CREATE_CQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
mmio_field(NVME_CREATE_CQ_QID, (uint16_t)cmd->qid);
uint32_t cdw11 = mmio_field(NVME_CREATE_CQ_IV, cmd->irq_vector) |
mmio_field(NVME_CREATE_CQ_IEN, cmd->irqs_enabled ? 1 : 0) |
mmio_field(NVME_CREATE_CQ_PC, 1); // Our buffer is physically contiguous.
mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->cq_addr);
mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
}
void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe) {
uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_SQ) |
mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
uint32_t cdw10 = mmio_field(NVME_CREATE_SQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
mmio_field(NVME_CREATE_SQ_QID, (uint16_t)cmd->qid);
uint32_t cdw11 = mmio_field(NVME_CREATE_SQ_CQID, (uint16_t)cmd->cqid) |
mmio_field(NVME_CREATE_SQ_QPRIO, 0) | // Priority is the highest.
mmio_field(NVME_CREATE_SQ_PC, 1); // Our buffer is physically contiguous.
mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->sq_addr);
mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
}
void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe) {
uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_IDENTIFY) |
mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
uint32_t cdw10 = mmio_field(NVME_IDENTIFY_CNTID, cmd->controller_id) | mmio_field(NVME_IDENTIFY_CNS, cmd->cns);
mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->buffer);
mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
}
dev/block/nvme/controller.c
+388
View File
@@ -0,0 +1,388 @@
// SPDX-License-Identifier: ISC
#define pr_fmt(fmt) "nvme: " fmt
#include <zinnia/alloc.h>
#include <zinnia/clock.h>
#include <zinnia/mmio.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <arch/mem.h>
#include <string.h>
#include "nvme.h"
constexpr size_t ADMIN_QUEUE_SIZE = 32;
constexpr size_t IO_QUEUE_SIZE_MAX = 1024;
constexpr size_t IDENTIFY_SIZE = 4096;
constexpr size_t NS_LIST_LEN = 1024;
constexpr mmio_reg8 ID_MDTS = {77, MMIO_LE};
constexpr mmio_reg32 ID_NN = {516, MMIO_LE};
constexpr mmio_reg64 ID_NSZE = {0, MMIO_LE};
constexpr mmio_reg8 ID_FLBAS = {26, MMIO_LE};
constexpr mmio_field8 ID_LBAF_IDX = {ID_FLBAS, 0, 4};
constexpr mmio_reg32 ID_LBAF0 = {128, MMIO_LE};
constexpr mmio_field32 ID_LBADS = {ID_LBAF0, 16, 8};
size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl) {
const size_t cap = 512 * arch_mem_page_size();
if (ctrl->mdts != 0)
return MIN(ctrl->mdts, cap);
return cap;
}
errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out) {
// Read controller version.
uint32_t vs = mmio_reg_read(regs, NVME_REG_VS);
log("NVMe controller version %u.%u.%u\n",
mmio_field_get(NVME_VS_MJR, vs),
mmio_field_get(NVME_VS_MNR, vs),
mmio_field_get(NVME_VS_TER, vs));
uint64_t cap = mmio_reg_read(regs, NVME_REG_CAP);
// Check if our host page size is supported.
const size_t mps_max = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMAX, cap) + 12);
const size_t mps_min = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMIN, cap) + 12);
const size_t page_size = arch_mem_page_size();
if (page_size < mps_min || page_size > mps_max) {
log_err("Host page size is not supported on this NVMe!\n");
return ENOTSUP;
}
struct nvme_ctrl* ctrl = mem_alloc(sizeof(*ctrl), 0);
if (!ctrl)
return ENOMEM;
ctrl->regs = regs;
ctrl->dma_tag = dma_tag;
ctrl->doorbell_stride = (size_t)4 << mmio_field_get(NVME_CAP_DSTRD, cap);
ctrl->mps_min = mps_min;
ctrl->mps = __builtin_ctzl(page_size) - 12;
mutex_init(&ctrl->admin_lock);
spin_init(&ctrl->io_lock);
atomic_init(&ctrl->ns_count, 0);
*out = ctrl;
return 0;
}
// Sets the CC.EN flag.
static errno_t set_status(struct nvme_ctrl* ctrl, bool enable) {
uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
// CAP.TO is in 500ms units.
const uint64_t timeout = MAX((uint64_t)mmio_field_get(NVME_CAP_TO, cap), 1) * 500'000'000;
const uint64_t deadline = clock_get_elapsed_ns() + timeout;
uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
cc &= ~mmio_field(NVME_CC_EN, UINT32_MAX);
cc |= mmio_field(NVME_CC_EN, enable ? 1 : 0);
mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
while (true) {
uint32_t csts = mmio_reg_read(ctrl->regs, NVME_REG_CSTS);
if (mmio_field_get(NVME_CSTS_CFS, csts) != 0)
return EIO;
bool ready = mmio_field_get(NVME_CSTS_RDY, csts) != 0;
if (ready == enable)
break;
if (clock_get_elapsed_ns() >= deadline)
return ETIMEDOUT;
}
return 0;
}
static errno_t admin_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe, struct nvme_completion* out) {
errno_t status = nvme_queue_submit_cmd(queue, sqe);
if (status)
return status;
return nvme_queue_next_completion(queue, out);
}
static errno_t identify(struct nvme_queue* queue, const struct nvme_identify_cmd* cmd) {
struct nvme_sqe sqe = {};
nvme_identify_cmd_write(cmd, &sqe);
struct nvme_completion comp;
errno_t status = admin_cmd(queue, &sqe, &comp);
if (status)
return status;
if (!nvme_status_is_success(comp.status)) {
log_err("Identification failed with status %x\n", comp.status);
return EIO;
}
return 0;
}
errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector) {
// Disable the controller.
errno_t status = set_status(ctrl, false);
if (status)
return status;
uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
const size_t queue_depth = MIN((size_t)mmio_field_get(NVME_CAP_MQES, cap) + 1, IO_QUEUE_SIZE_MAX);
// Create an admin queue first so we can create more queues.
struct nvme_queue* admin_queue;
status = nvme_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 0, ADMIN_QUEUE_SIZE, &admin_queue);
if (status)
return status;
// Set the admin queue sizes.
uint32_t aqa = 0;
aqa |= mmio_field(NVME_AQA_ACQS, admin_queue->depth - 1);
aqa |= mmio_field(NVME_AQA_ASQS, admin_queue->depth - 1);
mmio_reg_write(ctrl->regs, NVME_REG_AQA, aqa);
// Set the addresses to the admin submission and completion queue.
mmio_reg_write(ctrl->regs, NVME_REG_ASQ, admin_queue->sq.addr);
mmio_reg_write(ctrl->regs, NVME_REG_ACQ, admin_queue->cq.addr);
uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
cc &=
~(mmio_field(NVME_CC_MPS, UINT32_MAX) | mmio_field(NVME_CC_IOCQES, UINT32_MAX) |
mmio_field(NVME_CC_IOSQES, UINT32_MAX));
cc |= mmio_field(NVME_CC_MPS, ctrl->mps);
cc |= mmio_field(NVME_CC_IOCQES, __builtin_ctzl(sizeof(struct nvme_cqe)));
cc |= mmio_field(NVME_CC_IOSQES, __builtin_ctzl(sizeof(struct nvme_sqe)));
mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
// Re-enable the controller.
status = set_status(ctrl, true);
if (status)
goto fail_admin;
struct nvme_io_queue* io_queue;
status =
nvme_io_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 1, queue_depth, !irqs_enabled, &io_queue);
if (status)
goto fail_admin;
struct nvme_sqe sqe = {};
struct nvme_completion comp;
nvme_create_cq_cmd_write(
&(struct nvme_create_cq_cmd){
.cq_addr = io_queue->cq.addr,
.depth = io_queue->depth,
.qid = io_queue->queue_id,
.irqs_enabled = irqs_enabled,
.irq_vector = irqs_enabled ? io_irq_vector : 0,
},
&sqe
);
status = admin_cmd(admin_queue, &sqe, &comp);
if (!status && !nvme_status_is_success(comp.status))
status = EIO;
if (status)
goto fail_io;
sqe = (struct nvme_sqe){};
nvme_create_sq_cmd_write(
&(struct nvme_create_sq_cmd){
.sq_addr = io_queue->sq.addr,
.depth = io_queue->depth,
.qid = io_queue->queue_id,
.cqid = io_queue->queue_id,
},
&sqe
);
status = admin_cmd(admin_queue, &sqe, &comp);
if (!status && !nvme_status_is_success(comp.status))
status = EIO;
if (status)
goto fail_io;
mutex_lock(&ctrl->admin_lock);
struct nvme_queue* old_admin = ctrl->admin_queue;
ctrl->admin_queue = admin_queue;
mutex_unlock(&ctrl->admin_lock);
if (old_admin)
nvme_queue_free(old_admin);
spin_lock(&ctrl->io_lock);
ctrl->io_queue = io_queue;
spin_unlock(&ctrl->io_lock);
return 0;
fail_io:
nvme_io_queue_free(io_queue);
fail_admin:
nvme_queue_free(admin_queue);
return status;
}
static void trim_ascii_end(char* str, size_t len) {
while (len > 0 &&
(str[len - 1] == ' ' || str[len - 1] == '\t' || str[len - 1] == '\n' || str[len - 1] == '\r' ||
str[len - 1] == '\f'))
len--;
str[len] = '\0';
}
errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl) {
struct dma_buffer identify_buffer;
errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
if (status)
return ENOMEM;
mutex_lock(&ctrl->admin_lock);
if (!ctrl->admin_queue) {
status = ENXIO;
goto out;
}
status = identify(
ctrl->admin_queue,
&(struct nvme_identify_cmd){
.buffer = identify_buffer.addr,
.controller_id = 0, // TODO: Use a real number here
.cns = 1,
.nsid = 0,
}
);
// Check if the identification was successful.
if (status)
goto out;
dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
// Read the model strings.
const uint8_t* view = identify_buffer.virt;
memcpy(ctrl->serial, view + 4, sizeof(ctrl->serial) - 1);
memcpy(ctrl->model, view + 24, sizeof(ctrl->model) - 1);
memcpy(ctrl->revision, view + 64, sizeof(ctrl->revision) - 1);
trim_ascii_end(ctrl->serial, sizeof(ctrl->serial) - 1);
trim_ascii_end(ctrl->model, sizeof(ctrl->model) - 1);
trim_ascii_end(ctrl->revision, sizeof(ctrl->revision) - 1);
log("Model \"%s\"\n", ctrl->model);
atomic_store(&ctrl->ns_count, mmio_reg_read(identify_buffer.virt, ID_NN), ATOMIC_RELEASE);
uint8_t mdts = mmio_reg_read(identify_buffer.virt, ID_MDTS);
if (mdts != 0 && mdts < sizeof(size_t) * 8)
ctrl->mdts = ((size_t)1 << mdts) * ctrl->mps_min;
out:
mutex_unlock(&ctrl->admin_lock);
dma_buffer_free(&identify_buffer);
return status;
}
// Scans all existing namespaces.
errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out) {
vec_init(out);
uint32_t* ns_list = mem_alloc(NS_LIST_LEN * sizeof(uint32_t), 0);
if (!ns_list)
return ENOMEM;
struct dma_buffer identify_buffer;
errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
if (status) {
mem_free(ns_list);
return ENOMEM;
}
mutex_lock(&ctrl->admin_lock);
struct nvme_queue* aq = ctrl->admin_queue;
if (!aq) {
status = ENXIO;
goto out;
}
uint32_t nsid = 0;
while (true) {
const struct nvme_identify_cmd admin_ident_cmd = {
.buffer = identify_buffer.addr,
.controller_id = 0,
.cns = 2,
.nsid = nsid,
};
status = identify(aq, &admin_ident_cmd);
if (status)
goto out;
dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
// Parse the identifier list until we hit an namespace value of 0.
size_t ns_list_len = 0;
const uint32_t* raw_list = identify_buffer.virt;
while (ns_list_len < NS_LIST_LEN && raw_list[ns_list_len] != 0) {
ns_list[ns_list_len] = raw_list[ns_list_len];
ns_list_len++;
}
for (size_t i = 0; i < ns_list_len; i++) {
const uint32_t id = ns_list[i];
const struct nvme_identify_cmd ns_ident_cmd = {
.buffer = identify_buffer.addr,
.controller_id = 0,
.cns = 0,
.nsid = id,
};
status = identify(aq, &ns_ident_cmd);
if (status)
goto out;
dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
const uint64_t lba_count = mmio_reg_read(identify_buffer.virt, ID_NSZE);
// To get the block size of a namespace, we need to read a descriptor (see LBAF0-15).
// Get the format descriptor index used by this namespace.
const uint8_t lba_format_idx = mmio_field_get(ID_LBAF_IDX, mmio_reg_read(identify_buffer.virt, ID_FLBAS));
const mmio_reg32 lbaf = {ID_LBAF0.offset + lba_format_idx * sizeof(uint32_t), MMIO_LE};
const uint32_t lba_format = mmio_reg_read(identify_buffer.virt, lbaf);
uint8_t lba_shift = mmio_field_get(ID_LBADS, lba_format);
// If LBADS doesn't contain a non-zero value, we need to assume a default. Use 512 bytes here.
if (lba_shift == 0)
lba_shift = 9;
struct nvme_ns* ns;
status = nvme_ns_new(ctrl, id, lba_shift, lba_count, nvme_ctrl_max_transfer_bytes(ctrl), &ns);
if (status)
goto out;
status = vec_push(out, ns);
if (status) {
blockdev_unref(&ns->blk);
goto out;
}
}
// If there is no 0 in the buffer, there's more entries than can fit in `identify_buffer`.
if (ns_list_len < NS_LIST_LEN)
break;
nsid = ns_list[ns_list_len - 1];
}
out:
mutex_unlock(&ctrl->admin_lock);
dma_buffer_free(&identify_buffer);
mem_free(ns_list);
if (status) {
vec_foreach(out, i) {
blockdev_unref(&vec_elem_get(out, i)->blk);
}
vec_destroy(out);
}
return status;
}
struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl) {
spin_lock(&ctrl->io_lock);
struct nvme_io_queue* queue = ctrl->io_queue;
spin_unlock(&ctrl->io_lock);
return queue;
}
dev/block/nvme/core.c
-41
View File
@@ -1,41 +0,0 @@
#define pr_fmt(fmt) "nvme: " fmt
#include <zinnia/fdt.h>
#include <zinnia/module.h>
#include <zinnia/pci.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <config.h>
#if CONFIG_NVME_PCI
#include "nvme_pci.h"
#endif
struct nvme_dev {
#if CONFIG_NVME_PCI
struct pci_device pci_dev;
#endif
};
errno_t nvme_init(struct module* mod) {
errno_t status = 0;
#if CONFIG_NVME_PCI
status = pci_driver_register(&nvme_driver);
#endif
return status;
}
errno_t nvme_exit(struct module* mod) {
// TODO: Unregister
return 0;
}
MODULE_ENTRY(nvme_init);
MODULE_EXIT(nvme_exit);
MODULE_DESC("NVMe Block Devices");
MODULE_AUTHOR("Marvin Friedrich");
MODULE_LICENSE("ISC");
MODULE_VERSION("1.0");
dev/block/nvme/meson.build
+9 -8
View File
@@ -1,12 +1,13 @@
nvme_sources = files('core.c')
if kconfig.get('CONFIG_NVME_PCI', 'n') == 'y'
nvme_sources += files('pci.c')
endif
modules += {
'nvme': {
'modular': kconfig.get('CONFIG_NVME_CORE') == 'm',
'sources': nvme_sources,
'modular': kconfig.get('CONFIG_NVME') == 'm',
'sources': files(
'command.c',
'controller.c',
'namespace.c',
'pci.c',
'prp.c',
'queue.c',
),
},
}
dev/block/nvme/namespace.c
+117
View File
@@ -0,0 +1,117 @@
// SPDX-License-Identifier: ISC
#define pr_fmt(fmt) "nvme: " fmt
#include <zinnia/alloc.h>
#include <zinnia/clock.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <stdckdint.h>
#include "nvme.h"
static errno_t nvme_ns_submit(struct blockdev* blk, struct bio* bio) {
struct nvme_ns* ns = containerof(blk, struct nvme_ns, blk);
if (bio->op != BIO_FLUSH) {
uint64_t end_lba;
if (ckd_add(&end_lba, bio->lba, (uint64_t)bio->num_lbas)) {
bio_complete(bio, EOVERFLOW, 0);
return 0;
}
if (end_lba > ns->lba_count) {
bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
return 0;
}
}
struct nvme_io_queue* queue = nvme_ctrl_io_queue(ns->ctrl);
if (!queue) {
bio_complete(bio, EIO, 0);
return 0;
}
size_t bytes = 0;
for (size_t i = 0; i < bio->num_segs; i++)
bytes += bio->segs[i].len;
_Atomic(bool) done = false;
const bool poll = queue->poll;
uint16_t cid = nvme_io_queue_submit(
queue,
bio,
(struct nvme_rw_cmd){
.prp1 = 0,
.prp2 = 0,
.cid = 0,
.do_write = bio->op == BIO_WRITE,
.start_lba = bio->lba,
.num_lbas = bio->num_lbas,
.bytes = bytes,
.control = 0,
.ds_mgmt = 0,
.ref_tag = 0,
.app_tag = 0,
.app_mask = 0,
.nsid = ns->nsid,
},
poll ? &done : nullptr
);
// If we're not interrupt-driven, poll the IO completion queue.
if (poll) {
const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
while (!atomic_load(&done, ATOMIC_ACQUIRE)) {
nvme_io_queue_drain(queue);
if (atomic_load(&done, ATOMIC_ACQUIRE))
break;
if (clock_get_elapsed_ns() >= deadline && nvme_io_queue_abandon(queue, cid, bio)) {
bio_complete(bio, EIO, 0);
break;
}
}
}
return 0;
}
static void nvme_ns_destroy(struct blockdev* blk) {
mem_free(containerof(blk, struct nvme_ns, blk));
}
static const struct blockdev_ops nvme_ns_ops = {
.submit = nvme_ns_submit,
.destroy = nvme_ns_destroy,
};
errno_t nvme_ns_new(
struct nvme_ctrl* ctrl,
uint32_t nsid,
uint8_t lba_shift,
uint64_t lba_count,
size_t max_transfer_bytes,
struct nvme_ns** out
) {
log("Namespace %u: LBA size %zu bytes, %zu MBs total\n",
nsid,
(size_t)1 << lba_shift,
(size_t)(lba_count << lba_shift) / 1024 / 1024);
struct nvme_ns* ns = mem_alloc(sizeof(*ns), 0);
if (!ns)
return ENOMEM;
blockdev_init(&ns->blk, &nvme_ns_ops, NVME_MAJOR, nsid, (size_t)1 << lba_shift, lba_count);
ns->blk.limits = (struct blockdev_limits){
.max_lbas = MAX(max_transfer_bytes >> lba_shift, 1),
.max_segs = 512,
};
ns->ctrl = ctrl;
ns->nsid = nsid;
ns->lba_shift = lba_shift;
ns->lba_count = lba_count;
ns->max_transfer_bytes = max_transfer_bytes;
*out = ns;
return 0;
}
dev/block/nvme/nvme.h
+221
View File
@@ -0,0 +1,221 @@
// SPDX-License-Identifier: ISC
#pragma once
#include <zinnia/atomic.h>
#include <zinnia/blockdev.h>
#include <zinnia/dma.h>
#include <zinnia/errno.h>
#include <zinnia/event.h>
#include <zinnia/mutex.h>
#include <zinnia/pci.h>
#include <zinnia/spin.h>
#include <zinnia/vec.h>
#include <stddef.h>
#include <stdint.h>
#include "spec.h"
constexpr uint32_t NVME_MAJOR = 159;
constexpr uint64_t NVME_COMPLETION_TIMEOUT_NS = 30'000'000'000;
// An entry in the completion queue.
struct nvme_completion {
uint32_t result;
uint16_t sq_head;
uint16_t sq_id;
uint16_t cmd_id;
bool phase_tag;
uint16_t status;
};
static inline bool nvme_status_is_success(uint16_t status) {
return status == 0;
}
struct nvme_rw_cmd {
uint64_t prp1;
uint64_t prp2;
uint16_t cid;
bool do_write;
uint64_t start_lba;
size_t num_lbas;
size_t bytes;
uint16_t control;
uint32_t ds_mgmt;
uint32_t ref_tag;
uint16_t app_tag;
uint16_t app_mask;
uint32_t nsid;
};
struct nvme_flush_cmd {
uint16_t cid;
uint32_t nsid;
};
struct nvme_create_cq_cmd {
dma_addr_t cq_addr;
size_t depth;
size_t qid;
bool irqs_enabled;
uint16_t irq_vector;
};
struct nvme_create_sq_cmd {
dma_addr_t sq_addr;
size_t depth;
size_t qid;
size_t cqid;
};
struct nvme_identify_cmd {
dma_addr_t buffer;
uint16_t controller_id;
uint8_t cns;
uint32_t nsid;
};
void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe);
void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe);
void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe);
void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe);
void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe);
struct nvme_queue {
// Amount of queue entries.
size_t depth;
size_t doorbells_offset;
size_t doorbell_stride;
void* regs;
// Physical buffer for the completion queue.
struct dma_buffer cq;
// The index of the current completion queue entry.
size_t cq_head;
// Determines whether a completion queue entry is new.
uint8_t cq_phase;
// Physical buffer for the submission queue.
struct dma_buffer sq;
// The index of the current submission queue entry.
size_t sq_tail;
};
errno_t nvme_queue_new(
struct dma_tag* tag,
void* regs,
size_t doorbell_stride,
size_t queue_id,
size_t depth,
struct nvme_queue** out
);
void nvme_queue_free(struct nvme_queue* queue);
errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe);
errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out);
struct nvme_inflight {
bool active;
struct bio* bio;
size_t lbas;
struct dma_map* map;
struct dma_buffer prps;
_Atomic(bool)* done;
};
struct nvme_io_queue {
size_t queue_id;
size_t depth;
size_t doorbells_offset;
size_t doorbell_stride;
void* regs;
bool poll;
struct dma_tag* tag;
struct dma_buffer cq;
struct dma_buffer sq;
struct spinlock sq_lock;
size_t sq_tail;
struct spinlock cq_lock;
size_t cq_head;
uint8_t cq_phase;
struct spinlock cmds_lock;
struct nvme_inflight* slots;
uint16_t* free;
size_t num_free;
struct event slots_free;
};
errno_t nvme_io_queue_new(
struct dma_tag* tag,
void* regs,
size_t doorbell_stride,
size_t queue_id,
size_t depth,
bool poll,
struct nvme_io_queue** out
);
void nvme_io_queue_free(struct nvme_io_queue* queue);
uint16_t nvme_io_queue_submit(
struct nvme_io_queue* queue,
struct bio* bio,
struct nvme_rw_cmd cmd,
_Atomic(bool)* done
);
bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio);
size_t nvme_io_queue_drain(struct nvme_io_queue* queue);
errno_t nvme_build_prps(
struct dma_tag* tag,
struct bio* bio,
struct nvme_inflight* slot,
uint64_t* prp1,
uint64_t* prp2
);
void nvme_release_prps(struct nvme_inflight* slot);
struct nvme_ns;
struct nvme_ctrl {
void* regs;
struct dma_tag* dma_tag;
size_t doorbell_stride; // The amount of bytes between two doorbells.
size_t mps_min;
uint8_t mps;
char serial[20 + 1]; // Serial number of the controller.
char model[40 + 1]; // Model name of the controller.
char revision[8 + 1]; // Revision of the controller's firmware.
struct nvme_queue* admin_queue; // Queue to submit admin commands to.
mutex_t admin_lock;
struct nvme_io_queue* io_queue; // Queue to submit IO commands to.
struct spinlock io_lock;
_Atomic(uint32_t) ns_count; // Amount of namespaces.
size_t mdts; // Maximum data transfer size.
};
typedef vec(struct nvme_ns*) nvme_ns_vec_t;
errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out);
size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl);
errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector);
errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl);
errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out);
struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl);
struct nvme_ns {
struct blockdev blk;
struct nvme_ctrl* ctrl;
uint32_t nsid;
uint8_t lba_shift;
uint64_t lba_count;
size_t max_transfer_bytes;
};
errno_t nvme_ns_new(
struct nvme_ctrl* ctrl,
uint32_t nsid,
uint8_t lba_shift,
uint64_t lba_count,
size_t max_transfer_bytes,
struct nvme_ns** out
);
dev/block/nvme/nvme_pci.h
-5
View File
@@ -1,5 +0,0 @@
// SPDX-License-Identifier: ISC
#pragma once
extern struct pci_driver nvme_driver;
dev/block/nvme/pci.c
+136 -4
View File
@@ -1,13 +1,128 @@
// SPDX-License-Identifier: ISC
#define pr_fmt(fmt) "nvme: " fmt
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/errno.h>
#include <zinnia/irq.h>
#include <zinnia/module.h>
#include <zinnia/pci.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <zinnia/virt.h>
#include <zinnia/vma.h>
#include <arch/mem.h>
#include "nvme.h"
static _Atomic(size_t) nvme_counter;
struct nvme_irq_handler {
struct irq_handler handler;
struct nvme_ctrl* controller;
};
static enum irq_status nvme_irq_raise(struct irq_handler* self) {
struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler);
struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller);
if (queue)
nvme_io_queue_drain(queue);
return IRQ_STATUS_HANDLED;
}
static errno_t map_bar(struct pci_bar* bar, void** out) {
const size_t page_size = arch_mem_page_size();
const size_t len = ALIGN_UP(bar->size, page_size);
uintptr_t vaddr = vm_alloc(len);
if (!vaddr)
return ENOMEM;
for (size_t i = 0; i < len; i += page_size) {
errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, bar->address + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
if (status)
return status;
}
*out = (void*)vaddr;
return 0;
}
static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
pci_log(dev, "Detected controller \n");
pci_log(dev, "Probing NVMe device\n");
return 0;
// Enable MMIO decoding and DMA. We only support MSI-X or polling, so keep legacy INTx disabled.
uint16_t cmd = pci_read16(dev, PCI_COMMAND);
pci_write16(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEM | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE);
struct irq_line* irq_line = nullptr;
if (pci_setup_msix(dev, &irq_line)) {
pci_log_warn(dev, "NVMe MSI-X setup failed, falling back to polling completions\n");
irq_line = nullptr;
}
struct pci_bar bar;
errno_t status = pci_bar_get(dev, 0, &bar);
if (status)
return ENXIO;
if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
return ENXIO;
void* regs;
status = map_bar(&bar, &regs);
if (status)
return status;
struct nvme_ctrl* controller;
status = nvme_ctrl_new_pci(regs, dev->dma_tag, &controller);
if (status) {
pci_log_err(dev, "Failed to probe controller: %i\n", status);
return ENODEV;
}
if (irq_line) {
struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0);
if (!handler)
return ENOMEM;
handler->handler.raise = nvme_irq_raise;
handler->controller = controller;
irq_line_attach(irq_line, &handler->handler);
if (irq_line->unmask)
irq_line->unmask(irq_line);
}
// Reset the controller to initialize all queues and other structures.
status = nvme_ctrl_reset(controller, irq_line != nullptr, 0);
if (status) {
pci_log_err(dev, "Failed to reset controller: %i\n", status);
return ENODEV;
}
status = nvme_ctrl_identify(controller);
if (status) {
pci_log_err(dev, "Failed to identify controller: %i\n", status);
return ENODEV;
}
nvme_ns_vec_t namespaces;
status = nvme_ctrl_scan_namespaces(controller, &namespaces);
if (status) {
pci_log_err(dev, "Failed to identify controller: %i\n", status);
return ENODEV;
}
const size_t nvme_id = atomic_fetch_add(&nvme_counter, 1, ATOMIC_SEQ_CST);
vec_foreach(&namespaces, i) {
struct nvme_ns* ns = vec_elem_get(&namespaces, i);
char path[32];
ksnprintf(path, sizeof(path), "nvme%zun%u", nvme_id, ns->nsid);
status = blockdev_register(path, &ns->blk);
if (status)
break;
}
vec_destroy(&namespaces);
return status;
}
static const struct pci_match nvme_match[] = {
@@ -17,9 +132,26 @@ static const struct pci_match nvme_match[] = {
MATCH_TABLE("pci", nvme_match);
struct pci_driver nvme_driver = {
.name = "nvme_pci",
static struct pci_driver nvme_driver = {
.name = "nvme",
.match = nvme_match,
.num_matches = ARRAY_SIZE(nvme_match),
.probe = nvme_pci_probe,
};
errno_t nvme_init(struct module* mod) {
return pci_driver_register(&nvme_driver);
}
errno_t nvme_exit(struct module* mod) {
// TODO: Unregister
return 0;
}
MODULE_ENTRY(nvme_init);
MODULE_EXIT(nvme_exit);
MODULE_DESC("NVMe Block Devices");
MODULE_AUTHOR("Marvin Friedrich");
MODULE_LICENSE("ISC");
MODULE_VERSION("1.0");
dev/block/nvme/prp.c
+118
View File
@@ -0,0 +1,118 @@
// SPDX-License-Identifier: ISC
#include <zinnia/utils.h>
#include <arch/mem.h>
#include "nvme.h"
static size_t count_pages(struct dma_map* map, size_t page_size) {
size_t count = 0;
for (size_t i = 0; i < map->num_segs; i++) {
const struct dma_seg* seg = &map->segs[i];
if (seg->len == 0)
continue;
const dma_addr_t end = seg->addr + seg->len;
for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size)
count++;
}
return count;
}
static void fill_pages(struct dma_map* map, size_t page_size, uint64_t* out) {
bool first_page = true;
size_t idx = 0;
for (size_t i = 0; i < map->num_segs; i++) {
const struct dma_seg* seg = &map->segs[i];
if (seg->len == 0)
continue;
const dma_addr_t end = seg->addr + seg->len;
for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size) {
if (first_page)
first_page = false;
else
out[idx++] = page;
}
}
}
errno_t nvme_build_prps(
struct dma_tag* tag,
struct bio* bio,
struct nvme_inflight* slot,
uint64_t* prp1,
uint64_t* prp2
) {
const size_t page_size = arch_mem_page_size();
slot->map = nullptr;
slot->prps = (struct dma_buffer){};
if (bio->num_segs == 0)
return EIO;
errno_t status = dma_map_create(tag, SIZE_MAX, bio->num_segs, SIZE_MAX, 0, &slot->map);
if (status)
return status;
status = dma_map_load_bio(slot->map, bio);
if (status)
goto fail;
if (slot->map->num_segs == 0) {
status = EIO;
goto fail;
}
dma_map_sync(slot->map, 0, slot->map->mapped_len, bio->op == BIO_WRITE ? DMA_SYNC_PREWRITE : DMA_SYNC_PREREAD);
*prp1 = slot->map->segs[0].addr;
// Enumerate the base of every page the transfer touches in order.
const size_t pages = count_pages(slot->map, page_size);
const size_t rest_pages = pages ? pages - 1 : 0;
if (rest_pages == 0) {
*prp2 = 0;
return 0;
}
if (rest_pages == 1) {
uint64_t second;
fill_pages(slot->map, page_size, &second);
*prp2 = second;
return 0;
}
// We don't want to chain across pages.
const size_t entries_per_page = page_size / sizeof(uint64_t);
if (rest_pages > entries_per_page) {
status = ENOMEM;
goto fail;
}
status = dma_buffer_alloc(tag, page_size, &slot->prps);
if (status)
goto fail;
fill_pages(slot->map, page_size, slot->prps.virt);
dma_buffer_sync(&slot->prps, 0, rest_pages * sizeof(uint64_t), DMA_SYNC_PREWRITE);
*prp2 = slot->prps.addr;
return 0;
fail:
nvme_release_prps(slot);
return status;
}
void nvme_release_prps(struct nvme_inflight* slot) {
if (slot->map) {
dma_map_sync(
slot->map,
0,
slot->map->mapped_len,
slot->bio && slot->bio->op == BIO_WRITE ? DMA_SYNC_POSTWRITE : DMA_SYNC_POSTREAD
);
dma_map_unload(slot->map);
dma_map_destroy(slot->map);
slot->map = nullptr;
}
dma_buffer_free(&slot->prps);
}
dev/block/nvme/queue.c
+346
View File
@@ -0,0 +1,346 @@
// SPDX-License-Identifier: ISC
#define pr_fmt(fmt) "nvme: " fmt
#include <zinnia/alloc.h>
#include <zinnia/clock.h>
#include <zinnia/mmio.h>
#include <zinnia/print.h>
#include <zinnia/utils.h>
#include <string.h>
#include "nvme.h"
constexpr size_t DOORBELL_OFFSET = 0x1000;
static void ring_tail(void* regs, size_t doorbells_offset, size_t tail) {
mmio_reg_write(regs, ((mmio_reg32){doorbells_offset, MMIO_LE}), (uint32_t)tail);
}
static void ring_head(void* regs, size_t doorbells_offset, size_t doorbell_stride, size_t head) {
mmio_reg_write(regs, ((mmio_reg32){doorbells_offset + doorbell_stride, MMIO_LE}), (uint32_t)head);
}
// Creates a new submission and completion queue pair.
errno_t nvme_queue_new(
struct dma_tag* tag,
void* regs,
size_t doorbell_stride,
size_t queue_id,
size_t depth,
struct nvme_queue** out
) {
const size_t align = 0x1000;
const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
struct nvme_queue* queue = mem_alloc(sizeof(*queue), 0);
if (!queue)
return ENOMEM;
// Allocate memory the completion queue.
errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
if (status)
goto fail;
// Allocate memory for the submission queue.
status = dma_buffer_alloc(tag, sq_size, &queue->sq);
if (status)
goto fail;
// Calculate the offset of the doorbell registers. The stride is already precomputed here.
queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
queue->doorbell_stride = doorbell_stride;
queue->depth = depth;
queue->regs = regs;
queue->cq_head = 0;
queue->cq_phase = 1; // When the controller is enabled, the first phase is 1.
queue->sq_tail = 0;
*out = queue;
return 0;
fail:
nvme_queue_free(queue);
return status;
}
void nvme_queue_free(struct nvme_queue* queue) {
dma_buffer_free(&queue->cq);
dma_buffer_free(&queue->sq);
mem_free(queue);
}
// Submits a command to this queue.
errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe) {
struct nvme_sqe* entries = queue->sq.virt;
memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
queue->sq_tail++;
if (queue->sq_tail == queue->depth)
queue->sq_tail = 0;
// Notify the controller of the new tail index.
ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
return 0;
}
// Reads the next completion entry from the queue.
errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out) {
struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
// Wait until the phase for this entry has changed.
uint32_t dw3;
const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
while (true) {
dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
// The controller will flip the phase bit of the current entry when writing.
if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) == queue->cq_phase)
break;
if (clock_get_elapsed_ns() >= deadline)
return ETIMEDOUT;
}
// Then, read the rest of the completion queue entry.
uint32_t dw0 = mmio_reg_read(entry, NVME_CQE_DW0);
uint32_t dw2 = mmio_reg_read(entry, NVME_CQE_DW2);
*out = (struct nvme_completion){
.result = dw0,
.sq_head = mmio_field_get(NVME_CQE_SQ_HEAD, dw2),
.sq_id = mmio_field_get(NVME_CQE_SQ_IDENT, dw2),
.cmd_id = mmio_field_get(NVME_CQE_CID, dw3),
.status = mmio_field_get(NVME_CQE_STATUS, dw3),
.phase_tag = mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != 0,
};
queue->cq_head++;
if (queue->cq_head == queue->depth) {
queue->cq_head = 0;
queue->cq_phase ^= 1; // Flip the phase bit.
}
// Notify the controller of the new head index.
ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
return 0;
}
errno_t nvme_io_queue_new(
struct dma_tag* tag,
void* regs,
size_t doorbell_stride,
size_t queue_id,
size_t depth,
bool poll,
struct nvme_io_queue** out
) {
const size_t align = 0x1000;
const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
struct nvme_io_queue* queue = mem_alloc(sizeof(*queue), 0);
if (!queue)
return ENOMEM;
errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
if (status)
goto fail;
status = dma_buffer_alloc(tag, sq_size, &queue->sq);
if (status)
goto fail;
const size_t slots = depth - 1;
queue->slots = mem_alloc(slots * sizeof(*queue->slots), 0);
queue->free = mem_alloc(slots * sizeof(*queue->free), 0);
if (!queue->slots || !queue->free) {
status = ENOMEM;
goto fail;
}
for (size_t i = 0; i < slots; i++)
queue->free[i] = slots - 1 - i;
queue->num_free = slots;
queue->queue_id = queue_id;
queue->depth = depth;
queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
queue->doorbell_stride = doorbell_stride;
queue->regs = regs;
queue->poll = poll;
queue->tag = tag;
queue->sq_tail = 0;
queue->cq_head = 0;
queue->cq_phase = 1;
spin_init(&queue->sq_lock);
spin_init(&queue->cq_lock);
spin_init(&queue->cmds_lock);
event_init(&queue->slots_free);
*out = queue;
return 0;
fail:
nvme_io_queue_free(queue);
return status;
}
void nvme_io_queue_free(struct nvme_io_queue* queue) {
dma_buffer_free(&queue->cq);
dma_buffer_free(&queue->sq);
mem_free(queue->slots);
mem_free(queue->free);
mem_free(queue);
}
static uint16_t acquire_slot(struct nvme_io_queue* queue) {
while (true) {
spin_lock(&queue->cmds_lock);
if (queue->num_free) {
uint16_t cid = queue->free[--queue->num_free];
spin_unlock(&queue->cmds_lock);
return cid;
}
spin_unlock(&queue->cmds_lock);
if (queue->poll) {
nvme_io_queue_drain(queue);
continue;
}
nvme_io_queue_drain(queue);
event_reset(&queue->slots_free);
spin_lock(&queue->cmds_lock);
bool empty = queue->num_free == 0;
spin_unlock(&queue->cmds_lock);
if (empty)
event_wait(&queue->slots_free);
}
}
static void release_slot(struct nvme_io_queue* queue, uint16_t cid) {
spin_lock(&queue->cmds_lock);
queue->free[queue->num_free++] = cid;
spin_unlock(&queue->cmds_lock);
}
static void write_sqe(struct nvme_io_queue* queue, const struct nvme_sqe* sqe) {
spin_lock(&queue->sq_lock);
struct nvme_sqe* entries = queue->sq.virt;
memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
queue->sq_tail++;
if (queue->sq_tail == queue->depth)
queue->sq_tail = 0;
ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
spin_unlock(&queue->sq_lock);
}
uint16_t nvme_io_queue_submit(
struct nvme_io_queue* queue,
struct bio* bio,
struct nvme_rw_cmd cmd,
_Atomic(bool)* done
) {
const uint16_t cid = acquire_slot(queue);
struct nvme_inflight* slot = &queue->slots[cid];
struct nvme_sqe sqe = {};
if (bio->op == BIO_FLUSH) {
slot->map = nullptr;
slot->prps = (struct dma_buffer){};
nvme_flush_cmd_write(&(struct nvme_flush_cmd){.cid = cid, .nsid = cmd.nsid}, &sqe);
} else {
errno_t status = nvme_build_prps(queue->tag, bio, slot, &cmd.prp1, &cmd.prp2);
if (status) {
release_slot(queue, cid);
bio_complete(bio, EIO, 0);
if (done)
atomic_store(done, true, ATOMIC_RELEASE);
return cid;
}
cmd.cid = cid;
nvme_rw_cmd_write(&cmd, &sqe);
}
spin_lock(&queue->cmds_lock);
slot->active = true;
slot->bio = bio;
slot->lbas = bio->num_lbas;
slot->done = done;
spin_unlock(&queue->cmds_lock);
write_sqe(queue, &sqe);
return cid;
}
bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio) {
spin_lock(&queue->cmds_lock);
struct nvme_inflight* slot = &queue->slots[cid];
bool abandoned = slot->active && slot->bio == bio;
if (abandoned) {
slot->bio = nullptr;
slot->done = nullptr;
}
spin_unlock(&queue->cmds_lock);
return abandoned;
}
size_t nvme_io_queue_drain(struct nvme_io_queue* queue) {
spin_lock(&queue->cq_lock);
size_t completed = 0;
while (true) {
struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
uint32_t dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != queue->cq_phase)
break;
uint16_t cid = mmio_field_get(NVME_CQE_CID, dw3);
uint16_t status = mmio_field_get(NVME_CQE_STATUS, dw3);
queue->cq_head++;
if (queue->cq_head == queue->depth) {
queue->cq_head = 0;
queue->cq_phase ^= 1;
}
struct nvme_inflight inflight = {};
spin_lock(&queue->cmds_lock);
if (cid < queue->depth - 1 && queue->slots[cid].active) {
inflight = queue->slots[cid];
queue->slots[cid].active = false;
queue->free[queue->num_free++] = cid;
}
spin_unlock(&queue->cmds_lock);
if (inflight.active) {
nvme_release_prps(&inflight);
if (inflight.bio) {
if (nvme_status_is_success(status))
bio_complete(inflight.bio, 0, inflight.lbas);
else
bio_complete(inflight.bio, EIO, 0);
}
if (inflight.done)
atomic_store(inflight.done, true, ATOMIC_RELEASE);
}
completed++;
}
if (completed > 0)
ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
spin_unlock(&queue->cq_lock);
if (completed > 0)
event_set(&queue->slots_free);
return completed;
}
dev/block/nvme/spec.h
+149
View File
@@ -1,3 +1,152 @@
// SPDX-License-Identifier: ISC
#pragma once
#include <zinnia/compiler.h>
#include <zinnia/mmio.h>
#include <stddef.h>
#include <stdint.h>
// Submission queue entry.
struct [[__packed]] nvme_sqe {
uint32_t cdw0;
uint32_t nsid; // Namespace Identifier
uint32_t cdw2;
uint32_t cdw3;
uint64_t mptr; // Metadata Pointer
uint64_t dptr0; // Data Pointer 0
uint64_t dptr1; // Data Pointer 1
uint32_t cdw10;
uint32_t cdw11;
uint32_t cdw12;
uint32_t cdw13;
uint32_t cdw14;
uint32_t cdw15;
};
static_assert(sizeof(struct nvme_sqe) == 0x40);
constexpr mmio_reg32 NVME_SQE_CDW0 = {offsetof(struct nvme_sqe, cdw0), MMIO_LE};
constexpr mmio_field32 NVME_SQE_CID = {NVME_SQE_CDW0, 16, 16}; // Command Identifier
constexpr mmio_field32 NVME_SQE_PSDT = {NVME_SQE_CDW0, 14, 2}; // PRP or SGL for Data Transfer
constexpr mmio_field32 NVME_SQE_FUSE = {NVME_SQE_CDW0, 8, 2}; // Fused Operation
constexpr mmio_field32 NVME_SQE_OPC = {NVME_SQE_CDW0, 0, 8}; // Opcode
constexpr mmio_reg32 NVME_SQE_NSID = {offsetof(struct nvme_sqe, nsid), MMIO_LE}; // Namespace Identifier
constexpr mmio_reg64 NVME_SQE_MPTR = {offsetof(struct nvme_sqe, mptr), MMIO_LE}; // Metadata Pointer
constexpr mmio_reg64 NVME_SQE_DPTR0 = {offsetof(struct nvme_sqe, dptr0), MMIO_LE}; // Data Pointer 0
constexpr mmio_reg64 NVME_SQE_DPTR1 = {offsetof(struct nvme_sqe, dptr1), MMIO_LE}; // Data Pointer 1
constexpr mmio_reg32 NVME_SQE_CDW10 = {offsetof(struct nvme_sqe, cdw10), MMIO_LE};
constexpr mmio_reg32 NVME_SQE_CDW11 = {offsetof(struct nvme_sqe, cdw11), MMIO_LE};
constexpr mmio_reg32 NVME_SQE_CDW12 = {offsetof(struct nvme_sqe, cdw12), MMIO_LE};
constexpr mmio_reg32 NVME_SQE_CDW13 = {offsetof(struct nvme_sqe, cdw13), MMIO_LE};
constexpr mmio_reg32 NVME_SQE_CDW14 = {offsetof(struct nvme_sqe, cdw14), MMIO_LE};
constexpr mmio_reg32 NVME_SQE_CDW15 = {offsetof(struct nvme_sqe, cdw15), MMIO_LE};
constexpr uint8_t NVME_CREATE_CQ_OPCODE = 0x05;
constexpr mmio_field32 NVME_CREATE_CQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
constexpr mmio_field32 NVME_CREATE_CQ_QID = {NVME_SQE_CDW10, 0, 16}; // Queue Identifier
constexpr mmio_field32 NVME_CREATE_CQ_IV = {NVME_SQE_CDW11, 16, 16}; // Interrupt Vector
constexpr mmio_field32 NVME_CREATE_CQ_IEN = {NVME_SQE_CDW11, 1, 1}; // Interrupts Enabled
constexpr mmio_field32 NVME_CREATE_CQ_PC = {NVME_SQE_CDW11, 0, 1}; // Physically Contiguous
constexpr uint8_t NVME_CREATE_SQ_OPCODE = 0x01;
constexpr mmio_field32 NVME_CREATE_SQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
constexpr mmio_field32 NVME_CREATE_SQ_QID = {NVME_SQE_CDW10, 0, 16}; // Queue Identifier
constexpr mmio_field32 NVME_CREATE_SQ_CQID = {NVME_SQE_CDW11, 16, 16}; // Completion Queue Identifier
constexpr mmio_field32 NVME_CREATE_SQ_QPRIO = {NVME_SQE_CDW11, 1, 2}; // Queue Priority
constexpr mmio_field32 NVME_CREATE_SQ_PC = {NVME_SQE_CDW11, 0, 1}; // Physically Contiguous
constexpr mmio_field32 NVME_IDENTIFY_CNTID = {NVME_SQE_CDW10, 16, 16}; // Controller Identifier
constexpr mmio_field32 NVME_IDENTIFY_CNS = {NVME_SQE_CDW10, 0, 8}; // Controller or Namespace Structure
constexpr mmio_field32 NVME_RW_SLBA_LOW = {NVME_SQE_CDW10, 0, 32}; // Starting LBA Lower Bits
constexpr mmio_field32 NVME_RW_SLBA_HIGH = {NVME_SQE_CDW11, 0, 32}; // Starting LBA Higher Bits
constexpr mmio_field32 NVME_RW_LR = {NVME_SQE_CDW12, 31, 1}; // Limited Retry
constexpr mmio_field32 NVME_RW_FUA = {NVME_SQE_CDW12, 30, 1}; // Force Unit Access
constexpr mmio_field32 NVME_RW_PRINFO = {NVME_SQE_CDW12, 26, 4}; // Protection Information Field
constexpr mmio_field32 NVME_RW_NLB = {NVME_SQE_CDW12, 0, 16}; // Number of Logical Blocks
struct [[__packed]] nvme_cqe {
uint32_t dw0;
uint32_t dw1;
uint32_t dw2;
uint32_t dw3;
};
static_assert(sizeof(struct nvme_cqe) == 0x10);
constexpr mmio_reg32 NVME_CQE_DW0 = {offsetof(struct nvme_cqe, dw0), MMIO_LE};
constexpr mmio_reg32 NVME_CQE_DW2 = {offsetof(struct nvme_cqe, dw2), MMIO_LE};
constexpr mmio_field32 NVME_CQE_SQ_IDENT = {NVME_CQE_DW2, 16, 16};
constexpr mmio_field32 NVME_CQE_SQ_HEAD = {NVME_CQE_DW2, 0, 16};
constexpr mmio_reg32 NVME_CQE_DW3 = {offsetof(struct nvme_cqe, dw3), MMIO_LE};
constexpr mmio_field32 NVME_CQE_STATUS = {NVME_CQE_DW3, 17, 15};
constexpr mmio_field32 NVME_CQE_PHASE_TAG = {NVME_CQE_DW3, 16, 1};
constexpr mmio_field32 NVME_CQE_CID = {NVME_CQE_DW3, 0, 16};
constexpr mmio_reg64 NVME_REG_CAP = {0x00, MMIO_LE}; // Controller Capabilities
constexpr mmio_field64 NVME_CAP_MPSMAX = {NVME_REG_CAP, 52, 4}; // Memory Page Size Maximum
constexpr mmio_field64 NVME_CAP_MPSMIN = {NVME_REG_CAP, 48, 4}; // Memory Page Size Minimum
constexpr mmio_field64 NVME_CAP_BPS = {NVME_REG_CAP, 45, 1}; // Boot Partition Support
constexpr mmio_field64 NVME_CAP_CSS = {NVME_REG_CAP, 37, 8}; // Command Sets Supported
constexpr mmio_field64 NVME_CAP_NSSRS = {NVME_REG_CAP, 36, 1}; // NVM Subsystem Reset Supported
constexpr mmio_field64 NVME_CAP_DSTRD = {NVME_REG_CAP, 32, 4}; // Doorbell Stride
constexpr mmio_field64 NVME_CAP_TO = {NVME_REG_CAP, 24, 8}; // Timeout
constexpr mmio_field64 NVME_CAP_AMS = {NVME_REG_CAP, 17, 2}; // Arbitration Mechanism Supported
constexpr mmio_field64 NVME_CAP_CQR = {NVME_REG_CAP, 16, 1}; // Contiguous Queues Required
constexpr mmio_field64 NVME_CAP_MQES = {NVME_REG_CAP, 0, 16}; // Maximum Queue Entries Supported
constexpr mmio_reg32 NVME_REG_VS = {0x08, MMIO_LE}; // Version
constexpr mmio_field32 NVME_VS_MJR = {NVME_REG_VS, 16, 16}; // Major Version Number
constexpr mmio_field32 NVME_VS_MNR = {NVME_REG_VS, 8, 8}; // Minor Version Number
constexpr mmio_field32 NVME_VS_TER = {NVME_REG_VS, 0, 8}; // Tertiary Version Number
constexpr mmio_reg32 NVME_REG_INTMS = {0x0C, MMIO_LE}; // Interrupt Mask Set
constexpr mmio_reg32 NVME_REG_INTMC = {0x10, MMIO_LE}; // Interrupt Mask Clear
constexpr mmio_reg32 NVME_REG_CC = {0x14, MMIO_LE}; // Controller Configuration
constexpr mmio_field32 NVME_CC_IOCQES = {NVME_REG_CC, 20, 4}; // I/O Completion Queue Entry Size
constexpr mmio_field32 NVME_CC_IOSQES = {NVME_REG_CC, 16, 4}; // I/O Submission Queue Entry Size
constexpr mmio_field32 NVME_CC_SHN = {NVME_REG_CC, 14, 2}; // Shutdown Notification
constexpr mmio_field32 NVME_CC_AMS = {NVME_REG_CC, 11, 3}; // Arbitration Mechanism Selected
constexpr mmio_field32 NVME_CC_MPS = {NVME_REG_CC, 7, 4}; // Memory Page Size
constexpr mmio_field32 NVME_CC_CSS = {NVME_REG_CC, 4, 3}; // I/O Command Set Selected
constexpr mmio_field32 NVME_CC_EN = {NVME_REG_CC, 0, 1}; // Enable
constexpr mmio_reg32 NVME_REG_CSTS = {0x1C, MMIO_LE}; // Controller Status
constexpr mmio_field32 NVME_CSTS_PP = {NVME_REG_CSTS, 5, 1}; // Processing Paused
constexpr mmio_field32 NVME_CSTS_NSSRO = {NVME_REG_CSTS, 4, 1}; // NVM Subsystem Reset Occurred
constexpr mmio_field32 NVME_CSTS_SHST = {NVME_REG_CSTS, 2, 2}; // Shutdown Status
constexpr mmio_field32 NVME_CSTS_CFS = {NVME_REG_CSTS, 1, 1}; // Controller Fatal Status
constexpr mmio_field32 NVME_CSTS_RDY = {NVME_REG_CSTS, 0, 1}; // Ready
constexpr mmio_reg32 NVME_REG_AQA = {0x24, MMIO_LE}; // Admin Queue Attributes
constexpr mmio_field32 NVME_AQA_ACQS = {NVME_REG_AQA, 0, 12}; // Admin Completion Queue Size
constexpr mmio_field32 NVME_AQA_ASQS = {NVME_REG_AQA, 16, 12}; // Admin Submission Queue Size
constexpr mmio_reg64 NVME_REG_ASQ = {0x28, MMIO_LE}; // Admin Submission Queue
constexpr mmio_reg64 NVME_REG_ACQ = {0x30, MMIO_LE}; // Admin Completion Queue
// Generic Command Set
enum nvme_cmd : uint8_t {
NVME_CMD_FLUSH = 0x00,
NVME_CMD_WRITE = 0x01,
NVME_CMD_READ = 0x02,
};
// Admin Command Set
enum nvme_admin_cmd : uint8_t {
NVME_ADMIN_DELETE_SQ = 0x00,
NVME_ADMIN_CREATE_SQ = 0x01,
NVME_ADMIN_DELETE_CQ = 0x04,
NVME_ADMIN_CREATE_CQ = 0x05,
NVME_ADMIN_IDENTIFY = 0x06,
NVME_ADMIN_ABORT = 0x08,
NVME_ADMIN_SET_FEATURES = 0x09,
NVME_ADMIN_GET_FEATURES = 0x0A,
};
dev/pci/core.c
+121
View File
@@ -4,11 +4,18 @@
#include <zinnia/alloc.h>
#include <zinnia/init.h>
#include <zinnia/mmio.h>
#include <zinnia/module.h>
#include <zinnia/pci.h>
#include <zinnia/pmap.h>
#include <zinnia/print.h>
#include <zinnia/spin.h>
#include <zinnia/utils.h>
#include <zinnia/vec.h>
#include <zinnia/virt.h>
#include <zinnia/vma.h>
#include <arch/irq.h>
#include <arch/mem.h>
#include <string.h>
static struct spinlock access_lock = {};
@@ -170,6 +177,119 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) {
return 0;
}
constexpr uint32_t PCI_MSI_CTRL_ENABLE = 1 << 16;
constexpr uint32_t PCI_MSI_CTRL_MME = 0x7 << 20;
constexpr uint32_t PCI_MSI_CTRL_64BIT = 1 << 23;
constexpr uint32_t PCI_MSIX_CTRL_FUNCTION_MASK = 1 << 30;
constexpr uint32_t PCI_MSIX_CTRL_ENABLE = 1u << 31;
static void msix_set_state(struct pci_device* dev, uint8_t cap, bool status) {
uint32_t ctrl = pci_read32(dev, cap);
ctrl &= ~(PCI_MSIX_CTRL_ENABLE | PCI_MSIX_CTRL_FUNCTION_MASK);
if (status)
ctrl |= PCI_MSIX_CTRL_ENABLE;
pci_write32(dev, cap, ctrl);
}
errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out) {
// message address + data + vector control
constexpr size_t ENTRY_SIZE = 8 + 4 + 4;
const uint8_t cap = pci_cap_find(dev, PCI_CAP_MSIX);
if (cap == 0)
return ENODEV;
const uint32_t table = pci_read32(dev, cap + 4);
const uint8_t bir = table & 0x7;
const size_t table_offset = table & ~0x7;
pci_log(dev, "Using MSI-X for this device, table at BAR%hhu + %#zx\n", bir, table_offset);
// Map the first MSI-X table entry (16 bytes).
struct pci_bar bar;
errno_t status = pci_bar_get(dev, bir, &bar);
if (status)
return ENODEV;
if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
return EINVAL;
const size_t page_size = arch_mem_page_size();
const phys_t table_phys = bar.address + table_offset;
const phys_t page_base = ALIGN_DOWN(table_phys, page_size);
const size_t page_offset = table_phys - page_base;
const size_t map_len = ALIGN_UP(page_offset + ENTRY_SIZE, page_size);
uintptr_t vaddr = vm_alloc(map_len);
if (!vaddr)
return ENOMEM;
for (size_t i = 0; i < map_len; i += page_size) {
status = pmap_map(&kernel_vma.pmap, vaddr + i, page_base + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
if (status)
return status;
}
void* entry = (void*)(vaddr + page_offset);
struct msi_line* msi;
status = arch_irq_alloc_msi(&msi);
if (status)
return status;
irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
// Write entry 0.
const uint64_t msg_addr = msi->msg_addr(msi);
const uint32_t msg_data = msi->msg_data(msi);
mmio_write32(entry + 0x00, (uint32_t)msg_addr); // Address lo
mmio_write32(entry + 0x04, (uint32_t)(msg_addr >> 32)); // Address hi
mmio_write32(entry + 0x08, msg_data); // Data
mmio_write32(entry + 0x0C, 0); // Vector control
// Enable MSI-X on the device.
msix_set_state(dev, cap, true);
*out = &msi->line;
return 0;
}
errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out) {
uint8_t cap = pci_cap_find(dev, PCI_CAP_MSI);
if (cap == 0)
return ENODEV;
pci_log(dev, "Using MSI for this device\n");
struct msi_line* msi;
errno_t status = arch_irq_alloc_msi(&msi);
if (status)
return status;
irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
const uint64_t addr = msi->msg_addr(msi);
const uint16_t data = msi->msg_data(msi);
uint32_t ctrl = pci_read32(dev, cap);
pci_write32(dev, cap + 0x04, (uint32_t)addr);
uint16_t data_off = 0x08;
if (ctrl & PCI_MSI_CTRL_64BIT) {
pci_write32(dev, cap + 0x08, (uint32_t)(addr >> 32));
data_off = 0x0C;
}
pci_write32(dev, cap + data_off, data);
ctrl &= ~PCI_MSI_CTRL_MME;
ctrl |= PCI_MSI_CTRL_ENABLE;
pci_write32(dev, cap, ctrl);
*out = &msi->line;
return 0;
}
errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out) {
errno_t status = pci_setup_msix(dev, out);
if (status == ENODEV)
return pci_setup_msi(dev, out);
return status;
}
static void pci_device_name(const struct pci_device* dev, char buf[static 18]) {
ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
}
@@ -192,6 +312,7 @@ static bool scan_function(struct pci_access* access, struct pci_address addr) {
dev->address = addr;
dev->access = access;
dev->dma_tag = &dma_default_tag;
dev->id.vendor_id = vendor_id;
dev->id.device_id = device_id;
dev->class.prog_if = pci_read8(dev, PCI_PROG_IF);
include/zinnia/dma.h
+84
View File
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: ISC
#pragma once
#include <zinnia/errno.h>
#include <zinnia/iovec.h>
#include <zinnia/types.h>
#include <stddef.h>
#include <stdint.h>
typedef uint64_t dma_addr_t;
constexpr dma_addr_t DMA_ADDR_MAX = UINT64_MAX;
struct bio;
struct dma_tag;
struct dma_ops {
errno_t (*map)(struct dma_tag* tag, phys_t phys, size_t len, dma_addr_t* out);
void (*unmap)(struct dma_tag* tag, dma_addr_t addr, size_t len);
};
struct dma_tag {
const struct dma_ops* ops;
void* priv;
dma_addr_t max_addr;
};
extern struct dma_tag dma_default_tag;
void dma_tag_subregion(struct dma_tag* parent, dma_addr_t max_addr, struct dma_tag* out);
struct dma_seg {
dma_addr_t addr;
size_t len;
};
struct dma_buffer {
struct dma_tag* tag;
phys_t phys;
dma_addr_t addr;
void* virt;
size_t len;
};
errno_t dma_buffer_alloc(struct dma_tag* tag, size_t len, struct dma_buffer* out);
void dma_buffer_free(struct dma_buffer* buf);
struct dma_map {
struct dma_tag* tag;
size_t size;
size_t max_segs;
size_t max_seg_size;
size_t boundary;
size_t mapped_len;
size_t num_segs;
struct dma_seg segs[];
};
enum dma_sync {
DMA_SYNC_PREREAD = 1 << 0,
DMA_SYNC_POSTREAD = 1 << 1,
DMA_SYNC_PREWRITE = 1 << 2,
DMA_SYNC_POSTWRITE = 1 << 3,
};
errno_t dma_map_create(
struct dma_tag* tag,
size_t size,
size_t max_segs,
size_t max_seg_size,
size_t boundary,
struct dma_map** out
);
void dma_map_destroy(struct dma_map* map);
errno_t dma_map_load(struct dma_map* map, void* buf, size_t len);
errno_t dma_map_load_bio(struct dma_map* map, struct bio* bio);
errno_t dma_map_load_iter(struct dma_map* map, const iovec_iter_t* iter);
void dma_map_unload(struct dma_map* map);
void dma_map_sync(struct dma_map* map, size_t offset, size_t len, enum dma_sync ops);
void dma_buffer_sync(struct dma_buffer* buf, size_t offset, size_t len, enum dma_sync ops);
include/zinnia/irq.h
+15
View File
@@ -5,13 +5,16 @@
#include <zinnia/compiler.h>
#include <zinnia/errno.h>
#include <zinnia/list.h>
#include <zinnia/spin.h>
#include <zinnia/types.h>
#include <zinnia/workqueue.h>
#include <stddef.h>
#include <stdint.h>
enum irq_status : uint8_t {
IRQ_STATUS_IGNORED,
IRQ_STATUS_HANDLED,
IRQ_STATUS_WAKE_THREAD,
};
enum irq_polarity : uint8_t {
@@ -29,9 +32,17 @@ enum irq_mode : uint8_t {
IRQ_MODE_MASKABLE = 1 << 1,
};
struct irq_line;
struct irq_handler {
enum irq_status (*raise)(struct irq_handler* self);
void (*thread)(struct irq_handler* self);
list_link(struct irq_handler*) next;
struct irq_line* line;
struct workqueue* wq;
struct work work;
bool holds_mask;
};
struct irq_line {
@@ -43,6 +54,9 @@ struct irq_line {
bool is_busy;
list_head(struct irq_handler*) handlers;
enum irq_mode mode;
enum irq_trigger_mode trigger;
struct spinlock thread_lock;
size_t threads_pending;
};
struct msi_line {
@@ -62,6 +76,7 @@ void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum
void irq_line_raise(struct irq_line* line);
void irq_line_attach(struct irq_line* line, struct irq_handler* handler);
void irq_line_detach(struct irq_line* line, struct irq_handler* handler);
errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name);
void irq_lock();
void irq_unlock();
include/zinnia/pci.h
+8
View File
@@ -4,6 +4,7 @@
#include <zinnia/alloc.h>
#include <zinnia/compiler.h>
#include <zinnia/dma.h>
#include <zinnia/errno.h>
#include <zinnia/initgraph.h>
#include <zinnia/print.h>
@@ -106,6 +107,7 @@ struct pci_device {
struct pci_access* access;
uint8_t header_type;
struct pci_driver* driver;
struct dma_tag* dma_tag;
};
uint8_t pci_read8(struct pci_device* dev, uint16_t offset);
@@ -137,6 +139,12 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id);
constexpr uint16_t PCI_ANY_ID = 0xffff;
constexpr uint8_t PCI_ANY_CLASS = 0xff;
struct irq_line;
errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out);
errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out);
errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out);
// Matches a device by ID and class.
struct pci_match {
uint16_t vendor_id;
lib/irq.c
+60 -1
View File
@@ -5,6 +5,8 @@
#include <zinnia/list.h>
#include <zinnia/percpu.h>
#include <zinnia/print.h>
#include <zinnia/sched.h>
#include <zinnia/utils.h>
#include <arch/cpu.h>
#include <arch/irq.h>
@@ -39,9 +41,43 @@ void irq_unlock() {
void irq_line_program(struct irq_line* line, enum irq_trigger_mode trigger, enum irq_polarity polarity) {
irq_lock();
line->mode = line->set_config(line, trigger, polarity);
line->trigger = trigger;
irq_unlock();
}
static bool holds_mask_while_threaded(struct irq_line* line) {
return line->trigger == IRQ_TRIGGER_LEVEL && (line->mode & IRQ_MODE_MASKABLE) && line->mask && line->unmask;
}
static void wake_thread(struct irq_line* line, struct irq_handler* handler) {
if (holds_mask_while_threaded(line)) {
spin_lock(&line->thread_lock);
if (!handler->holds_mask) {
handler->holds_mask = true;
if (line->threads_pending++ == 0)
line->mask(line);
}
spin_unlock(&line->thread_lock);
}
work_queue(handler->wq, &handler->work);
}
static void irq_thread_work(struct work* work) {
struct irq_handler* handler = containerof(work, struct irq_handler, work);
struct irq_line* line = handler->line;
handler->thread(handler);
spin_lock(&line->thread_lock);
if (handler->holds_mask) {
handler->holds_mask = false;
if (--line->threads_pending == 0)
line->unmask(line);
}
spin_unlock(&line->thread_lock);
}
void irq_line_raise(struct irq_line* line) {
assert(!arch_irq_get_state(), "Raised an IRQ line with interrupts enabled");
@@ -51,8 +87,13 @@ void irq_line_raise(struct irq_line* line) {
bool claimed = false;
struct irq_handler* handler;
list_foreach(handler, &line->handlers, next) {
if (handler->raise(handler) == IRQ_STATUS_HANDLED)
enum irq_status status = handler->raise ? handler->raise(handler) : IRQ_STATUS_WAKE_THREAD;
if (status == IRQ_STATUS_WAKE_THREAD && handler->wq) {
wake_thread(line, handler);
claimed = true;
} else if (status != IRQ_STATUS_IGNORED) {
claimed = true;
}
}
if (!claimed)
@@ -83,3 +124,21 @@ void irq_line_detach(struct irq_line* line, struct irq_handler* handler) {
while (__atomic_load_n(&line->is_busy, __ATOMIC_ACQUIRE))
arch_cpu_pause();
}
errno_t irq_line_attach_threaded(struct irq_line* line, struct irq_handler* handler, const char* name) {
if (!handler->thread)
return EINVAL;
char task_name[TASK_NAME_MAX];
ksnprintf(task_name, sizeof(task_name), "irq/%s", name);
handler->line = line;
work_init(&handler->work, irq_thread_work);
handler->holds_mask = false;
errno_t status = workqueue_create(task_name, SCHED_PRIO_IRQ, &handler->wq);
if (status)
return status;
irq_line_attach(line, handler);
return 0;
}
mem/dma.c
+272
View File
@@ -0,0 +1,272 @@
// SPDX-License-Identifier: ISC
#include <zinnia/alloc.h>
#include <zinnia/atomic.h>
#include <zinnia/blockdev.h>
#include <zinnia/dma.h>
#include <zinnia/percpu.h>
#include <zinnia/pmap.h>
#include <zinnia/sched.h>
#include <zinnia/utils.h>
#include <zinnia/vma.h>
#include <arch/mem.h>
static errno_t dma_noop_map(struct dma_tag*, phys_t phys, size_t, dma_addr_t* out) {
*out = phys;
return 0;
}
static void dma_noop_unmap(struct dma_tag*, dma_addr_t, size_t) {
// Does nothing.
}
static const struct dma_ops dma_noop_ops = {
.map = dma_noop_map,
.unmap = dma_noop_unmap,
};
struct dma_tag dma_default_tag = {
.ops = &dma_noop_ops,
.max_addr = DMA_ADDR_MAX,
};
void dma_tag_subregion(struct dma_tag* parent, dma_addr_t max_addr, struct dma_tag* out) {
*out = *parent;
out->max_addr = MIN(parent->max_addr, max_addr);
}
static errno_t tag_alloc_flags(struct dma_tag* tag, enum alloc_flags* out) {
if (tag->max_addr == DMA_ADDR_MAX)
*out = 0;
else if (tag->max_addr >= UINT32_MAX)
*out = ALLOC_MEM32;
else if (tag->max_addr >= ((dma_addr_t)1 << 20) - 1)
*out = ALLOC_MEM20;
else
return EINVAL;
return 0;
}
errno_t dma_buffer_alloc(struct dma_tag* tag, size_t len, struct dma_buffer* out) {
if (len == 0)
return EINVAL;
enum alloc_flags flags;
errno_t status = tag_alloc_flags(tag, &flags);
if (status)
return status;
const size_t num_pages = ROUND_UP(len, arch_mem_page_size());
phys_t phys;
status = mem_phys_alloc(num_pages, flags, &phys);
if (status)
return status;
const size_t size = num_pages * arch_mem_page_size();
dma_addr_t addr;
status = tag->ops->map(tag, phys, size, &addr);
if (status) {
mem_phys_free(phys, num_pages);
return status;
}
if (addr + size - 1 > tag->max_addr) {
tag->ops->unmap(tag, addr, size);
mem_phys_free(phys, num_pages);
return ENOMEM;
}
*out = (struct dma_buffer){
.tag = tag,
.phys = phys,
.addr = addr,
.virt = HHDM_PTR(phys),
.len = len,
};
return 0;
}
void dma_buffer_free(struct dma_buffer* buf) {
if (buf->virt == nullptr)
return;
const size_t num_pages = ROUND_UP(buf->len, arch_mem_page_size());
buf->tag->ops->unmap(buf->tag, buf->addr, num_pages * arch_mem_page_size());
mem_phys_free(buf->phys, num_pages);
*buf = (struct dma_buffer){0};
}
errno_t dma_map_create(
struct dma_tag* tag,
size_t size,
size_t max_segs,
size_t max_seg_size,
size_t boundary,
struct dma_map** out
) {
if (max_segs == 0 || max_seg_size == 0)
return EINVAL;
if (boundary & (boundary - 1))
return EINVAL;
struct dma_map* map = mem_alloc(sizeof(*map) + max_segs * sizeof(struct dma_seg), 0);
if (map == nullptr)
return ENOMEM;
map->tag = tag;
map->size = size;
map->max_segs = max_segs;
map->max_seg_size = max_seg_size;
map->boundary = boundary;
map->mapped_len = 0;
map->num_segs = 0;
*out = map;
return 0;
}
void dma_map_destroy(struct dma_map* map) {
mem_free(map);
}
static errno_t map_add_range(struct dma_map* map, phys_t phys, size_t len) {
if (len == 0)
return 0;
dma_addr_t addr;
errno_t status = map->tag->ops->map(map->tag, phys, len, &addr);
if (status)
return status;
if (addr + len - 1 > map->tag->max_addr) {
map->tag->ops->unmap(map->tag, addr, len);
return EINVAL;
}
while (len > 0) {
size_t chunk = MIN(len, map->max_seg_size);
if (map->boundary) {
const dma_addr_t limit = ALIGN_DOWN(addr, map->boundary) + map->boundary;
chunk = MIN(chunk, limit - addr);
}
struct dma_seg* last = map->num_segs ? &map->segs[map->num_segs - 1] : nullptr;
if (last && last->addr + last->len == addr && last->len + chunk <= map->max_seg_size &&
(!map->boundary || ALIGN_DOWN(last->addr, map->boundary) == ALIGN_DOWN(addr + chunk - 1, map->boundary))) {
last->len += chunk;
} else {
if (map->num_segs == map->max_segs) {
map->tag->ops->unmap(map->tag, addr, len);
return EFBIG;
}
map->segs[map->num_segs++] = (struct dma_seg){.addr = addr, .len = chunk};
}
map->mapped_len += chunk;
addr += chunk;
len -= chunk;
}
return 0;
}
static errno_t map_add_virt(struct dma_map* map, struct pmap* pmap, uintptr_t vaddr, size_t len) {
const size_t page_size = arch_mem_page_size();
while (len > 0) {
const size_t offset = vaddr % page_size;
const size_t chunk = MIN(len, page_size - offset);
phys_t phys;
if (pmap_extract(pmap, vaddr - offset, &phys))
return EFAULT;
errno_t status = map_add_range(map, phys + offset, chunk);
if (status)
return status;
vaddr += chunk;
len -= chunk;
}
return 0;
}
static errno_t map_load_finish(struct dma_map* map, errno_t status) {
if (status)
dma_map_unload(map);
return status;
}
errno_t dma_map_load(struct dma_map* map, void* buf, size_t len) {
if (map->num_segs || len > map->size)
return EINVAL;
return map_load_finish(map, map_add_virt(map, &kernel_vma.pmap, (uintptr_t)buf, len));
}
errno_t dma_map_load_bio(struct dma_map* map, struct bio* bio) {
if (map->num_segs)
return EINVAL;
size_t total = 0;
for (size_t i = 0; i < bio->num_segs; i++)
total += bio->segs[i].len;
if (total > map->size)
return EINVAL;
errno_t status = 0;
for (size_t i = 0; i < bio->num_segs && !status; i++)
status = map_add_range(map, bio->segs[i].addr, bio->segs[i].len);
return map_load_finish(map, status);
}
errno_t dma_map_load_iter(struct dma_map* map, const iovec_iter_t* iter) {
if (map->num_segs || iter->total_len - iter->total_offset > map->size)
return EINVAL;
struct pmap* pmap = &kernel_vma.pmap;
if (iter->user)
pmap = &percpu_get()->sched.current->space->pmap;
errno_t status = 0;
size_t offset = iter->current_offset;
for (size_t i = iter->current_idx; i < iter->iovecs_len && !status; i++) {
const iovec_t* iov = &iter->iovecs[i];
if (offset >= iov->len) {
offset = 0;
continue;
}
const uintptr_t base = iov->base + offset;
const size_t len = iov->len - offset;
offset = 0;
if (iter->user && (!arch_mem_addr_is_user(base) || !arch_mem_addr_is_user(base + len - 1))) {
status = EFAULT;
break;
}
status = map_add_virt(map, pmap, base, len);
}
return map_load_finish(map, status);
}
void dma_map_unload(struct dma_map* map) {
for (size_t i = 0; i < map->num_segs; i++)
map->tag->ops->unmap(map->tag, map->segs[i].addr, map->segs[i].len);
map->num_segs = 0;
map->mapped_len = 0;
}
static void dma_sync(enum dma_sync ops) {
if (ops & (DMA_SYNC_PREREAD | DMA_SYNC_PREWRITE))
atomic_fence(ATOMIC_RELEASE);
if (ops & (DMA_SYNC_POSTREAD | DMA_SYNC_POSTWRITE))
atomic_fence(ATOMIC_ACQUIRE);
}
void dma_map_sync(struct dma_map* map, size_t offset, size_t len, enum dma_sync ops) {
dma_sync(ops);
}
void dma_buffer_sync(struct dma_buffer* buf, size_t offset, size_t len, enum dma_sync ops) {
dma_sync(ops);
}
mem/meson.build
+1
View File
@@ -1,5 +1,6 @@
kernel_sources += files(
'amap.c',
'dma.c',
'init.c',
'page.c',
'pmm.c',