dev/block: Add NVMe driver

dev/pci/core.c

@@ -4,11 +4,18 @@
 
 #include <zinnia/alloc.h>
 #include <zinnia/init.h>
+#include <zinnia/mmio.h>
 #include <zinnia/module.h>
 #include <zinnia/pci.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/spin.h>
+#include <zinnia/utils.h>
 #include <zinnia/vec.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
 #include <string.h>
 
 static struct spinlock access_lock = {};
@@ -170,6 +177,119 @@ uint8_t pci_cap_find(struct pci_device* dev, uint8_t id) {
     return 0;
 }
 
+constexpr uint32_t PCI_MSI_CTRL_ENABLE = 1 << 16;
+constexpr uint32_t PCI_MSI_CTRL_MME = 0x7 << 20;
+constexpr uint32_t PCI_MSI_CTRL_64BIT = 1 << 23;
+constexpr uint32_t PCI_MSIX_CTRL_FUNCTION_MASK = 1 << 30;
+constexpr uint32_t PCI_MSIX_CTRL_ENABLE = 1u << 31;
+
+static void msix_set_state(struct pci_device* dev, uint8_t cap, bool status) {
+    uint32_t ctrl = pci_read32(dev, cap);
+    ctrl &= ~(PCI_MSIX_CTRL_ENABLE | PCI_MSIX_CTRL_FUNCTION_MASK);
+    if (status)
+        ctrl |= PCI_MSIX_CTRL_ENABLE;
+    pci_write32(dev, cap, ctrl);
+}
+
+errno_t pci_setup_msix(struct pci_device* dev, struct irq_line** out) {
+    // message address + data + vector control
+    constexpr size_t ENTRY_SIZE = 8 + 4 + 4;
+
+    const uint8_t cap = pci_cap_find(dev, PCI_CAP_MSIX);
+    if (cap == 0)
+        return ENODEV;
+
+    const uint32_t table = pci_read32(dev, cap + 4);
+    const uint8_t bir = table & 0x7;
+    const size_t table_offset = table & ~0x7;
+
+    pci_log(dev, "Using MSI-X for this device, table at BAR%hhu + %#zx\n", bir, table_offset);
+
+    // Map the first MSI-X table entry (16 bytes).
+    struct pci_bar bar;
+    errno_t status = pci_bar_get(dev, bir, &bar);
+    if (status)
+        return ENODEV;
+    if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
+        return EINVAL;
+
+    const size_t page_size = arch_mem_page_size();
+    const phys_t table_phys = bar.address + table_offset;
+    const phys_t page_base = ALIGN_DOWN(table_phys, page_size);
+    const size_t page_offset = table_phys - page_base;
+    const size_t map_len = ALIGN_UP(page_offset + ENTRY_SIZE, page_size);
+
+    uintptr_t vaddr = vm_alloc(map_len);
+    if (!vaddr)
+        return ENOMEM;
+    for (size_t i = 0; i < map_len; i += page_size) {
+        status = pmap_map(&kernel_vma.pmap, vaddr + i, page_base + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status)
+            return status;
+    }
+    void* entry = (void*)(vaddr + page_offset);
+
+    struct msi_line* msi;
+    status = arch_irq_alloc_msi(&msi);
+    if (status)
+        return status;
+    irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
+
+    // Write entry 0.
+    const uint64_t msg_addr = msi->msg_addr(msi);
+    const uint32_t msg_data = msi->msg_data(msi);
+    mmio_write32(entry + 0x00, (uint32_t)msg_addr);         // Address lo
+    mmio_write32(entry + 0x04, (uint32_t)(msg_addr >> 32)); // Address hi
+    mmio_write32(entry + 0x08, msg_data);                   // Data
+    mmio_write32(entry + 0x0C, 0);                          // Vector control
+
+    // Enable MSI-X on the device.
+    msix_set_state(dev, cap, true);
+
+    *out = &msi->line;
+    return 0;
+}
+
+errno_t pci_setup_msi(struct pci_device* dev, struct irq_line** out) {
+    uint8_t cap = pci_cap_find(dev, PCI_CAP_MSI);
+    if (cap == 0)
+        return ENODEV;
+
+    pci_log(dev, "Using MSI for this device\n");
+
+    struct msi_line* msi;
+    errno_t status = arch_irq_alloc_msi(&msi);
+    if (status)
+        return status;
+    irq_line_program(&msi->line, IRQ_TRIGGER_EDGE, IRQ_POLARITY_HIGH);
+
+    const uint64_t addr = msi->msg_addr(msi);
+    const uint16_t data = msi->msg_data(msi);
+
+    uint32_t ctrl = pci_read32(dev, cap);
+    pci_write32(dev, cap + 0x04, (uint32_t)addr);
+    uint16_t data_off = 0x08;
+    if (ctrl & PCI_MSI_CTRL_64BIT) {
+        pci_write32(dev, cap + 0x08, (uint32_t)(addr >> 32));
+        data_off = 0x0C;
+    }
+    pci_write32(dev, cap + data_off, data);
+
+    ctrl &= ~PCI_MSI_CTRL_MME;
+    ctrl |= PCI_MSI_CTRL_ENABLE;
+    pci_write32(dev, cap, ctrl);
+
+    *out = &msi->line;
+    return 0;
+}
+
+errno_t pci_setup_irq(struct pci_device* dev, struct irq_line** out) {
+    errno_t status = pci_setup_msix(dev, out);
+    if (status == ENODEV)
+        return pci_setup_msi(dev, out);
+    return status;
+}
+
 static void pci_device_name(const struct pci_device* dev, char buf[static 18]) {
     ksnprintf(buf, 18, "pci-" PCI_ADDR_FMT, PCI_ADDR_ARGS(&dev->address));
 }
@@ -192,6 +312,7 @@ static bool scan_function(struct pci_access* access, struct pci_address addr) {
 
     dev->address = addr;
     dev->access = access;
+    dev->dma_tag = &dma_default_tag;
     dev->id.vendor_id = vendor_id;
     dev->id.device_id = device_id;
     dev->class.prog_if = pci_read8(dev, PCI_PROG_IF);