mirror of
https://github.com/Mathewnd/Astral.git
synced 2026-10-30 14:08:21 +00:00
nvme: command submission and creation of I/O queues
This commit is contained in:
Matheus
+498
-17
@@ -3,6 +3,13 @@
|
||||
#include <kernel/vmm.h>
|
||||
#include <util.h>
|
||||
#include <kernel/pmm.h>
|
||||
#include <kernel/alloc.h>
|
||||
#include <kernel/interrupt.h>
|
||||
#include <kernel/dpc.h>
|
||||
#include <hashtable.h>
|
||||
#include <event.h>
|
||||
#include <arch/cpu.h>
|
||||
#include <errno.h>
|
||||
|
||||
#define CC_ENABLE(cc) cc = (cc) | 1
|
||||
#define CC_DISABLE(cc) cc = (cc) & ~1
|
||||
@@ -14,6 +21,8 @@
|
||||
#define CC_COMMANDSET_NVM 0
|
||||
#define CC_ARBITRATION_ROUNDROBIN 0
|
||||
|
||||
#define CAP_MAXENTRIES(cap) ((cap) & 0xff)
|
||||
#define CAP_DOORBELLSTRIDE(cap) (((cap) >> 32) & 0xf)
|
||||
#define CAP_COMMANDSET(cap) (((cap) >> 37) & 0xff)
|
||||
#define CAP_MAXPAGESIZE(cap) (((cap) >> 52) & 0xf)
|
||||
#define CAP_MINPAGESIZE(cap) (((cap) >> 48) & 0xf)
|
||||
@@ -29,6 +38,151 @@
|
||||
#define AQATTR_SETSUBSIZE(attr, size) attr = ((attr) & ~0xfff) | (size)
|
||||
#define AQATTR_SETCOMPSIZE(attr, size) attr = ((attr) & ~0xfff0000) | ((size) << 16)
|
||||
|
||||
#define SUB_SETDW0OPCODE(dw0, opcode) dw0 = (dw0 & ~0xff) | ((opcode) & 0xff)
|
||||
#define SUB_SETDW0FUSED(dw0, fused) dw0 = (dw0 & ~0b1100000000) | (((fused) & 0x3) << 8)
|
||||
#define SUB_SETDW0MEMACCESS(dw0, memaccess) dw0 = (dw0 & ~0b1100000000000000) | (((memaccess) & 0x3) << 14)
|
||||
#define SUB_SETDW0ID(dw0, id) dw0 = (dw0 & ~0xffff0000) | (((id) & 0xffff) << 16)
|
||||
#define SUB_INIT(sub, opcode, fused, memaccess, nsid) \
|
||||
SUB_SETDW0OPCODE((sub)->dword0, opcode); \
|
||||
SUB_SETDW0FUSED((sub)->dword0, fused); \
|
||||
SUB_SETDW0MEMACCESS((sub)->dword0, memaccess); \
|
||||
(sub)->namespace = nsid;
|
||||
|
||||
#define PAIR_INIT(pair, opcode, fused, memaccess, nsid) \
|
||||
EVENT_INIT(&(pair)->event); \
|
||||
SUB_INIT(&(pair)->sub, opcode, fused, memaccess, nsid);
|
||||
|
||||
#define SUB_DW0_OPCODE_CREATEIOSUBQUEUE 0x1
|
||||
#define SUB_DW0_OPCODE_CREATEIOCOMPQUEUE 0x5
|
||||
#define SUB_DW0_OPCODE_IDENTIFY 0x6
|
||||
#define SUB_DW0_OPCODE_SETFEATURES 0x9
|
||||
#define SUB_DW0_UNFUSED 0
|
||||
#define SUB_DW0_PRP 0
|
||||
|
||||
#define COMP_CMDINFO_CMDID(x) ((x) & 0xffff)
|
||||
#define COMP_CMDINFO_PHASE(x) (((x) >> 16) & 1)
|
||||
#define COMP_CMDINFO_STATUS(x) (((x) >> 17) & 0x7fff)
|
||||
|
||||
#define GET_DOORBELL(bar0, id, completion, stride) (volatile uint32_t *)((uintptr_t)(bar0) + 0x1000 + (((id) + (completion)) * (4 << (stride))))
|
||||
|
||||
typedef struct {
|
||||
uint16_t metadatasize;
|
||||
uint8_t lbadatasize; // power of two
|
||||
uint8_t relativeperformance;
|
||||
} __attribute__((packed)) lbaformat_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t lbasize;
|
||||
uint64_t lbacapacity;
|
||||
uint64_t lbautilized;
|
||||
uint8_t features;
|
||||
uint8_t lbaformatcount;
|
||||
uint8_t lbaformattedsize;
|
||||
uint8_t metadatacap;
|
||||
uint8_t endtoendprot;
|
||||
uint8_t endtoendprotsettings;
|
||||
uint8_t sharingcap;
|
||||
uint8_t rescap;
|
||||
uint8_t fpi;
|
||||
uint8_t deallocate;
|
||||
uint16_t atomicwrite;
|
||||
uint16_t atomicwritepowerfail;
|
||||
uint16_t atomiccomparewrite;
|
||||
uint16_t atomicboundarysize;
|
||||
uint16_t atomicboundaryoffset;
|
||||
uint16_t atomicboundarypowerfail;
|
||||
uint16_t optimalioboundary;
|
||||
uint64_t nvmcapacity[2];
|
||||
uint16_t preferredwritegranularity;
|
||||
uint16_t preferredwritealignment;
|
||||
uint16_t preferreddeallocategranularity;
|
||||
uint16_t preferreddeallocatealignment;
|
||||
uint16_t optimalwritesize;
|
||||
uint8_t _reserved[18];
|
||||
uint32_t anagrpid;
|
||||
uint8_t __reserved[3];
|
||||
uint8_t namespaceattr;
|
||||
uint16_t nvmsetid;
|
||||
uint16_t endgid;
|
||||
uint64_t namespaceguid[2];
|
||||
uint64_t eui64;
|
||||
lbaformat_t lbaformat[16];
|
||||
} __attribute__((packed)) namespaceid_t;
|
||||
|
||||
#define CONTROLLER_TYPE_IO 1
|
||||
typedef struct {
|
||||
uint16_t pcivendor;
|
||||
uint16_t pcisubsystemvendor;
|
||||
uint8_t serialnum[20];
|
||||
uint8_t modelnum[40];
|
||||
uint64_t fwrevision;
|
||||
uint8_t recommendedburst;
|
||||
uint8_t ouiid[3];
|
||||
uint8_t namespacesharing;
|
||||
uint8_t maxdatatransfer;
|
||||
uint16_t controllerid;
|
||||
uint32_t version;
|
||||
uint32_t rtd3resumelatency;
|
||||
uint32_t rtd3entrylatency;
|
||||
uint32_t optionalasyncevents;
|
||||
uint32_t attributes;
|
||||
uint16_t rrlsupported;
|
||||
uint8_t reserved[9];
|
||||
uint8_t type;
|
||||
uint64_t fguid[2];
|
||||
uint16_t crdt[3];
|
||||
uint8_t _reserved[106];
|
||||
uint8_t mi_res[16];
|
||||
uint16_t admincmdsupport;
|
||||
uint8_t abortlimit;
|
||||
uint8_t asynceventrqlimit;
|
||||
uint8_t firmwareupdates;
|
||||
uint8_t logpages;
|
||||
uint8_t errorlogpages;
|
||||
uint8_t powerstatenum;
|
||||
uint8_t adminvendorspecific;
|
||||
uint8_t apsta;
|
||||
uint16_t wctemp;
|
||||
uint16_t cctemp;
|
||||
uint16_t mtfa;
|
||||
uint32_t hmpre;
|
||||
uint32_t hmmin;
|
||||
uint64_t tnvmcap[2];
|
||||
uint64_t unvmcap[2];
|
||||
uint32_t rpmbs;
|
||||
uint16_t edstt;
|
||||
uint8_t dsto;
|
||||
uint8_t fwug;
|
||||
uint16_t kas;
|
||||
uint16_t hctma;
|
||||
uint16_t mntmt;
|
||||
uint16_t mxtmt;
|
||||
uint32_t sanicap;
|
||||
uint32_t hmminds;
|
||||
uint16_t hmmaxd;
|
||||
uint16_t nsetidmax;
|
||||
uint16_t endgidmax;
|
||||
uint8_t anatt;
|
||||
uint8_t anacap;
|
||||
uint32_t anagrpmax;
|
||||
uint32_t nanagrpid;
|
||||
uint32_t pels;
|
||||
uint8_t __reserved[156];
|
||||
uint8_t sqentrysize; // powers of 2
|
||||
uint8_t cqentrysize;
|
||||
uint16_t maxcmd;
|
||||
uint32_t namespacecount;
|
||||
uint16_t oncs;
|
||||
uint16_t fuses;
|
||||
uint8_t fna;
|
||||
uint8_t vwc;
|
||||
uint16_t awun;
|
||||
uint16_t awupf;
|
||||
uint8_t nvscc;
|
||||
uint8_t nwpc;
|
||||
uint16_t acwu;
|
||||
} __attribute__((packed)) controllerid_t;
|
||||
|
||||
typedef struct {
|
||||
uint32_t dword0;
|
||||
uint32_t namespace;
|
||||
@@ -42,15 +196,9 @@ typedef struct {
|
||||
uint32_t value;
|
||||
uint32_t reserved;
|
||||
uint32_t subinfo;
|
||||
uint32_t commandinfo;
|
||||
uint32_t cmdinfo;
|
||||
} __attribute__((packed)) compentry_t;
|
||||
|
||||
uint32_t reserved;
|
||||
uint32_t csts;
|
||||
uint32_t nssr;
|
||||
uint32_t aqa;
|
||||
uint64_t asq;
|
||||
uint64_t acq;
|
||||
typedef struct {
|
||||
uint64_t cap;
|
||||
uint32_t version;
|
||||
@@ -65,11 +213,273 @@ typedef struct {
|
||||
uint64_t acqbase;
|
||||
} __attribute__((packed)) nvmebar0_t;
|
||||
|
||||
typedef struct {
|
||||
compentry_t comp;
|
||||
subentry_t sub;
|
||||
event_t event;
|
||||
} entrypair_t;
|
||||
|
||||
typedef struct {
|
||||
void *address;
|
||||
size_t entrycount;
|
||||
int index;
|
||||
volatile uint32_t *doorbell;
|
||||
int phase;
|
||||
} queuedesc_t;
|
||||
|
||||
#define QUEUEPAIR_ENTRY_COUNT 256
|
||||
|
||||
typedef struct {
|
||||
queuedesc_t submission;
|
||||
queuedesc_t completion;
|
||||
struct nvmecontroller_t *controller;
|
||||
dpc_t dpc;
|
||||
spinlock_t lock;
|
||||
semaphore_t entrysem;
|
||||
entrypair_t *entries[QUEUEPAIR_ENTRY_COUNT];
|
||||
} queuepair_t;
|
||||
|
||||
typedef struct nvmecontroller_t {
|
||||
volatile nvmebar0_t *bar0;
|
||||
int id;
|
||||
size_t dbstride;
|
||||
size_t maxentries;
|
||||
queuepair_t adminqueue;
|
||||
size_t paircount;
|
||||
uintmax_t queueindex;
|
||||
queuepair_t *ioqueues;
|
||||
} nvmecontroller_t;
|
||||
|
||||
#define CAP_COMMANDSET_NVM 1
|
||||
|
||||
static void nvme_dpc(context_t *, dpcarg_t arg) {
|
||||
queuepair_t *pair = arg;
|
||||
spinlock_acquire(&pair->lock);
|
||||
|
||||
compentry_t *queue = (compentry_t *)pair->completion.address;
|
||||
|
||||
int count = 0;
|
||||
|
||||
while (COMP_CMDINFO_PHASE(queue[pair->completion.index].cmdinfo) == pair->completion.phase) {
|
||||
int subid = COMP_CMDINFO_CMDID(queue[pair->completion.index].cmdinfo);
|
||||
pair->entries[subid]->comp = queue[pair->completion.index];
|
||||
|
||||
event_signal(&pair->entries[subid]->event);
|
||||
|
||||
pair->entries[subid] = NULL;
|
||||
|
||||
++pair->completion.index;
|
||||
pair->completion.index %= pair->completion.entrycount;
|
||||
if (pair->completion.index == 0)
|
||||
pair->completion.phase = !pair->completion.phase;
|
||||
|
||||
++count;
|
||||
}
|
||||
|
||||
if (count)
|
||||
*pair->completion.doorbell = pair->completion.index;
|
||||
|
||||
spinlock_release(&pair->lock);
|
||||
}
|
||||
|
||||
static hashtable_t irqtable;
|
||||
|
||||
static void nvme_irq(isr_t *isr, context_t *context) {
|
||||
void *v;
|
||||
__assert(hashtable_get(&irqtable, &v, &isr->id, sizeof(isr->id)) == 0);
|
||||
queuepair_t *pair = v;
|
||||
dpc_enqueue(&pair->dpc, nvme_dpc, pair);
|
||||
}
|
||||
static isr_t *msixnewisrforqueue(queuepair_t *queuepair) {
|
||||
isr_t *isr = interrupt_allocate(nvme_irq, ARCH_EOI, IPL_DISK);
|
||||
__assert(isr);
|
||||
__assert(hashtable_set(&irqtable, queuepair, &isr->id, sizeof(isr->id), true) == 0);
|
||||
return isr;
|
||||
}
|
||||
|
||||
static void enqueue(queuepair_t *queuepair, entrypair_t *entries) {
|
||||
bool intstatus = interrupt_set(false);
|
||||
semaphore_wait(&queuepair->entrysem, false);
|
||||
|
||||
subentry_t *subqueue = queuepair->submission.address;
|
||||
spinlock_acquire(&queuepair->lock);
|
||||
|
||||
int pair = 0;
|
||||
while (queuepair->entries[pair]) ++pair;
|
||||
|
||||
SUB_SETDW0ID(entries->sub.dword0, pair);
|
||||
queuepair->entries[pair] = entries;
|
||||
|
||||
subqueue[queuepair->submission.index++] = entries->sub;
|
||||
queuepair->submission.index %= queuepair->submission.entrycount;
|
||||
|
||||
*queuepair->submission.doorbell = queuepair->submission.index;
|
||||
spinlock_release(&queuepair->lock);
|
||||
interrupt_set(intstatus);
|
||||
}
|
||||
|
||||
static void enqueueandwait(queuepair_t *queuepair, entrypair_t *entries) {
|
||||
bool intstatus = interrupt_set(false);
|
||||
|
||||
enqueue(queuepair, entries);
|
||||
event_wait(&entries->event, false);
|
||||
|
||||
interrupt_set(intstatus);
|
||||
}
|
||||
|
||||
#define IDENTIFY_SIZE 4096
|
||||
#define IDENTIFY_WHAT_NAMESPACE 0
|
||||
#define IDENTIFY_WHAT_CONTROLLER 1
|
||||
#define IDENTIFY_WHAT_NAMESPACELIST 2
|
||||
|
||||
static int identify(nvmecontroller_t *controller, void *buffer, int what, int namespace) {
|
||||
entrypair_t pair = {0};
|
||||
PAIR_INIT(&pair, SUB_DW0_OPCODE_IDENTIFY, SUB_DW0_UNFUSED, SUB_DW0_PRP, 0);
|
||||
|
||||
pair.sub.datapointer[0] = (uint64_t)pmm_alloc(1, PMM_SECTION_DEFAULT);
|
||||
if (pair.sub.datapointer[0] == 0)
|
||||
return ENOMEM;
|
||||
|
||||
pair.sub.command[0] = what;
|
||||
pair.sub.namespace = namespace;
|
||||
|
||||
enqueueandwait(&controller->adminqueue, &pair);
|
||||
|
||||
memcpy(buffer, MAKE_HHDM(pair.sub.datapointer[0]), IDENTIFY_SIZE);
|
||||
return COMP_CMDINFO_STATUS(pair.comp.cmdinfo) ? EIO : 0;
|
||||
}
|
||||
|
||||
#define IDENTIFY_CONTROLLER(ctlr, buff) identify(ctlr, buff, IDENTIFY_WHAT_CONTROLLER, 0)
|
||||
#define IDENTIFY_NAMESPACE(ctlr, buff, nsid) identify(ctlr, buff, IDENTIFY_WHAT_NAMESPACE, nsid)
|
||||
#define IDENTIFY_NAMESPACELIST(ctlr, buff) identify(ctlr, buff, IDENTIFY_WHAT_NAMESPACELIST, 0)
|
||||
|
||||
#define FEATURE_NUMBEROFQUEUES 0x07
|
||||
#define FEATURE_SOFTWAREPROGRESS 0x80
|
||||
|
||||
static int resetsoftwareprogress(nvmecontroller_t *controller) {
|
||||
entrypair_t pair = {0};
|
||||
PAIR_INIT(&pair, SUB_DW0_OPCODE_SETFEATURES, SUB_DW0_UNFUSED, SUB_DW0_PRP, 0);
|
||||
|
||||
pair.sub.command[0] = FEATURE_SOFTWAREPROGRESS;
|
||||
|
||||
enqueueandwait(&controller->adminqueue, &pair);
|
||||
|
||||
return COMP_CMDINFO_STATUS(pair.comp.cmdinfo) ? EIO : 0;
|
||||
}
|
||||
|
||||
static int allocateioqueues(nvmecontroller_t *controller, size_t count, size_t *returned) {
|
||||
entrypair_t pair = {0};
|
||||
PAIR_INIT(&pair, SUB_DW0_OPCODE_SETFEATURES, SUB_DW0_UNFUSED, SUB_DW0_PRP, 0);
|
||||
|
||||
count -= 1;
|
||||
pair.sub.command[0] = FEATURE_NUMBEROFQUEUES;
|
||||
pair.sub.command[1] = (count << 16) | count;
|
||||
|
||||
enqueueandwait(&controller->adminqueue, &pair);
|
||||
|
||||
size_t subcount = pair.comp.value & 0xffff;
|
||||
size_t compcount = (pair.comp.value >> 16) & 0xffff;
|
||||
|
||||
*returned = min(subcount + 1, compcount + 1);
|
||||
|
||||
return COMP_CMDINFO_STATUS(pair.comp.cmdinfo) ? EIO : 0;
|
||||
}
|
||||
|
||||
#define COMPQUEUE_INTENABLED 0x2
|
||||
#define IOQUEUE_PHYSICALLYCONTIGUOUS 0x1
|
||||
|
||||
static int createiocompqueue(nvmecontroller_t *controller, queuedesc_t *desc, size_t entrycount, int id, int vec, int flags) {
|
||||
entrypair_t pair = {0};
|
||||
PAIR_INIT(&pair, SUB_DW0_OPCODE_CREATEIOCOMPQUEUE, SUB_DW0_UNFUSED, SUB_DW0_PRP, 0);
|
||||
|
||||
size_t size = entrycount * sizeof(compentry_t);
|
||||
size_t pagesize = ROUND_UP(size, PAGE_SIZE) / PAGE_SIZE;
|
||||
|
||||
void *queuepages = pmm_alloc(pagesize, PMM_SECTION_DEFAULT);
|
||||
if (queuepages == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
void *pageshhdm = MAKE_HHDM(queuepages);
|
||||
memset(pageshhdm, 0, size);
|
||||
|
||||
pair.sub.datapointer[0] = (uint64_t)queuepages;
|
||||
pair.sub.command[0] = (id & 0xffff) | (entrycount - 1) << 16;
|
||||
pair.sub.command[1] = COMPQUEUE_INTENABLED | IOQUEUE_PHYSICALLYCONTIGUOUS | (vec << 16);
|
||||
|
||||
enqueueandwait(&controller->adminqueue, &pair);
|
||||
|
||||
if (COMP_CMDINFO_STATUS(pair.comp.cmdinfo)) {
|
||||
pmm_free(queuepages, pagesize);
|
||||
return EIO;
|
||||
}
|
||||
|
||||
desc->address = pageshhdm;
|
||||
desc->entrycount = entrycount;
|
||||
desc->doorbell = GET_DOORBELL(controller->bar0, id, 1, controller->dbstride);
|
||||
desc->phase = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int createiosubqueue(nvmecontroller_t *controller, queuedesc_t *desc, size_t entrycount, int id, int compid, int flags) {
|
||||
entrypair_t pair = {0};
|
||||
PAIR_INIT(&pair, SUB_DW0_OPCODE_CREATEIOSUBQUEUE, SUB_DW0_UNFUSED, SUB_DW0_PRP, 0);
|
||||
|
||||
size_t size = entrycount * sizeof(compentry_t);
|
||||
size_t pagesize = ROUND_UP(size, PAGE_SIZE) / PAGE_SIZE;
|
||||
|
||||
void *queuepages = pmm_alloc(pagesize, PMM_SECTION_DEFAULT);
|
||||
if (queuepages == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
void *pageshhdm = MAKE_HHDM(queuepages);
|
||||
memset(pageshhdm, 0, size);
|
||||
|
||||
pair.sub.datapointer[0] = (uint64_t)queuepages;
|
||||
pair.sub.command[0] = (id & 0xffff) | (entrycount - 1) << 16;
|
||||
pair.sub.command[1] = IOQUEUE_PHYSICALLYCONTIGUOUS | (compid << 16);
|
||||
|
||||
enqueueandwait(&controller->adminqueue, &pair);
|
||||
|
||||
if (COMP_CMDINFO_STATUS(pair.comp.cmdinfo)) {
|
||||
pmm_free(queuepages, pagesize);
|
||||
return EIO;
|
||||
}
|
||||
|
||||
desc->address = pageshhdm;
|
||||
desc->entrycount = entrycount;
|
||||
desc->doorbell = GET_DOORBELL(controller->bar0, id, 0, controller->dbstride);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void newioqueuepair(nvmecontroller_t *controller, queuepair_t *pair, int id) {
|
||||
__assert(createiocompqueue(controller, &pair->completion, QUEUEPAIR_ENTRY_COUNT, id, id, 0) == 0);
|
||||
__assert(createiosubqueue(controller, &pair->submission, QUEUEPAIR_ENTRY_COUNT, id, id, 0) == 0);
|
||||
pair->controller = controller;
|
||||
SPINLOCK_INIT(pair->lock);
|
||||
SEMAPHORE_INIT(&pair->entrysem, QUEUEPAIR_ENTRY_COUNT);
|
||||
}
|
||||
|
||||
static void initnamespace(nvmecontroller_t *controller, int id) {
|
||||
// identify namespace
|
||||
namespaceid_t *namespaceid = alloc(IDENTIFY_SIZE);
|
||||
__assert(namespaceid);
|
||||
__assert(IDENTIFY_NAMESPACE(controller, namespaceid, id) == 0);
|
||||
|
||||
// allocate namespace structure
|
||||
// get lba format
|
||||
// get blocksize
|
||||
// get capacity
|
||||
// register to the outside
|
||||
|
||||
free(namespaceid);
|
||||
}
|
||||
|
||||
#define MAX_PAIRS_PER_CONTROLLER 16
|
||||
|
||||
static int ctlrid;
|
||||
static void initcontroller(pcienum_t *e) {
|
||||
pcibar_t bar0p = pci_getbar(e, 0);
|
||||
volatile nvmebar0_t *bar0 = pci_mapbar(bar0p);
|
||||
volatile nvmebar0_t *bar0 = (volatile nvmebar0_t *)bar0p.address;
|
||||
__assert(bar0);
|
||||
|
||||
int vmajor = VERSION_MAJOR(bar0->version);
|
||||
@@ -93,22 +503,24 @@ static void initcontroller(pcienum_t *e) {
|
||||
printf("nvme: controller doesn't support the processor page size\n");
|
||||
}
|
||||
|
||||
// set command stuff in config space
|
||||
|
||||
pci_setcommand(e, PCI_COMMAND_MMIO, 1);
|
||||
pci_setcommand(e, PCI_COMMAND_IO, 0);
|
||||
|
||||
// enable interrupts
|
||||
|
||||
size_t intcount;
|
||||
if (e->msix.exists) {
|
||||
|
||||
intcount = pci_initmsix(e);
|
||||
} else if (e->msi.exists) {
|
||||
|
||||
printf("nvme: driver only supports msi-x currently\n");
|
||||
return;
|
||||
} else {
|
||||
printf("nvme: controller doesn't support msi-x or msi\n");
|
||||
return;
|
||||
}
|
||||
|
||||
__assert(intcount);
|
||||
|
||||
// reset controller
|
||||
uint32_t cc = bar0->cc;
|
||||
CC_DISABLE(cc);
|
||||
@@ -132,12 +544,12 @@ static void initcontroller(pcienum_t *e) {
|
||||
|
||||
bar0->aqattr = aqattr;
|
||||
|
||||
void *acs = pmm_alloc(2, PMM_SECTION_DEFAULT);
|
||||
__assert(acs);
|
||||
memset(MAKE_HHDM(acs), 0, PAGE_SIZE * 2);
|
||||
void *asq = (void *)((uintptr_t)acs + PAGE_SIZE);
|
||||
void *acq = pmm_alloc(2, PMM_SECTION_DEFAULT);
|
||||
__assert(acq);
|
||||
memset(MAKE_HHDM(acq), 0, PAGE_SIZE * 2);
|
||||
void *asq = (void *)((uintptr_t)acq + PAGE_SIZE);
|
||||
|
||||
bar0->acqbase = (uint64_t)acs;
|
||||
bar0->acqbase = (uint64_t)acq;
|
||||
bar0->asqbase = (uint64_t)asq;
|
||||
|
||||
// enable and wait for completion
|
||||
@@ -150,9 +562,78 @@ static void initcontroller(pcienum_t *e) {
|
||||
printf("nvme: controller returned fatal while initializing\n");
|
||||
return;
|
||||
}
|
||||
|
||||
nvmecontroller_t *controller = alloc(sizeof(nvmecontroller_t));
|
||||
__assert(controller);
|
||||
|
||||
controller->id = ctlrid++;
|
||||
controller->bar0 = bar0;
|
||||
controller->dbstride = CAP_DOORBELLSTRIDE(bar0->cap);
|
||||
controller->maxentries = CAP_MAXENTRIES(bar0->cap);
|
||||
|
||||
controller->adminqueue.submission.address = MAKE_HHDM(asq);
|
||||
controller->adminqueue.submission.entrycount = PAGE_SIZE / sizeof(subentry_t);
|
||||
controller->adminqueue.submission.doorbell = GET_DOORBELL(controller->bar0, 0, 0, controller->dbstride);
|
||||
|
||||
controller->adminqueue.completion.address = MAKE_HHDM(acq);
|
||||
controller->adminqueue.completion.entrycount = PAGE_SIZE / sizeof(compentry_t);
|
||||
controller->adminqueue.completion.doorbell = GET_DOORBELL(controller->bar0, 0, 1, controller->dbstride);
|
||||
controller->adminqueue.completion.phase = 1;
|
||||
|
||||
SPINLOCK_INIT(controller->adminqueue.lock);
|
||||
SEMAPHORE_INIT(&controller->adminqueue.entrysem, QUEUEPAIR_ENTRY_COUNT);
|
||||
|
||||
isr_t *adminisr = NULL;
|
||||
if (e->msix.exists) {
|
||||
adminisr = msixnewisrforqueue(&controller->adminqueue);
|
||||
pci_msixadd(e, 0, INTERRUPT_IDTOVECTOR(adminisr->id), 1, 0);
|
||||
pci_msixsetmask(e, 0);
|
||||
}
|
||||
|
||||
// identify controller
|
||||
controllerid_t *controllerid = alloc(IDENTIFY_SIZE);
|
||||
__assert(controllerid);
|
||||
__assert(IDENTIFY_CONTROLLER(controller, controllerid) == 0);
|
||||
|
||||
__assert(controllerid->type == CONTROLLER_TYPE_IO);
|
||||
|
||||
resetsoftwareprogress(controller);
|
||||
|
||||
// get namespace list
|
||||
uint32_t *namespacelist = alloc(IDENTIFY_SIZE);
|
||||
__assert(namespacelist);
|
||||
__assert(IDENTIFY_NAMESPACELIST(controller, namespacelist) == 0);
|
||||
|
||||
// determine how many pairs to use
|
||||
size_t iomin = min(intcount, MAX_PAIRS_PER_CONTROLLER);
|
||||
size_t iocount;
|
||||
__assert(allocateioqueues(controller, iomin, &iocount) == 0);
|
||||
iocount = min(iomin, iocount);
|
||||
printf("nvme: using %lu I/O queues\n", iocount);
|
||||
controller->paircount = iocount;
|
||||
|
||||
// create i/o queues
|
||||
controller->ioqueues = alloc(sizeof(queuepair_t) * iocount);
|
||||
__assert(controller->ioqueues);
|
||||
|
||||
for (int i = 0; i < iocount; ++i) {
|
||||
newioqueuepair(controller, &controller->ioqueues[i], i + 1);
|
||||
if (e->msix.exists) {
|
||||
isr_t *isr = msixnewisrforqueue(&controller->ioqueues[i]);
|
||||
pci_msixadd(e, i + 1, INTERRUPT_IDTOVECTOR(isr->id), 1, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// initialize namespaces
|
||||
for (int i = 0; i < 1024 && namespacelist[i]; ++i)
|
||||
initnamespace(controller, namespacelist[i]);
|
||||
|
||||
free(namespacelist);
|
||||
free(controllerid);
|
||||
}
|
||||
|
||||
void nvme_init() {
|
||||
__assert(hashtable_init(&irqtable, 10) == 0);
|
||||
int i = 0;
|
||||
for (;;) {
|
||||
pcienum_t *e = pci_getenum(PCI_CLASS_STORAGE, PCI_SUBCLASS_STORAGE_NVM, -1, -1, -1, -1, i++);
|
||||
|
||||
Reference in New Issue
Block a user