From ca2ad66dc61e677b85a4f59193d7beaebcc013f6 Mon Sep 17 00:00:00 2001 From: Matheus Date: Sat, 10 Aug 2024 22:43:09 -0300 Subject: [PATCH] block: read page by page rather than block by block --- kernel-src/include/kernel/vmm.h | 2 +- kernel-src/io/block/block.c | 16 +++---- kernel-src/io/block/nvme.c | 38 +++++++--------- kernel-src/io/block/vioblk.c | 80 ++++++++++++++++++--------------- 4 files changed, 71 insertions(+), 65 deletions(-) diff --git a/kernel-src/include/kernel/vmm.h b/kernel-src/include/kernel/vmm.h index 3b5601c1..86872ca9 100644 --- a/kernel-src/include/kernel/vmm.h +++ b/kernel-src/include/kernel/vmm.h @@ -11,6 +11,7 @@ #define VMM_FLAGS_FILE 8 #define VMM_FLAGS_EXACT 16 #define VMM_FLAGS_SHARED 32 +#define VMM_FLAGS_REPLACE 64 #define VMM_PERMANENT_FLAGS_MASK (VMM_FLAGS_FILE | VMM_FLAGS_SHARED | VMM_FLAGS_PHYSICAL) @@ -55,7 +56,6 @@ typedef struct vmmcache_t { typedef struct { mutex_t lock; - mutex_t pflock; vmmrange_t *ranges; void *start; void *end; diff --git a/kernel-src/io/block/block.c b/kernel-src/io/block/block.c index ee46cfeb..875234d3 100644 --- a/kernel-src/io/block/block.c +++ b/kernel-src/io/block/block.c @@ -179,7 +179,7 @@ static int registerdesc(blockdesc_t *desc, char *name) { static int detectpart(blockdesc_t *desc) { // read first two sectors - void *sects = alloc(desc->blocksize * 2); + void *sects = vmm_map(NULL, desc->blocksize * 2, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL); __assert(sects); __assert(DISK_READ(desc, sects, 0, 2) == 0); @@ -195,13 +195,13 @@ static int detectpart(blockdesc_t *desc) { else if (bootmagic == MBR_BOOT_MAGIC) ret = PART_MBR; - free(sects); + vmm_unmap(sects, desc->blocksize * 2, 0); return ret; } static void dogpt(blockdesc_t *desc, char *name) { // get header - void *lba1 = alloc(desc->blocksize); + void *lba1 = vmm_map(NULL, desc->blocksize, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL); __assert(lba1); __assert(DISK_READ(desc, lba1, 1, 1) == 0); gptheader_t *header = lba1; @@ -209,7 +209,7 @@ static void dogpt(blockdesc_t *desc, char *name) { // get partition table size_t tablebytesize = header->entrybytesize * header->entrycount; size_t tablelbasize = ROUND_UP(tablebytesize, desc->blocksize) / desc->blocksize; - void *tablebuffer = alloc(tablelbasize * desc->blocksize); + void *tablebuffer = vmm_map(NULL, tablelbasize * desc->blocksize, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL); __assert(tablebuffer); __assert(DISK_READ(desc, tablebuffer, header->entryarraylbastart, tablelbasize) == 0); @@ -237,13 +237,13 @@ static void dogpt(blockdesc_t *desc, char *name) { __assert(registerdesc(partdesc, partname) == 0); } - free(tablebuffer); - free(lba1); + vmm_unmap(tablebuffer, tablelbasize * desc->blocksize, 0); + vmm_unmap(lba1, desc->blocksize, 0); } static void dombr(blockdesc_t *desc, char *name) { // get entries - void *lba0 = alloc(desc->blocksize); + void *lba0 = vmm_map(NULL, desc->blocksize, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL); __assert(lba0); __assert(DISK_READ(desc, lba0, 0, 1) == 0); mbrentry_t *mbrents = (mbrentry_t *)((uintptr_t)lba0 + MBR_ENTRIES_OFFSET); @@ -270,7 +270,7 @@ static void dombr(blockdesc_t *desc, char *name) { __assert(registerdesc(partdesc, partname) == 0); } - free(lba0); + vmm_unmap(lba0, desc->blocksize, 0); } void block_register(blockdesc_t *desc, char *name) { diff --git a/kernel-src/io/block/nvme.c b/kernel-src/io/block/nvme.c index 06d35de6..1eb2f136 100644 --- a/kernel-src/io/block/nvme.c +++ b/kernel-src/io/block/nvme.c @@ -506,44 +506,40 @@ static int iowrite(nvmenamespace_t *namespace, uint64_t prp[2], uint64_t lba, ui #define PAGES_FLAGS (ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC) -static int rwblocks(nvmenamespace_t *namespace, void *buffer, uintmax_t lba, size_t count, bool write, bool pagealignedbuffer) { - // TODO this is pretty inefficient and will be replaced later +static int rwblocks(nvmenamespace_t *namespace, void *buffer, uintmax_t lba, size_t count, bool write) { __assert(namespace->blocksize <= PAGE_SIZE); - uint64_t prp[2] = {(uint64_t)pmm_allocpage(PMM_SECTION_DEFAULT), 0}; int err = 0; - if (prp[0] == 0) - goto cleanup; - for (int i = 0; i < count; ++i) { - void *bufferp = (void *)((uintptr_t)buffer + i * namespace->blocksize); + uint64_t prp[2] = {0, 0}; + __assert(((uintptr_t)buffer % namespace->blocksize) == 0); + uintmax_t pageoffset = (uintptr_t)buffer - ROUND_DOWN((uintptr_t)buffer, PAGE_SIZE); - if (write) { - memcpy(MAKE_HHDM(prp[0]), bufferp, namespace->blocksize); - } + int done = 0; + while (done < count) { + size_t docount = (done == 0) ? (PAGE_SIZE - pageoffset) / namespace->blocksize : PAGE_SIZE / namespace->blocksize; + docount = min(docount, count - done); + + prp[0] = (uint64_t)vmm_getphysical((void *)((uintptr_t)buffer + done * namespace->blocksize)); + __assert(prp[0]); + + err = write ? iowrite(namespace, prp, lba + done, docount) : ioread(namespace, prp, lba + done, docount); - err = write ? iowrite(namespace, prp, lba + i, 1) : ioread(namespace, prp, lba + i, 1); if (err) - goto cleanup; + break; - if (!write) { - memcpy(bufferp, MAKE_HHDM(prp[0]), namespace->blocksize); - } + done += docount; } - cleanup: - if (prp[0]) - pmm_release((void *)prp[0]); - return err; } static int read(void *private, void *buffer, uintmax_t lba, size_t count) { - return rwblocks(private, buffer, lba, count, false, false); + return rwblocks(private, buffer, lba, count, false); } static int write(void *private, void *buffer, uintmax_t lba, size_t count) { - return rwblocks(private, buffer, lba, count, true, false); + return rwblocks(private, buffer, lba, count, true); } static void initnamespace(nvmecontroller_t *controller, int id) { diff --git a/kernel-src/io/block/vioblk.c b/kernel-src/io/block/vioblk.c index 30f63eaf..003dfb6d 100644 --- a/kernel-src/io/block/vioblk.c +++ b/kernel-src/io/block/vioblk.c @@ -8,8 +8,7 @@ #include #include -#define QUEUE_MAX_SIZE 128 -#define OTHER_BUFFER 64 +#define QUEUE_MAX_SIZE 144 typedef struct { uint64_t capacity; @@ -23,14 +22,14 @@ typedef struct { dpc_t queuedpc; spinlock_t queuelock; semaphore_t queuesem; - thread_t *queuewaiting[QUEUE_MAX_SIZE / 2]; + thread_t *queuewaiting[QUEUE_MAX_SIZE / 3]; } vioblkdev_t; typedef struct { uint32_t type; uint32_t reserved; uint64_t sector; -} requestheader_t; +} __attribute__((packed)) requestheader_t; #define HEADER_TYPE_READ 0 #define HEADER_TYPE_WRITE 1 @@ -44,12 +43,12 @@ static void vioblk_dpc(context_t *context, dpcarg_t arg) { while (blkdev->queue.lastusedindex != VIO_QUEUE_DEV_IDX(&blkdev->queue)) { int idx = blkdev->queue.lastusedindex++ % blkdev->queue.size; int buffidx = VIO_QUEUE_DEV_RING(&blkdev->queue)[idx].index; - __assert(buffidx < OTHER_BUFFER); - __assert(blkdev->queuewaiting[buffidx]); - sched_wakeup(blkdev->queuewaiting[buffidx], SCHED_WAKEUP_REASON_NORMAL); - blkdev->queuewaiting[buffidx] = NULL; + __assert(blkdev->queuewaiting[buffidx / 3]); + sched_wakeup(blkdev->queuewaiting[buffidx / 3], SCHED_WAKEUP_REASON_NORMAL); + blkdev->queuewaiting[buffidx / 3] = NULL; buffers[buffidx].address = 0; - buffers[buffidx + OTHER_BUFFER].address = 0; + buffers[buffidx + 1].address = 0; + buffers[buffidx + 2].address = 0; semaphore_signal(&blkdev->queuesem); } spinlock_release(&blkdev->queuelock); @@ -63,32 +62,37 @@ static void vioblk_irq(isr_t *isr, context_t *context) { } // driver and device are physical addresses -static void vioblk_enqueue(vioblkdev_t *blkdev, void *driver, size_t driverlen, void *device, size_t devicelen) { +static void vioblk_enqueue(vioblkdev_t *blkdev, requestheader_t *headerphys, void *bufferphys, size_t bufferlen, uint8_t *statusphys, bool write) { bool intstatus = interrupt_set(false); semaphore_wait(&blkdev->queuesem, false); spinlock_acquire(&blkdev->queuelock); int idx = 0; volatile viobuffer_t *buffers = VIO_QUEUE_BUFFERS(&blkdev->queue); - while (buffers[idx].address && idx < QUEUE_MAX_SIZE / 2) - ++idx; + while (buffers[idx].address && idx < blkdev->queue.size) + idx += 3; - __assert(idx < QUEUE_MAX_SIZE); + __assert(idx < blkdev->queue.size); - buffers[idx].address = (uint64_t)driver; - buffers[idx].length = driverlen; + buffers[idx].address = (uint64_t)headerphys; + buffers[idx].length = sizeof(requestheader_t); buffers[idx].flags = VIO_QUEUE_BUFFER_NEXT; - buffers[idx].next = idx + OTHER_BUFFER; + buffers[idx].next = idx + 1; - buffers[idx + OTHER_BUFFER].address = (uint64_t)device; - buffers[idx + OTHER_BUFFER].length = devicelen; - buffers[idx + OTHER_BUFFER].flags = VIO_QUEUE_BUFFER_DEVICE; + buffers[idx + 1].address = (uint64_t)bufferphys; + buffers[idx + 1].length = bufferlen; + buffers[idx + 1].flags = (write ? 0 : VIO_QUEUE_BUFFER_DEVICE) | VIO_QUEUE_BUFFER_NEXT; + buffers[idx + 1].next = idx + 2; + + buffers[idx + 2].address = (uint64_t)statusphys; + buffers[idx + 2].length = 1; + buffers[idx + 2].flags = VIO_QUEUE_BUFFER_DEVICE; size_t driveridx = VIO_QUEUE_DRV_IDX(&blkdev->queue); VIO_QUEUE_DRV_RING(&blkdev->queue)[driveridx % blkdev->queue.size] = idx; sched_preparesleep(false); - blkdev->queuewaiting[idx] = _cpu()->thread; + blkdev->queuewaiting[idx / 3] = _cpu()->thread; ++VIO_QUEUE_DRV_IDX(&blkdev->queue); *blkdev->queue.notify = 0; @@ -105,28 +109,33 @@ static int vioblk_rw(vioblkdev_t *blkdev, void *buffer, uintmax_t lba, size_t co return ENOMEM; int err = 0; - size_t drvlen = sizeof(requestheader_t) + (write ? 512 : 0); - size_t devlen = write ? 1 : 513; - requestheader_t header = { - .type = write ? HEADER_TYPE_WRITE : HEADER_TYPE_READ - }; + requestheader_t *header = physpage; + requestheader_t *headerhhdm = MAKE_HHDM(physpage); + uint8_t *status = (uint8_t *)(header + 1); + uint8_t *statushhdm = MAKE_HHDM(status); - for (int off = 0; off < count; ++off) { - header.sector = lba + off; - memcpy(MAKE_HHDM(physpage), &header, sizeof(requestheader_t)); + headerhhdm->type = write ? HEADER_TYPE_WRITE : HEADER_TYPE_READ; - if (write) - memcpy(MAKE_HHDM((uintptr_t)physpage + sizeof(requestheader_t)), (void *)((uintptr_t)buffer + off * 512), 512); + __assert(((uintptr_t)buffer % 512) == 0); + uintmax_t pageoffset = (uintptr_t)buffer - ROUND_DOWN((uintptr_t)buffer, PAGE_SIZE); - vioblk_enqueue(blkdev, physpage, drvlen, (void *)((uintptr_t)physpage + drvlen), devlen); + int done = 0; + while (done < count) { + headerhhdm->sector = lba + done; + size_t docount = (done == 0) ? (PAGE_SIZE - pageoffset) / 512 : PAGE_SIZE / 512; + docount = min(docount, count - done); - if (*(uint8_t *)((uintptr_t)MAKE_HHDM(physpage) + drvlen + devlen - 1)) { + void *bufferphys = vmm_getphysical((void *)((uintptr_t)buffer + done * 512)); + __assert(bufferphys); + + vioblk_enqueue(blkdev, header, bufferphys, docount * 512, status, write); + + if (*statushhdm) { err = EIO; break; } - if (write == false) - memcpy((void *)((uintptr_t)buffer + off * 512), MAKE_HHDM((uintptr_t)physpage + drvlen), 512); + done += docount; } pmm_release(physpage); @@ -173,8 +182,9 @@ int vioblk_newdevice(viodevice_t *viodevice) { // initialize queue size_t size = min(QUEUE_MAX_SIZE, virtio_queuesize(viodevice, 0)); + size = size - (size % 3); virtio_createqueue(viodevice, &blkdev->queue, 0, size, 0); - SEMAPHORE_INIT(&blkdev->queuesem, blkdev->queue.size / 2); + SEMAPHORE_INIT(&blkdev->queuesem, blkdev->queue.size / 3); SPINLOCK_INIT(blkdev->queuelock); virtio_enablequeue(viodevice, 0);