block: read page by page rather than block by block

kernel-src/include/kernel/vmm.h

@@ -11,6 +11,7 @@
 #define VMM_FLAGS_FILE     8
 #define VMM_FLAGS_EXACT   16
 #define VMM_FLAGS_SHARED  32
+#define VMM_FLAGS_REPLACE 64
 
 #define VMM_PERMANENT_FLAGS_MASK (VMM_FLAGS_FILE | VMM_FLAGS_SHARED | VMM_FLAGS_PHYSICAL)
 
@@ -55,7 +56,6 @@ typedef struct vmmcache_t {
 
 typedef struct {
 	mutex_t lock;
-	mutex_t pflock;
 	vmmrange_t *ranges;
 	void *start;
 	void *end;

kernel-src/io/block/block.c

@@ -179,7 +179,7 @@ static int registerdesc(blockdesc_t *desc, char *name) {
 
 static int detectpart(blockdesc_t *desc) {
 	// read first two sectors
-	void *sects = alloc(desc->blocksize * 2);
+	void *sects = vmm_map(NULL, desc->blocksize * 2, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL);
 	__assert(sects);
 	__assert(DISK_READ(desc, sects, 0, 2) == 0);
 
@@ -195,13 +195,13 @@ static int detectpart(blockdesc_t *desc) {
 	else if (bootmagic == MBR_BOOT_MAGIC)
 		ret = PART_MBR;
 
-	free(sects);
+	vmm_unmap(sects, desc->blocksize * 2, 0);
 	return ret;
 }
 
 static void dogpt(blockdesc_t *desc, char *name) {
 	// get header
-	void *lba1 = alloc(desc->blocksize);
+	void *lba1 = vmm_map(NULL, desc->blocksize, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL);
 	__assert(lba1);
 	__assert(DISK_READ(desc, lba1, 1, 1) == 0);
 	gptheader_t *header = lba1;
@@ -209,7 +209,7 @@ static void dogpt(blockdesc_t *desc, char *name) {
 	// get partition table
 	size_t tablebytesize = header->entrybytesize * header->entrycount;
 	size_t tablelbasize = ROUND_UP(tablebytesize, desc->blocksize) / desc->blocksize;
-	void *tablebuffer = alloc(tablelbasize * desc->blocksize);
+	void *tablebuffer = vmm_map(NULL, tablelbasize * desc->blocksize, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL);
 	__assert(tablebuffer);
 	__assert(DISK_READ(desc, tablebuffer, header->entryarraylbastart, tablelbasize) == 0);
 
@@ -237,13 +237,13 @@ static void dogpt(blockdesc_t *desc, char *name) {
 		__assert(registerdesc(partdesc, partname) == 0);
 	}
 
-	free(tablebuffer);
+	vmm_unmap(tablebuffer, tablelbasize * desc->blocksize, 0);
-	free(lba1);
+	vmm_unmap(lba1, desc->blocksize, 0);
 }
 
 static void dombr(blockdesc_t *desc, char *name) {
 	// get entries
-	void *lba0 = alloc(desc->blocksize);
+	void *lba0 = vmm_map(NULL, desc->blocksize, VMM_FLAGS_ALLOCATE, MAP_FLAGS, NULL);
 	__assert(lba0);
 	__assert(DISK_READ(desc, lba0, 0, 1) == 0);
 	mbrentry_t *mbrents = (mbrentry_t *)((uintptr_t)lba0 + MBR_ENTRIES_OFFSET);
@@ -270,7 +270,7 @@ static void dombr(blockdesc_t *desc, char *name) {
 		__assert(registerdesc(partdesc, partname) == 0);
 	}
 
-	free(lba0);
+	vmm_unmap(lba0, desc->blocksize, 0);
 }
 
 void block_register(blockdesc_t *desc, char *name) {

kernel-src/io/block/nvme.c

@@ -506,44 +506,40 @@ static int iowrite(nvmenamespace_t *namespace, uint64_t prp[2], uint64_t lba, ui
 
 #define PAGES_FLAGS (ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC)
 
-static int rwblocks(nvmenamespace_t *namespace, void *buffer, uintmax_t lba, size_t count, bool write, bool pagealignedbuffer) {
+static int rwblocks(nvmenamespace_t *namespace, void *buffer, uintmax_t lba, size_t count, bool write) {
-	// TODO this is pretty inefficient and will be replaced later
 	__assert(namespace->blocksize <= PAGE_SIZE);
 
-	uint64_t prp[2] = {(uint64_t)pmm_allocpage(PMM_SECTION_DEFAULT), 0};
 	int err = 0;
-	if (prp[0] == 0)
-		goto cleanup;
 
-	for (int i = 0; i < count; ++i) {
+	uint64_t prp[2] = {0, 0};
-		void *bufferp = (void *)((uintptr_t)buffer + i * namespace->blocksize);
+	__assert(((uintptr_t)buffer % namespace->blocksize) == 0);
+	uintmax_t pageoffset = (uintptr_t)buffer - ROUND_DOWN((uintptr_t)buffer, PAGE_SIZE);
 
-		if (write) {
+	int done = 0;
-			memcpy(MAKE_HHDM(prp[0]), bufferp, namespace->blocksize);
+	while (done < count) {
-		}
+		size_t docount = (done == 0) ? (PAGE_SIZE - pageoffset) / namespace->blocksize : PAGE_SIZE / namespace->blocksize;
+		docount = min(docount, count - done);
+
+		prp[0] = (uint64_t)vmm_getphysical((void *)((uintptr_t)buffer + done * namespace->blocksize));
+		__assert(prp[0]);
+
+		err = write ? iowrite(namespace, prp, lba + done, docount) : ioread(namespace, prp, lba + done, docount);
 
-		err = write ? iowrite(namespace, prp, lba + i, 1) : ioread(namespace, prp, lba + i, 1);
 		if (err)
-			goto cleanup;
+			break;
 
-		if (!write) {
+		done += docount;
-			memcpy(bufferp, MAKE_HHDM(prp[0]), namespace->blocksize);
-		}
 	}
 
-	cleanup:
-	if (prp[0])
-		pmm_release((void *)prp[0]);
-
 	return err;
 }
 
 static int read(void *private, void *buffer, uintmax_t lba, size_t count) {
-	return rwblocks(private, buffer, lba, count, false, false);
+	return rwblocks(private, buffer, lba, count, false);
 }
 
 static int write(void *private, void *buffer, uintmax_t lba, size_t count) {
-	return rwblocks(private, buffer, lba, count, true, false);
+	return rwblocks(private, buffer, lba, count, true);
 }
 
 static void initnamespace(nvmecontroller_t *controller, int id) {

kernel-src/io/block/vioblk.c

@@ -8,8 +8,7 @@
 #include <kernel/pmm.h>
 #include <string.h>
 
-#define QUEUE_MAX_SIZE 128
+#define QUEUE_MAX_SIZE 144
-#define OTHER_BUFFER 64
 
 typedef struct {
 	uint64_t capacity;
@@ -23,14 +22,14 @@ typedef struct {
 	dpc_t queuedpc;
 	spinlock_t queuelock;
 	semaphore_t queuesem;
-	thread_t *queuewaiting[QUEUE_MAX_SIZE / 2];
+	thread_t *queuewaiting[QUEUE_MAX_SIZE / 3];
 } vioblkdev_t;
 
 typedef struct {
 	uint32_t type;
 	uint32_t reserved;
 	uint64_t sector;
-} requestheader_t;
+} __attribute__((packed)) requestheader_t;
 
 #define HEADER_TYPE_READ 0
 #define HEADER_TYPE_WRITE 1
@@ -44,12 +43,12 @@ static void vioblk_dpc(context_t *context, dpcarg_t arg) {
 	while (blkdev->queue.lastusedindex != VIO_QUEUE_DEV_IDX(&blkdev->queue)) {
 		int idx = blkdev->queue.lastusedindex++ % blkdev->queue.size;
 		int buffidx = VIO_QUEUE_DEV_RING(&blkdev->queue)[idx].index;
-		__assert(buffidx < OTHER_BUFFER);
+		__assert(blkdev->queuewaiting[buffidx / 3]);
-		__assert(blkdev->queuewaiting[buffidx]);
+		sched_wakeup(blkdev->queuewaiting[buffidx / 3], SCHED_WAKEUP_REASON_NORMAL);
-		sched_wakeup(blkdev->queuewaiting[buffidx], SCHED_WAKEUP_REASON_NORMAL);
+		blkdev->queuewaiting[buffidx / 3] = NULL;
-		blkdev->queuewaiting[buffidx] = NULL;
 		buffers[buffidx].address = 0;
-		buffers[buffidx + OTHER_BUFFER].address = 0;
+		buffers[buffidx + 1].address = 0;
+		buffers[buffidx + 2].address = 0;
 		semaphore_signal(&blkdev->queuesem);
 	}
 	spinlock_release(&blkdev->queuelock);
@@ -63,32 +62,37 @@ static void vioblk_irq(isr_t *isr, context_t *context) {
 }
 
 // driver and device are physical addresses
-static void vioblk_enqueue(vioblkdev_t *blkdev, void *driver, size_t driverlen, void *device, size_t devicelen) {
+static void vioblk_enqueue(vioblkdev_t *blkdev, requestheader_t *headerphys, void *bufferphys, size_t bufferlen, uint8_t *statusphys, bool write) {
 	bool intstatus = interrupt_set(false);
 	semaphore_wait(&blkdev->queuesem, false);
 
 	spinlock_acquire(&blkdev->queuelock);
 	int idx = 0;
 	volatile viobuffer_t *buffers = VIO_QUEUE_BUFFERS(&blkdev->queue);
-	while (buffers[idx].address && idx < QUEUE_MAX_SIZE / 2)
+	while (buffers[idx].address && idx < blkdev->queue.size)
-		++idx;
+		idx += 3;
 
-	__assert(idx < QUEUE_MAX_SIZE);
+	__assert(idx < blkdev->queue.size);
 
-	buffers[idx].address = (uint64_t)driver;
+	buffers[idx].address = (uint64_t)headerphys;
-	buffers[idx].length = driverlen;
+	buffers[idx].length = sizeof(requestheader_t);
 	buffers[idx].flags = VIO_QUEUE_BUFFER_NEXT;
-	buffers[idx].next = idx + OTHER_BUFFER;
+	buffers[idx].next = idx + 1;
 
-	buffers[idx + OTHER_BUFFER].address = (uint64_t)device;
+	buffers[idx + 1].address = (uint64_t)bufferphys;
-	buffers[idx + OTHER_BUFFER].length = devicelen;
+	buffers[idx + 1].length = bufferlen;
-	buffers[idx + OTHER_BUFFER].flags = VIO_QUEUE_BUFFER_DEVICE;
+	buffers[idx + 1].flags = (write ? 0 : VIO_QUEUE_BUFFER_DEVICE) | VIO_QUEUE_BUFFER_NEXT;
+	buffers[idx + 1].next = idx + 2;
+
+	buffers[idx + 2].address = (uint64_t)statusphys;
+	buffers[idx + 2].length = 1;
+	buffers[idx + 2].flags = VIO_QUEUE_BUFFER_DEVICE;
 
 	size_t driveridx = VIO_QUEUE_DRV_IDX(&blkdev->queue);
 	VIO_QUEUE_DRV_RING(&blkdev->queue)[driveridx % blkdev->queue.size] = idx;
 
 	sched_preparesleep(false);
-	blkdev->queuewaiting[idx] = _cpu()->thread;
+	blkdev->queuewaiting[idx / 3] = _cpu()->thread;
 
 	++VIO_QUEUE_DRV_IDX(&blkdev->queue);
 	*blkdev->queue.notify = 0;
@@ -105,28 +109,33 @@ static int vioblk_rw(vioblkdev_t *blkdev, void *buffer, uintmax_t lba, size_t co
 		return ENOMEM;
 
 	int err = 0;
-	size_t drvlen = sizeof(requestheader_t) + (write ? 512 : 0);
+	requestheader_t *header = physpage;
-	size_t devlen = write ? 1 : 513;
+	requestheader_t *headerhhdm = MAKE_HHDM(physpage);
-	requestheader_t header = {
+	uint8_t *status = (uint8_t *)(header + 1);
-		.type = write ? HEADER_TYPE_WRITE : HEADER_TYPE_READ
+	uint8_t *statushhdm = MAKE_HHDM(status);
-	};
 
-	for (int off = 0; off < count; ++off) {
+	headerhhdm->type = write ? HEADER_TYPE_WRITE : HEADER_TYPE_READ;
-		header.sector = lba + off;
-		memcpy(MAKE_HHDM(physpage), &header, sizeof(requestheader_t));
 
-		if (write)
+	__assert(((uintptr_t)buffer % 512) == 0);
-			memcpy(MAKE_HHDM((uintptr_t)physpage + sizeof(requestheader_t)), (void *)((uintptr_t)buffer + off * 512), 512);
+	uintmax_t pageoffset = (uintptr_t)buffer - ROUND_DOWN((uintptr_t)buffer, PAGE_SIZE);
 
-		vioblk_enqueue(blkdev, physpage, drvlen, (void *)((uintptr_t)physpage + drvlen), devlen);
+	int done = 0;
+	while (done < count) {
+		headerhhdm->sector = lba + done;
+		size_t docount = (done == 0) ? (PAGE_SIZE - pageoffset) / 512 : PAGE_SIZE / 512;
+		docount = min(docount, count - done);
 
-		if (*(uint8_t *)((uintptr_t)MAKE_HHDM(physpage) + drvlen + devlen - 1)) {
+		void *bufferphys = vmm_getphysical((void *)((uintptr_t)buffer + done * 512));
+		__assert(bufferphys);
+
+		vioblk_enqueue(blkdev, header, bufferphys, docount * 512, status, write);
+
+		if (*statushhdm) {
 			err = EIO;
 			break;
 		}
 
-		if (write == false)
+		done += docount;
-			memcpy((void *)((uintptr_t)buffer + off * 512), MAKE_HHDM((uintptr_t)physpage + drvlen), 512);
 	}
 
 	pmm_release(physpage);
@@ -173,8 +182,9 @@ int vioblk_newdevice(viodevice_t *viodevice) {
 
 	// initialize queue
 	size_t size = min(QUEUE_MAX_SIZE, virtio_queuesize(viodevice, 0));
+	size = size - (size % 3);
 	virtio_createqueue(viodevice, &blkdev->queue, 0, size, 0);
-	SEMAPHORE_INIT(&blkdev->queuesem, blkdev->queue.size / 2);
+	SEMAPHORE_INIT(&blkdev->queuesem, blkdev->queue.size / 3);
 	SPINLOCK_INIT(blkdev->queuelock);
 
 	virtio_enablequeue(viodevice, 0);