mm: properly implement page locking

kernel-src/include/kernel/iovec.h

@@ -66,10 +66,12 @@ size_t iovec_iterator_read_from_ringbuffer(iovec_iterator_t *iovec_iterator, rin
 // NOTE: the iovec can have userspace addresses. in this case, the copy can possibly fail with RINGBUFFER_USER_COPY_FAILED
 size_t iovec_iterator_peek_from_ringbuffer(iovec_iterator_t *iovec_iterator, ringbuffer_t *ringbuffer, size_t byte_count, uintmax_t ringbuffer_offset);
 
-// returns the physical address of the next page in the iovec_iterator, adding a reference count while at it
+// returns the physical address of the next page in the iovec_iterator, adding a reference count and locking it for DMA. 
+// The write bool controls whether the hint passed to mm_get_physical_address is a read or write (i.e. reading data from the page or 
+// writing data to the page)
 // sets the page_offset to the offset in the page, page_remaining to the remaining number of bytes in the page and page to the physical address of the page
 // if no more pages in iovec_iterator, returns success with page being set to NULL
 // fails with EFAULT if the page is not mapped
-int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offset, size_t *page_remaining, void **page);
+int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offset, size_t *page_remaining, void **page, bool write);
 
 #endif

kernel-src/include/kernel/mm.h

@@ -90,6 +90,8 @@ void mm_switch_context(mm_context_t *ctx);
 
 #define MM_GET_PHYSICAL_ADDRESS_FLAGS_HOLD 1
 #define MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK 2
+#define MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_READ 4
+#define MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_WRITE 8
 void *mm_get_physical_address(void *addr, int flags);
 void mm_apinit();
 void mm_init();

kernel-src/include/kernel/page.h

@@ -3,6 +3,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
 
 #define MEMORY_SECTION_COUNT 3
 #define MEMORY_SECTION_1MB 0
@@ -44,11 +45,14 @@ page_t *mm_get_page(void *addr);
 void *mm_get_page_address(page_t *);
 void mm_hold_page(void *addr);
 void mm_release_page(void *addr);
+void mm_unlock_and_release_page(void *addr);
 void mm_force_free_page(void *address, size_t count);
 void *mm_alloc_pages(size_t size, int section);
 void mm_release_range(void *addr, size_t size);
 void mm_page_init();
 void mm_get_page_statistics(size_t *total_pages, size_t *free_pages);
+void mm_unlock_page(page_t *page);
+bool mm_is_page_locked(page_t *page);
 
 extern uintptr_t hhdm_base;
 

kernel-src/include/kernel/xhci.h

@@ -322,6 +322,7 @@ struct xhci_submission {
 	list_node_t list_node;
 	// Set whenever a submission is queued
 	bool valid;
+	bool unlock_pages;
 	struct xhci_ring *ring;
 
 	// The data associated with this submission

kernel-src/io/block/ahci.c

@@ -308,12 +308,12 @@ static int cmd_identify(ahci_t *ahci, int port, identify_t *results_phys) {
 static void release_prdt(command_table_t *command_table, uint16_t prdtl) {
 	for (int i = 0; i < prdtl; ++i) {
 		void *address = (void *)((uint64_t)command_table->prdt[i].base_low | ((uint64_t)command_table->prdt[i].base_high << 32));
-		mm_release_page(address);
+		mm_unlock_and_release_page(address);
 	}
 }
 
 #define PRDTL_LIMIT ((PAGE_SIZE - sizeof(command_table_t)) / sizeof(prdt_t))
-static int setup_prdt(iovec_iterator_t *iterator, command_table_t *command_table, uint16_t *prdtl, size_t *requested_size) {
+static int setup_prdt(iovec_iterator_t *iterator, command_table_t *command_table, uint16_t *prdtl, size_t *requested_size, bool write) {
 	int err;
 	size_t block_done = 0;
 	size_t prdt_done = 0;
@@ -321,14 +321,14 @@ static int setup_prdt(iovec_iterator_t *iterator, command_table_t *command_table
 	while (block_done < *requested_size && prdt_done < PRDTL_LIMIT) {
 		void *page;
 		size_t page_offset, page_remaining;
-		err = iovec_iterator_next_page(iterator, &page_offset, &page_remaining, &page);
+		err = iovec_iterator_next_page(iterator, &page_offset, &page_remaining, &page, !write);
 		if (err)
 			goto cleanup;
 
 		__assert(page);
 		// not block aligned
 		if (page_remaining % 512) {
-			mm_release_page(page);
+			mm_unlock_and_release_page(page);
 			err = EINVAL;
 			goto cleanup;
 		}
@@ -386,7 +386,7 @@ static int rw(port_data_t *port_data, iovec_iterator_t *iterator, uintmax_t lba,
 	while (done != count) {
 		size_t do_count = count - done;
 		uint16_t prdtl_tmp;
-		error = setup_prdt(iterator, command_table, &prdtl_tmp, &do_count);
+		error = setup_prdt(iterator, command_table, &prdtl_tmp, &do_count, write);
 		if (error)
 			break;
 

kernel-src/io/block/nvme.c

@@ -510,26 +510,29 @@ static int iowrite(nvmenamespace_t *namespace, uint64_t prp[2], uint64_t lba, ui
 #define PAGES_IN_PRP_PAGE (PAGE_SIZE / sizeof(uint64_t))
 
 static void free_prps(uint64_t prp[2], bool has_list) {
-	mm_release_page((void *)ROUND_DOWN(prp[0], PAGE_SIZE));
-	if (has_list) {
-		uint64_t *prp_list = MAKE_HHDM((uint64_t *)prp[1]);
-		for (int i = 0; prp_list[i] && i < PAGES_IN_PRP_PAGE; ++i)
-			mm_release_page((void *)prp_list[i]);
-	}
+	void *prp1 = (void *)prp[1];
 
-	if (prp[1])
-		mm_release_page((void *)prp[1]);
+	mm_unlock_and_release_page((void *)ROUND_DOWN(prp[0], PAGE_SIZE));
+	if (has_list) {
+		uint64_t *prp_list = MAKE_HHDM(prp1);
+		for (int i = 0; i < PAGES_IN_PRP_PAGE && prp_list[i]; ++i)
+			mm_unlock_and_release_page((void *)prp_list[i]);
+
+		mm_release_page(prp1);
+	} else if (prp1) {
+		mm_unlock_and_release_page(prp1);
+	}
 }
 
 // sets up prps based on an iovec iterator.
 // takes in the wanted block count and outputs an io_count for the i/o operation (for example, if it did not fit entirely on the prp)
-static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterator, uint64_t prp[2], size_t count, size_t *io_count, bool *has_list) {
+static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterator, uint64_t prp[2], size_t count, size_t *io_count, bool *has_list, bool write) {
 	count = min(count, namespace->max_transfer_size_blocks);
 
 	// first page
 	void *page;
 	size_t page_offset, page_remaining;
-	int error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
+	int error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
 	if (error)
 		return error;
 	__assert(page);
@@ -553,7 +556,7 @@ static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterat
 	}
 
 	// second page
-	error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
+	error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
 	if (error) {
 		free_prps(prp, false);
 		return error;
@@ -591,7 +594,7 @@ static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterat
 
 	// loop over the pages
 	for (int i = 1; i < PAGES_IN_PRP_PAGE; ++i) {
-		error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
+		error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
 		if (error) {
 			prp_list[i] = 0;
 			free_prps(prp, true);
@@ -603,7 +606,7 @@ static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterat
 		if (page_offset || page_remaining % namespace->blocksize) {
 			prp_list[i] = 0;
 			free_prps(prp, true);
-			mm_release_page(page);
+			mm_unlock_and_release_page(page);
 			return EINVAL;
 		}
 
@@ -641,7 +644,7 @@ static int rwblocks(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterator
 	for (;;) {
 		size_t io_count;
 		bool has_list;
-		error = setup_prps(namespace, iovec_iterator, prp, count, &io_count, &has_list);
+		error = setup_prps(namespace, iovec_iterator, prp, count, &io_count, &has_list, write);
 		if (error)
 			return error;
 

kernel-src/io/block/vioblk.c

@@ -116,7 +116,7 @@ static int vioblk_rw(vioblkdev_t *blkdev, iovec_iterator_t *iovec_iterator, uint
 	while (done < count) {
 		void *page;
 		size_t page_offset, page_remaining;
-		err = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
+		err = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
 		if (err)
 			break;
 
@@ -133,7 +133,7 @@ static int vioblk_rw(vioblkdev_t *blkdev, iovec_iterator_t *iovec_iterator, uint
 
 		vioblk_enqueue(blkdev, header, (void *)((uintptr_t)page + page_offset), PAGE_SIZE, status, write);
 
-		mm_release_page(page);
+		mm_unlock_and_release_page(page);
 
 		// if we didnt use the whole space in the page, set the iterator back a bit
 		size_t diff_between_available_and_used = page_remaining - docount * 512;

kernel-src/io/iovec.c

@@ -235,7 +235,7 @@ size_t iovec_iterator_read_from_ringbuffer(iovec_iterator_t *iovec_iterator, rin
 	return done;
 }
 
-int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offset, size_t *page_remaining, void **page) {
+int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offset, size_t *page_remaining, void **page, bool write) {
 	iovec_iterator_skip(iovec_iterator, 0); // this will skip any zero length entries
 	void *addr = (void *)((uintptr_t)iovec_iterator->current->addr + iovec_iterator->current_offset);
 	size_t offset_in_page = ((uintptr_t)addr % PAGE_SIZE);
@@ -246,7 +246,11 @@ int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offs
 		return 0;
 	}
 
-	void *phys = mm_get_physical_address((void *)((uintptr_t)addr - offset_in_page), MM_GET_PHYSICAL_ADDRESS_FLAGS_HOLD | MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK);
+	void *phys = mm_get_physical_address((void *)((uintptr_t)addr - offset_in_page),
+			MM_GET_PHYSICAL_ADDRESS_FLAGS_HOLD |
+			MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK |
+			(write ? MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_WRITE :
+				 MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_READ));
 	if (phys == NULL)
 		return EFAULT;
 

kernel-src/io/usb/ctrl/xhci.c

@@ -27,8 +27,13 @@ static void xhci_signal_callback_thread(xhci_ctrl_t *xhci) {
 static void xhci_release_submission_page(xhci_submission_t *submission) {
 	uint32_t cmd_type = xhci_trb_type(&submission->cmd_trb);
 
-	if ((cmd_type == TRB_NORMAL || cmd_type == TRB_DATA_STAGE) && submission->cmd_trb.parameters != 0)
-		mm_release_page((void *)(submission->cmd_trb.parameters & ~(PAGE_SIZE - 1)));
+	if ((cmd_type == TRB_NORMAL || cmd_type == TRB_DATA_STAGE) && submission->cmd_trb.parameters != 0) {
+		void *page = (void *)(submission->cmd_trb.parameters & ~(PAGE_SIZE - 1));
+		if (submission->unlock_pages)
+			mm_unlock_and_release_page(page);
+		else
+			mm_release_page(page);
+	}
 }
 
 static bool xhci_reclaim_submission_slot(xhci_submission_t *slot, xhci_submission_t *submission) {

kernel-src/io/usb/ctrl/xhci_ring.c

@@ -83,6 +83,8 @@ xhci_submission_t *xhci_ring_submit_locked(xhci_ring_t *r, xhci_trb_t *trb, xhci
 		bool old_valid = __atomic_exchange_n(&r_sub->valid, true, __ATOMIC_SEQ_CST);
 		__assert(!old_valid);
 
+		r_sub->unlock_pages = sub->unlock_pages;
+
 		r_sub->ring = r;
 		r_sub->device = sub->device;
 		r_sub->callback = sub->callback;

kernel-src/io/usb/ctrl/xhci_xfer.c

@@ -12,7 +12,7 @@ typedef struct {
 	size_t data_offset;
 } sg_trb_t;
 
-static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *trb_count) {
+static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *trb_count, bool write) {
 	iovec_iterator_t iter = *iov;
 	size_t done = 0;
 	size_t count = 0;
@@ -21,13 +21,13 @@ static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *
 		size_t page_offset, page_remaining;
 		void *page;
 
-		int err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page);
+		int err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page, write);
 		if (err)
 			return err;
 		if (page == NULL)
 			return EFAULT;
 
-		mm_release_page(page);
+		mm_unlock_and_release_page(page);
 
 		done += min(page_remaining, total_size - done);
 		count++;
@@ -41,6 +41,8 @@ int xhci_data_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, xhci
 	__assert(xfer->ep != NULL);
 	__assert(xfer->setup == NULL);
 
+	sub->unlock_pages = false;
+
 	uint32_t ep_num = xfer->ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_NUM_MASK;
 	uint32_t ep_index = (ep_num << 1) | (xfer->ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_DIR_IN ? 1 : 0);
 
@@ -99,6 +101,7 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
 	uint16_t max_packet_size = usb_endpoint_max_packet_size(xfer->ep->desc);
 	uint32_t td_count = ROUND_UP(xfer->iov->total_size, max_packet_size) / max_packet_size;
 	uint32_t done = 0;
+	sub->unlock_pages = true;
 
 	if (xfer->iov->total_size == 0) {
 		if (sub->callback)
@@ -107,7 +110,7 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
 	}
 
 	size_t trb_count;
-	int err = xhci_count_sg_trbs(xfer->iov, xfer->iov->total_size, &trb_count);
+	int err = xhci_count_sg_trbs(xfer->iov, xfer->iov->total_size, &trb_count, xfer->flags & USB_XFER_FLAG_TO_HOST);
 	if (err)
 		return err;
 
@@ -127,9 +130,10 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
 		size_t page_offset, page_remaining;
 		void *page;
 
-		err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page);
+		err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page, xfer->flags & USB_XFER_FLAG_TO_HOST);
 		if (err)
 			goto fail;
+
 		if (page == NULL) {
 			err = EFAULT;
 			goto fail;
@@ -187,7 +191,7 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
 
 	fail:
 	for (size_t i = 0; i < prepared_count; ++i)
-		mm_release_page(prepared[i].page);
+		mm_unlock_and_release_page(prepared[i].page);
 	free(prepared);
 	xhci_ring_unreserve(ring, trb_count);
 	return err;
@@ -197,6 +201,8 @@ int xhci_control_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, x
 	__assert(xfer->ep == NULL);
 	__assert(xfer->setup != NULL);
 
+	sub->unlock_pages = false;
+
 	// Why would anyone want to do scatter-gather control transfers???
 	__assert(!(xfer->flags & USB_XFER_FLAG_IOVEC));
 

kernel-src/mm/mm.c

@@ -161,7 +161,7 @@ int mm_full_page_in(mm_range_t *range, void *vaddr, page_t **resulting_page) {
 			return ENOMEM;
 		}
 
-		if (cacheable) {
+		if (cacheable && (range->mmuflags & ARCH_MMU_FLAGS_WRITE)) {
 			VOP_LOCK(range->vnode);
 			mm_cache_make_dirty(vn_page);
 			VOP_UNLOCK(range->vnode);
@@ -318,11 +318,20 @@ bool mm_handle_page_fault(void *addr, bool user, int actions) {
 	return status;
 }
 
-// TODO read-only permission checking for DMA
 // space is expected to be locked on call.
-static int lock_page(mm_space_t *space, void *vaddr) {
+static int lock_page(mm_space_t *space, void *vaddr, int hint) {
+	// TODO: with a read hint, we could theoretically only partially page in.
 	mm_range_t *range = mm_get_range(space, vaddr);
-	if (!range)
+	if (!range || 
+	((hint & MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_WRITE) && !(range->mmuflags & ARCH_MMU_FLAGS_WRITE)) ||
+	((hint & MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_READ) && !(range->mmuflags & ARCH_MMU_FLAGS_READ)))
+		return EFAULT;
+
+	// since we cannot guarantee the mapping of a character device actually points to physical memory,
+	// we cannot reliably lock it. returning an error here also protects from trying to DMA into non-physical memory
+	// like the framebuffer.
+	// same thing applies to an abstract physical memory mapping.
+	if (((range->flags & MM_RANGE_FLAGS_FILE) && (range->vnode->type == V_TYPE_CHDEV)) || (range->flags & MM_RANGE_FLAGS_PHYSICAL))
 		return EFAULT;
 
 	// we need to make sure the address is fully paged in before locking, as it needs
@@ -350,7 +359,7 @@ void *mm_get_physical_address(void *addr, int flags) {
 	MUTEX_ACQUIRE(&space->lock);
 
 	void *physical;
-	if ((flags & MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK) && lock_page(space, aligned_addr)) {
+	if ((flags & MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK) && lock_page(space, aligned_addr, flags)) {
 		physical = NULL;
 		goto leave;
 	}
@@ -567,8 +576,6 @@ mm_context_t *mm_fork_context(mm_context_t *old_context) {
 		if (range->flags & MM_RANGE_FLAGS_FILE)
 			VOP_HOLD(range->vnode);
 
-		// copy any pages that are mapped
-
 		for (uintptr_t offset = 0; offset < new_range->size; offset += PAGE_SIZE) {
 			// XXX some types of mappings, like framebuffer shared mappings, will break if done this way
 			void *vaddr = (void *)((uintptr_t)new_range->start + offset);
@@ -576,12 +583,30 @@ mm_context_t *mm_fork_context(mm_context_t *old_context) {
 			if (phys == NULL)
 				continue;
 
-			if (arch_mmu_map(new_context->pagetable, phys, vaddr, new_range->mmuflags & ~ARCH_MMU_FLAGS_WRITE) == false)
-				goto error;
+			bool writeable = arch_mmu_iswritable(old_context->pagetable, vaddr);
+			bool private = !(range->flags & MM_RANGE_FLAGS_SHARED);
+			page_t *page = mm_get_page(phys);
+			if (mm_is_page_locked(page) && writeable && private) {
+				// the page is a private locked page, eagerly copy it to ensure fork/futex correctness
+				void *new_phys = mm_alloc_page(MEMORY_SECTION_DEFAULT);
+				if (new_phys == NULL)
+					goto error;
 
-			mm_hold_page(phys);
+				if (arch_mmu_map(new_context->pagetable, new_phys, vaddr, new_range->mmuflags) == false) {
+					mm_release_page(new_phys);
+					goto error;
+				}
 
-			arch_mmu_remap(old_context->pagetable, phys, vaddr, new_range->mmuflags & ~ARCH_MMU_FLAGS_WRITE);
+				memcpy(MAKE_HHDM(new_phys), MAKE_HHDM(phys), PAGE_SIZE);
+			} else {
+				// go through the normal CoW path
+				if (arch_mmu_map(new_context->pagetable, phys, vaddr, new_range->mmuflags & ~ARCH_MMU_FLAGS_WRITE) == false)
+					goto error;
+
+				mm_hold_page(phys);
+
+				arch_mmu_remap(old_context->pagetable, phys, vaddr, new_range->mmuflags & ~ARCH_MMU_FLAGS_WRITE);
+			}
 		}
 
 		rbtree = rbtree_successor(rbtree);

kernel-src/mm/page.c

@@ -146,6 +146,7 @@ void mm_release_page(void *addr) {
 	MUTEX_ACQUIRE(&free_list_mutex);
 	__assert(page->refcount != 0);
 	if (--page->refcount == 0) {
+		__assert(!mm_is_page_locked(page));
 		__assert((page->flags & PAGE_FLAGS_DIRTY) == 0);
 		insert_in_free_list(page);
 		if (page->backing == NULL)
@@ -154,6 +155,20 @@ void mm_release_page(void *addr) {
 	MUTEX_RELEASE(&free_list_mutex);
 }
 
+void mm_unlock_and_release_page(void *addr) {
+	page_t *page = &pages[(uintptr_t)addr / PAGE_SIZE];
+	mm_unlock_page(page);
+	mm_release_page(addr);
+}
+
+bool mm_is_page_locked(page_t *page) {
+	return __atomic_load_n(&page->lock_count, __ATOMIC_SEQ_CST) > 0;
+}
+
+void mm_unlock_page(page_t *page) {
+	__assert(__atomic_fetch_sub(&page->lock_count, 1, __ATOMIC_SEQ_CST) > 0);
+}
+
 static void do_alloc(page_t *page) {
 	__assert(page->refcount == 0);
 	memset(page, 0, sizeof(page_t));

kernel-src/sys/syscalls/futex.c

@@ -62,11 +62,17 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value
 
 	bool doleave = true;
 	uint32_t word;
+	uint32_t *physical = NULL;
 	ret.errno = usercopy_fromuseratomic32(futexp, &word);
 	if (unlikely(ret.errno))
 		goto cleanup;
 
-	uint32_t *physical = mm_get_physical_address(futexp, 0);
+	physical = mm_get_physical_address(futexp, MM_GET_PHYSICAL_ADDRESS_FLAGS_HOLD | MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK);
+	if (unlikely(physical == NULL)) {
+		ret.errno = EFAULT;
+		goto cleanup;
+	}
+
 	futex_t *futex = getfutex(physical);
 
 	switch (op) {
@@ -162,6 +168,9 @@ syscallret_t syscall_futex(context_t *, uint32_t *futexp, int op, uint32_t value
 
 	poll_destroydesc(&desc);
 
+	if (physical)
+		mm_unlock_and_release_page(physical);
+
 	MUTEX_RELEASE(&futexmutex);
 	return ret;
 }