mirrors/Astral
1
0
Fork 0
mirror of https://github.com/Mathewnd/Astral.git synced 2026-10-31 22:48:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

mm: properly implement page locking

Browse Source
This commit is contained in:
Matheus
2026-07-08 01:37:12 -03:00
parent af30088c0d
commit 1512b2f3c6
14 changed files with 123 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel-src/io/block/ahci.c
+5 -5
View File
@@ -308,12 +308,12 @@ static int cmd_identify(ahci_t *ahci, int port, identify_t *results_phys) {
static void release_prdt(command_table_t *command_table, uint16_t prdtl) {
for (int i = 0; i < prdtl; ++i) {
void *address = (void *)((uint64_t)command_table->prdt[i].base_low | ((uint64_t)command_table->prdt[i].base_high << 32));
mm_release_page(address);
mm_unlock_and_release_page(address);
}
}
#define PRDTL_LIMIT ((PAGE_SIZE - sizeof(command_table_t)) / sizeof(prdt_t))
static int setup_prdt(iovec_iterator_t *iterator, command_table_t *command_table, uint16_t *prdtl, size_t *requested_size) {
static int setup_prdt(iovec_iterator_t *iterator, command_table_t *command_table, uint16_t *prdtl, size_t *requested_size, bool write) {
int err;
size_t block_done = 0;
size_t prdt_done = 0;
@@ -321,14 +321,14 @@ static int setup_prdt(iovec_iterator_t *iterator, command_table_t *command_table
while (block_done < *requested_size && prdt_done < PRDTL_LIMIT) {
void *page;
size_t page_offset, page_remaining;
err = iovec_iterator_next_page(iterator, &page_offset, &page_remaining, &page);
err = iovec_iterator_next_page(iterator, &page_offset, &page_remaining, &page, !write);
if (err)
goto cleanup;
__assert(page);
// not block aligned
if (page_remaining % 512) {
mm_release_page(page);
mm_unlock_and_release_page(page);
err = EINVAL;
goto cleanup;
}
@@ -386,7 +386,7 @@ static int rw(port_data_t *port_data, iovec_iterator_t *iterator, uintmax_t lba,
while (done != count) {
size_t do_count = count - done;
uint16_t prdtl_tmp;
error = setup_prdt(iterator, command_table, &prdtl_tmp, &do_count);
error = setup_prdt(iterator, command_table, &prdtl_tmp, &do_count, write);
if (error)
break;
kernel-src/io/block/nvme.c
+17 -14
View File
@@ -510,26 +510,29 @@ static int iowrite(nvmenamespace_t *namespace, uint64_t prp[2], uint64_t lba, ui
#define PAGES_IN_PRP_PAGE (PAGE_SIZE / sizeof(uint64_t))
static void free_prps(uint64_t prp[2], bool has_list) {
mm_release_page((void *)ROUND_DOWN(prp[0], PAGE_SIZE));
if (has_list) {
uint64_t *prp_list = MAKE_HHDM((uint64_t *)prp[1]);
for (int i = 0; prp_list[i] && i < PAGES_IN_PRP_PAGE; ++i)
mm_release_page((void *)prp_list[i]);
}
void *prp1 = (void *)prp[1];
if (prp[1])
mm_release_page((void *)prp[1]);
mm_unlock_and_release_page((void *)ROUND_DOWN(prp[0], PAGE_SIZE));
if (has_list) {
uint64_t *prp_list = MAKE_HHDM(prp1);
for (int i = 0; i < PAGES_IN_PRP_PAGE && prp_list[i]; ++i)
mm_unlock_and_release_page((void *)prp_list[i]);
mm_release_page(prp1);
} else if (prp1) {
mm_unlock_and_release_page(prp1);
}
}
// sets up prps based on an iovec iterator.
// takes in the wanted block count and outputs an io_count for the i/o operation (for example, if it did not fit entirely on the prp)
static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterator, uint64_t prp[2], size_t count, size_t *io_count, bool *has_list) {
static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterator, uint64_t prp[2], size_t count, size_t *io_count, bool *has_list, bool write) {
count = min(count, namespace->max_transfer_size_blocks);
// first page
void *page;
size_t page_offset, page_remaining;
int error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
int error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
if (error)
return error;
__assert(page);
@@ -553,7 +556,7 @@ static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterat
}
// second page
error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
if (error) {
free_prps(prp, false);
return error;
@@ -591,7 +594,7 @@ static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterat
// loop over the pages
for (int i = 1; i < PAGES_IN_PRP_PAGE; ++i) {
error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
error = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
if (error) {
prp_list[i] = 0;
free_prps(prp, true);
@@ -603,7 +606,7 @@ static int setup_prps(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterat
if (page_offset || page_remaining % namespace->blocksize) {
prp_list[i] = 0;
free_prps(prp, true);
mm_release_page(page);
mm_unlock_and_release_page(page);
return EINVAL;
}
@@ -641,7 +644,7 @@ static int rwblocks(nvmenamespace_t *namespace, iovec_iterator_t *iovec_iterator
for (;;) {
size_t io_count;
bool has_list;
error = setup_prps(namespace, iovec_iterator, prp, count, &io_count, &has_list);
error = setup_prps(namespace, iovec_iterator, prp, count, &io_count, &has_list, write);
if (error)
return error;
kernel-src/io/block/vioblk.c
+2 -2
View File
@@ -116,7 +116,7 @@ static int vioblk_rw(vioblkdev_t *blkdev, iovec_iterator_t *iovec_iterator, uint
while (done < count) {
void *page;
size_t page_offset, page_remaining;
err = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page);
err = iovec_iterator_next_page(iovec_iterator, &page_offset, &page_remaining, &page, !write);
if (err)
break;
@@ -133,7 +133,7 @@ static int vioblk_rw(vioblkdev_t *blkdev, iovec_iterator_t *iovec_iterator, uint
vioblk_enqueue(blkdev, header, (void *)((uintptr_t)page + page_offset), PAGE_SIZE, status, write);
mm_release_page(page);
mm_unlock_and_release_page(page);
// if we didnt use the whole space in the page, set the iterator back a bit
size_t diff_between_available_and_used = page_remaining - docount * 512;
kernel-src/io/iovec.c
+6 -2
View File
@@ -235,7 +235,7 @@ size_t iovec_iterator_read_from_ringbuffer(iovec_iterator_t *iovec_iterator, rin
return done;
}
int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offset, size_t *page_remaining, void **page) {
int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offset, size_t *page_remaining, void **page, bool write) {
iovec_iterator_skip(iovec_iterator, 0); // this will skip any zero length entries
void *addr = (void *)((uintptr_t)iovec_iterator->current->addr + iovec_iterator->current_offset);
size_t offset_in_page = ((uintptr_t)addr % PAGE_SIZE);
@@ -246,7 +246,11 @@ int iovec_iterator_next_page(iovec_iterator_t *iovec_iterator, size_t *page_offs
return 0;
}
void *phys = mm_get_physical_address((void *)((uintptr_t)addr - offset_in_page), MM_GET_PHYSICAL_ADDRESS_FLAGS_HOLD | MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK);
void *phys = mm_get_physical_address((void *)((uintptr_t)addr - offset_in_page),
MM_GET_PHYSICAL_ADDRESS_FLAGS_HOLD |
MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK |
(write ? MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_WRITE :
MM_GET_PHYSICAL_ADDRESS_FLAGS_LOCK_HINT_READ));
if (phys == NULL)
return EFAULT;
kernel-src/io/usb/ctrl/xhci.c
+7 -2
View File
@@ -27,8 +27,13 @@ static void xhci_signal_callback_thread(xhci_ctrl_t *xhci) {
static void xhci_release_submission_page(xhci_submission_t *submission) {
uint32_t cmd_type = xhci_trb_type(&submission->cmd_trb);
if ((cmd_type == TRB_NORMAL || cmd_type == TRB_DATA_STAGE) && submission->cmd_trb.parameters != 0)
mm_release_page((void *)(submission->cmd_trb.parameters & ~(PAGE_SIZE - 1)));
if ((cmd_type == TRB_NORMAL || cmd_type == TRB_DATA_STAGE) && submission->cmd_trb.parameters != 0) {
void *page = (void *)(submission->cmd_trb.parameters & ~(PAGE_SIZE - 1));
if (submission->unlock_pages)
mm_unlock_and_release_page(page);
else
mm_release_page(page);
}
}
static bool xhci_reclaim_submission_slot(xhci_submission_t *slot, xhci_submission_t *submission) {
kernel-src/io/usb/ctrl/xhci_ring.c
+2
View File
@@ -83,6 +83,8 @@ xhci_submission_t *xhci_ring_submit_locked(xhci_ring_t *r, xhci_trb_t *trb, xhci
bool old_valid = __atomic_exchange_n(&r_sub->valid, true, __ATOMIC_SEQ_CST);
__assert(!old_valid);
r_sub->unlock_pages = sub->unlock_pages;
r_sub->ring = r;
r_sub->device = sub->device;
r_sub->callback = sub->callback;
kernel-src/io/usb/ctrl/xhci_xfer.c
+12 -6
View File
@@ -12,7 +12,7 @@ typedef struct {
size_t data_offset;
} sg_trb_t;
static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *trb_count) {
static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *trb_count, bool write) {
iovec_iterator_t iter = *iov;
size_t done = 0;
size_t count = 0;
@@ -21,13 +21,13 @@ static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *
size_t page_offset, page_remaining;
void *page;
int err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page);
int err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page, write);
if (err)
return err;
if (page == NULL)
return EFAULT;
mm_release_page(page);
mm_unlock_and_release_page(page);
done += min(page_remaining, total_size - done);
count++;
@@ -41,6 +41,8 @@ int xhci_data_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, xhci
__assert(xfer->ep != NULL);
__assert(xfer->setup == NULL);
sub->unlock_pages = false;
uint32_t ep_num = xfer->ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_NUM_MASK;
uint32_t ep_index = (ep_num << 1) | (xfer->ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_DIR_IN ? 1 : 0);
@@ -99,6 +101,7 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
uint16_t max_packet_size = usb_endpoint_max_packet_size(xfer->ep->desc);
uint32_t td_count = ROUND_UP(xfer->iov->total_size, max_packet_size) / max_packet_size;
uint32_t done = 0;
sub->unlock_pages = true;
if (xfer->iov->total_size == 0) {
if (sub->callback)
@@ -107,7 +110,7 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
}
size_t trb_count;
int err = xhci_count_sg_trbs(xfer->iov, xfer->iov->total_size, &trb_count);
int err = xhci_count_sg_trbs(xfer->iov, xfer->iov->total_size, &trb_count, xfer->flags & USB_XFER_FLAG_TO_HOST);
if (err)
return err;
@@ -127,9 +130,10 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
size_t page_offset, page_remaining;
void *page;
err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page);
err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page, xfer->flags & USB_XFER_FLAG_TO_HOST);
if (err)
goto fail;
if (page == NULL) {
err = EFAULT;
goto fail;
@@ -187,7 +191,7 @@ int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev,
fail:
for (size_t i = 0; i < prepared_count; ++i)
mm_release_page(prepared[i].page);
mm_unlock_and_release_page(prepared[i].page);
free(prepared);
xhci_ring_unreserve(ring, trb_count);
return err;
@@ -197,6 +201,8 @@ int xhci_control_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, x
__assert(xfer->ep == NULL);
__assert(xfer->setup != NULL);
sub->unlock_pages = false;
// Why would anyone want to do scatter-gather control transfers???
__assert(!(xfer->flags & USB_XFER_FLAG_IOVEC));