diff --git a/kernel-src/arch/x86-64/mmu.c b/kernel-src/arch/x86-64/mmu.c index 82da1c50..008ebad4 100644 --- a/kernel-src/arch/x86-64/mmu.c +++ b/kernel-src/arch/x86-64/mmu.c @@ -68,7 +68,7 @@ static bool add_page(pagetableptr_t top, void *vaddr, uint64_t entry, int depth) uint64_t *pdpt = next(pml4[pml4offset]); if (pdpt == NULL) { - pdpt = pmm_alloc(1, PMM_SECTION_DEFAULT); + pdpt = pmm_allocpage(PMM_SECTION_DEFAULT); if (pdpt == NULL) return false; pml4[pml4offset] = (uint64_t)pdpt | INTERMEDIATE_FLAGS; @@ -83,7 +83,7 @@ static bool add_page(pagetableptr_t top, void *vaddr, uint64_t entry, int depth) uint64_t *pd = next(pdpt[pdptoffset]); if (pd == NULL) { - pd = pmm_alloc(1, PMM_SECTION_DEFAULT); + pd = pmm_allocpage(PMM_SECTION_DEFAULT); if (pd == NULL) return false; pdpt[pdptoffset] = (uint64_t)pd | INTERMEDIATE_FLAGS; @@ -98,7 +98,7 @@ static bool add_page(pagetableptr_t top, void *vaddr, uint64_t entry, int depth) uint64_t *pt = next(pd[pdoffset]); if (pt == NULL) { - pt = pmm_alloc(1, PMM_SECTION_DEFAULT); + pt = pmm_allocpage(PMM_SECTION_DEFAULT); if (pt == NULL) return false; pd[pdoffset] = (uint64_t)pt | INTERMEDIATE_FLAGS; @@ -120,13 +120,13 @@ static void destroy(uint64_t *table, int depth) { if (depth > 0) destroy(MAKE_HHDM(addr), depth - 1); - pmm_free(addr, 1); + pmm_release(addr); } } void arch_mmu_destroytable(pagetableptr_t table) { destroy(MAKE_HHDM(table), 3); - pmm_free(table, 1); + pmm_release(table); } bool arch_mmu_map(pagetableptr_t table, void *paddr, void *vaddr, mmuflags_t flags) { @@ -170,7 +170,7 @@ void arch_mmu_switch(pagetableptr_t table) { static pagetableptr_t template; pagetableptr_t arch_mmu_newtable() { - pagetableptr_t table = pmm_alloc(1, PMM_SECTION_DEFAULT); + pagetableptr_t table = pmm_allocpage(PMM_SECTION_DEFAULT); if (table == NULL) return NULL; memcpy(MAKE_HHDM(table), template, PAGE_SIZE); @@ -227,13 +227,13 @@ static void pfisr(isr_t *self, context_t *ctx) { } void arch_mmu_init() { - template = pmm_alloc(1, PMM_SECTION_DEFAULT); + template = pmm_allocpage(PMM_SECTION_DEFAULT); __assert(template); template = MAKE_HHDM(template); memset(template, 0, PAGE_SIZE); for (int i = 256; i < 512; ++i) { - uint64_t *entry = pmm_alloc(1, PMM_SECTION_DEFAULT); + uint64_t *entry = pmm_allocpage(PMM_SECTION_DEFAULT); __assert(entry); memset(MAKE_HHDM(entry), 0, PAGE_SIZE); template[i] = (uint64_t)entry | INTERMEDIATE_FLAGS; diff --git a/kernel-src/arch/x86-64/syscalllog.c b/kernel-src/arch/x86-64/syscalllog.c index f5a94671..54925581 100644 --- a/kernel-src/arch/x86-64/syscalllog.c +++ b/kernel-src/arch/x86-64/syscalllog.c @@ -86,7 +86,7 @@ static char *args[] = { "fds %p nfds %d timeout %d", // poll "time %p remaining %p", // nanosleep "fd %d size %lu", // ftruncate - "backing %s mountpoint %s fs %s flags %d data %p" // mount + "backing %p mountpoint %s fs %s flags %d data %p" // mount }; #endif diff --git a/kernel-src/fs/ext2.c b/kernel-src/fs/ext2.c index a4d1792b..4a749690 100644 --- a/kernel-src/fs/ext2.c +++ b/kernel-src/fs/ext2.c @@ -1435,12 +1435,12 @@ static int ext2_symlink(vnode_t *vnode, char *name, vattr_t *attr, char *path, c #define MMAPTMPFLAGS (ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_WRITE) static int ext2_mmap(vnode_t *node, void *addr, uintmax_t offset, int flags, cred_t *cred) { - void *paddr = pmm_alloc(1, PMM_SECTION_DEFAULT); + void *paddr = pmm_allocpage(PMM_SECTION_DEFAULT); if (paddr == NULL) return ENOMEM; if (arch_mmu_map(_cpu()->vmmctx->pagetable, paddr, addr, MMAPTMPFLAGS) == false) { - pmm_free(paddr, 1); + pmm_release(paddr); return ENOMEM; } @@ -1448,7 +1448,7 @@ static int ext2_mmap(vnode_t *node, void *addr, uintmax_t offset, int flags, cre int e = ext2_read(node, addr, PAGE_SIZE, offset, flags, &readc, cred); if (e) { arch_mmu_unmap(_cpu()->vmmctx->pagetable, addr); - pmm_free(paddr, 1); + pmm_release(paddr); return e; } @@ -1469,7 +1469,7 @@ static int ext2_munmap(vnode_t *node, void *addr, uintmax_t offset, int flags, c void *paddr = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr); __assert(paddr); arch_mmu_unmap(_cpu()->vmmctx->pagetable, addr); - pmm_free(paddr, 1); + pmm_release(paddr); return 0; } diff --git a/kernel-src/fs/initrd.c b/kernel-src/fs/initrd.c index ac746650..dc5a8b4a 100644 --- a/kernel-src/fs/initrd.c +++ b/kernel-src/fs/initrd.c @@ -126,7 +126,7 @@ void initrd_unpack() { for (;;) { if (cleanupbytespassed >= PAGE_SIZE) { size_t pagec = cleanupbytespassed / PAGE_SIZE; - pmm_free(FROM_HHDM(cleanupptr), pagec); + pmm_makefree(FROM_HHDM(cleanupptr), pagec); cleanupptr = (void *)((uintptr_t)cleanupptr + ROUND_DOWN(cleanupbytespassed, PAGE_SIZE)); cleanupbytespassed %= PAGE_SIZE; } @@ -174,5 +174,5 @@ void initrd_unpack() { } // free remaining pages to be freed - pmm_free(FROM_HHDM(cleanupptr), ROUND_UP(cleanupbytespassed, PAGE_SIZE) / PAGE_SIZE); + pmm_makefree(FROM_HHDM(cleanupptr), ROUND_UP(cleanupbytespassed, PAGE_SIZE) / PAGE_SIZE); } diff --git a/kernel-src/fs/tmpfs.c b/kernel-src/fs/tmpfs.c index 21695c35..6705c9af 100644 --- a/kernel-src/fs/tmpfs.c +++ b/kernel-src/fs/tmpfs.c @@ -392,12 +392,12 @@ static int tmpfs_readlink(vnode_t *node, char **link, cred_t *cred) { #define MMAPTMPFLAGS (ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_WRITE) static int tmpfs_mmap(vnode_t *node, void *addr, uintmax_t offset, int flags, cred_t *cred) { - void *paddr = pmm_alloc(1, PMM_SECTION_DEFAULT); + void *paddr = pmm_allocpage(PMM_SECTION_DEFAULT); if (paddr == NULL) return ENOMEM; if (arch_mmu_map(_cpu()->vmmctx->pagetable, paddr, addr, MMAPTMPFLAGS) == false) { - pmm_free(paddr, 1); + pmm_release(paddr); return ENOMEM; } @@ -405,7 +405,7 @@ static int tmpfs_mmap(vnode_t *node, void *addr, uintmax_t offset, int flags, cr int e = tmpfs_read(node, addr, PAGE_SIZE, offset, flags, &readc, cred); if (e) { arch_mmu_unmap(_cpu()->vmmctx->pagetable, addr); - pmm_free(paddr, 1); + pmm_release(paddr); return e; } @@ -426,7 +426,7 @@ static int tmpfs_munmap(vnode_t *node, void *addr, uintmax_t offset, int flags, void *paddr = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr); __assert(paddr); arch_mmu_unmap(_cpu()->vmmctx->pagetable, addr); - pmm_free(paddr, 1); + pmm_release(paddr); return 0; } diff --git a/kernel-src/include/kernel/pmm.h b/kernel-src/include/kernel/pmm.h index c5cb1561..24df1237 100644 --- a/kernel-src/include/kernel/pmm.h +++ b/kernel-src/include/kernel/pmm.h @@ -9,9 +9,37 @@ #define PMM_SECTION_4GB 1 #define PMM_SECTION_DEFAULT 2 -void *pmm_alloc(size_t count, int flags); -void pmm_free(void *addr, size_t count); -void pmm_init(); +#define PAGE_FLAGS_FREE 1 + +typedef struct page_t { + struct vnode_t *backing; + uintmax_t offset; + struct page_t *hashnext; + struct page_t *hashprev; + struct page_t *vnodenext; + struct page_t *vnodeprev; + union { + struct { + struct page_t *freenext; + struct page_t *freeprev; + }; + struct { + struct page_t *writenext; + struct page_t *writeprev; + }; + }; + uintmax_t refcount; + int flags; +} page_t; + +void *pmm_allocpage(int section); +page_t *pmm_getpage(void *addr); +void pmm_hold(void *addr); +void pmm_release(void *addr); +void pmm_makefree(void *address, size_t count); +void *pmm_alloc(size_t size, int section); +void pmm_free(void *addr, size_t size); +void pmm_init(); extern uintptr_t hhdmbase; diff --git a/kernel-src/io/block/nvme.c b/kernel-src/io/block/nvme.c index f299b69c..22733cd6 100644 --- a/kernel-src/io/block/nvme.c +++ b/kernel-src/io/block/nvme.c @@ -233,7 +233,7 @@ typedef struct { int phase; } queuedesc_t; -#define QUEUEPAIR_ENTRY_COUNT 256 +#define QUEUEPAIR_ENTRY_COUNT (PAGE_SIZE / sizeof(subentry_t)) typedef struct { queuedesc_t submission; @@ -350,7 +350,7 @@ static int identify(nvmecontroller_t *controller, void *buffer, int what, int na entrypair_t pair = {0}; PAIR_INIT(&pair, SUB_DW0_OPCODE_IDENTIFY, SUB_DW0_UNFUSED, SUB_DW0_PRP, 0); - pair.sub.datapointer[0] = (uint64_t)pmm_alloc(1, PMM_SECTION_DEFAULT); + pair.sub.datapointer[0] = (uint64_t)pmm_allocpage(PMM_SECTION_DEFAULT); if (pair.sub.datapointer[0] == 0) return ENOMEM; @@ -519,30 +519,33 @@ static int iowrite(nvmenamespace_t *namespace, uint64_t prp[2], uint64_t lba, ui static int rwblocks(nvmenamespace_t *namespace, void *buffer, uintmax_t lba, size_t count, bool write, bool pagealignedbuffer) { // TODO this is pretty inefficient and will be replaced later - __assert(namespace->blocksize < PAGE_SIZE * 2); - void *phys = pmm_alloc(2, PMM_SECTION_DEFAULT); - if (phys == NULL) - return ENOMEM; + __assert(namespace->blocksize <= PAGE_SIZE); - uint64_t prp[2] = {(uint64_t)phys, (uint64_t)phys + PAGE_SIZE}; + uint64_t prp[2] = {(uint64_t)pmm_allocpage(PMM_SECTION_DEFAULT), 0}; int err = 0; + if (prp[0] == 0) + goto cleanup; for (int i = 0; i < count; ++i) { void *bufferp = (void *)((uintptr_t)buffer + i * namespace->blocksize); - if (write) - memcpy(MAKE_HHDM(phys), bufferp, namespace->blocksize); + if (write) { + memcpy(MAKE_HHDM(prp[0]), bufferp, namespace->blocksize); + } err = write ? iowrite(namespace, prp, lba + i, 1) : ioread(namespace, prp, lba + i, 1); if (err) goto cleanup; - if (!write) - memcpy(bufferp, MAKE_HHDM(phys), namespace->blocksize); + if (!write) { + memcpy(bufferp, MAKE_HHDM(prp[0]), namespace->blocksize); + } } cleanup: - pmm_free(phys, 2); + if (prp[0]) + pmm_release((void *)prp[0]); + return err; } diff --git a/kernel-src/io/fb.c b/kernel-src/io/fb.c index 4bdcdb92..4f99a1ae 100644 --- a/kernel-src/io/fb.c +++ b/kernel-src/io/fb.c @@ -129,14 +129,14 @@ static int mmap(int minor, void *addr, uintmax_t offset, int flags) { return arch_mmu_map(_cpu()->vmmctx->pagetable, FROM_HHDM((void *)((uintptr_t)fbs[minor]->address + offset)), addr, vnodeflagstommuflags(flags)) ? 0 : ENOMEM; } else { size_t size = offset + PAGE_SIZE < end ? PAGE_SIZE : end - offset; - paddr = pmm_alloc(1, PMM_SECTION_DEFAULT); + paddr = pmm_allocpage(PMM_SECTION_DEFAULT); if (paddr == NULL) return ENOMEM; memcpy(MAKE_HHDM(paddr), (void *)((uintptr_t)fbs[minor]->address + offset), size); if (arch_mmu_map(_cpu()->vmmctx->pagetable, paddr, addr, vnodeflagstommuflags(flags)) == false) { - pmm_free(paddr, 1); + pmm_release(paddr); return ENOMEM; } @@ -148,7 +148,7 @@ static int munmap(int minor, void *addr, uintmax_t offset, int flags) { void *phys = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, addr); arch_mmu_unmap(_cpu()->vmmctx->pagetable, addr); if ((flags & V_FFLAGS_SHARED) == 0) { - pmm_free(phys, 1); + pmm_release(phys); } return 0; diff --git a/kernel-src/mm/pgcache.c b/kernel-src/mm/pgcache.c new file mode 100644 index 00000000..e4263e6b --- /dev/null +++ b/kernel-src/mm/pgcache.c @@ -0,0 +1,3 @@ +void pgcache_init() { + +} diff --git a/kernel-src/mm/pmm.c b/kernel-src/mm/pmm.c index 229033a0..737b73c9 100644 --- a/kernel-src/mm/pmm.c +++ b/kernel-src/mm/pmm.c @@ -4,22 +4,30 @@ #include #include #include +#include uintptr_t hhdmbase; static size_t memorysize; -static size_t physicalpagecount; -static size_t used; +static size_t pagecount; + +static spinlock_t freelistlock; +static page_t *freelists[PMM_SECTION_COUNT]; +static page_t *standbylists[PMM_SECTION_COUNT]; typedef struct { - spinlock_t lock; - uintmax_t base; - uintmax_t top; + uintmax_t baseid; + uintmax_t topid; uintmax_t searchstart; -} bmsection_t; +} section_t; -static bmsection_t bmsections[PMM_SECTION_COUNT]; +#define TOP_1MB (0x100000 / PAGE_SIZE) +#define TOP_4GB ((uint64_t)0x100000000 / PAGE_SIZE) -static uint64_t *bitmap; +static section_t sections[PMM_SECTION_COUNT] = { + {0, TOP_1MB, 0}, + {TOP_1MB, TOP_4GB, TOP_1MB}, + {TOP_4GB, 0xffffffffffffffffl, TOP_4GB} +}; static volatile struct limine_hhdm_request hhdmreq = { .id = LIMINE_HHDM_REQUEST, @@ -31,169 +39,215 @@ volatile struct limine_memmap_request pmm_liminemap = { .revision = 0 }; -static inline void bmset(void *addr, uint64_t v) { - uintmax_t page = (uintptr_t)addr / PAGE_SIZE; - uintmax_t entry = page / 64; - uintmax_t offset = page % 64; +static page_t* pages; - uint64_t entryval = bitmap[entry]; - entryval &= ~((uint64_t)1 << offset); - entryval |= v << offset; - bitmap[entry] = entryval; +#define PAGE_GETID(page) (((uintptr_t)(page) - (uintptr_t)pages) / sizeof(page_t)) +#define PAGE_BOUNDARYCHECK(pageid) \ + __assert((pageid) * PAGE_SIZE < (uintptr_t)pages || (pageid) * PAGE_SIZE >= (uintptr_t)&pages[pagecount]) + +static void insertinfreelist(page_t *page) { + uintmax_t pageid = PAGE_GETID(page); + PAGE_BOUNDARYCHECK(pageid); + struct page_t **list; + + int section; + + if (pageid < TOP_1MB) + section = PMM_SECTION_1MB; + else if (pageid < TOP_4GB) + section = PMM_SECTION_4GB; + else + section = PMM_SECTION_DEFAULT; + + list = page->backing ? &standbylists[section] : &freelists[section]; + + if (sections[section].searchstart > pageid) + sections[section].searchstart = pageid; + + page->freenext = *list; + page->freeprev = NULL; + *list = page; + if (page->freenext) + page->freenext->freeprev = page; } -static inline bool bmget(void *addr) { - uintmax_t page = (uintptr_t)addr / PAGE_SIZE; - uintmax_t entry = page / 64; - uintmax_t offset = page % 64; +static void removefromfreelist(page_t *page) { + uintmax_t pageid = PAGE_GETID(page); + PAGE_BOUNDARYCHECK(pageid); + struct page_t **list; - uint64_t entryval = bitmap[entry]; - entryval &= ((uint64_t)1 << offset); - return entryval; + int section; + + if (pageid < TOP_1MB) + section = PMM_SECTION_1MB; + else if (pageid < TOP_4GB) + section = PMM_SECTION_4GB; + else + section = PMM_SECTION_DEFAULT; + + list = page->backing ? &standbylists[section] : &freelists[section]; + + if (page->freeprev) + page->freeprev->freenext = page->freenext; + else + *list = page->freenext; + + if (page->freenext) + page->freenext->freeprev = page->freeprev; } -static uintmax_t getfreearea(bmsection_t *section, size_t size) { - uintmax_t result = 0; - uintmax_t found = 0; +void pmm_hold(void *addr) { + page_t *page = &pages[((uintptr_t)addr / PAGE_SIZE)]; + __atomic_add_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST); +} - for (uintmax_t search = section->searchstart; search < section->top && found < size; ++search) { - uintmax_t entry = search / 64; - uintmax_t offset = search % 64; +void pmm_release(void *addr) { + page_t *page = &pages[(uintptr_t)addr / PAGE_SIZE]; + __assert(page->refcount != 0); - if (bitmap[entry] == 0) { - search += 63 - offset; // 63 because search gets incremented on continue - result = 0; - found = 0; - continue; - } - - if (((bitmap[entry] >> offset) & 1) == 0) { - found = 0; - result = 0; - continue; - } - - if (found == 0) - result = search; - - ++found; + uintmax_t newrefcount = __atomic_sub_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST); + if (newrefcount == 0) { + spinlock_acquire(&freelistlock); + insertinfreelist(page); + if (page->backing == NULL) + page->flags |= PAGE_FLAGS_FREE; + spinlock_release(&freelistlock); } - - return result; } -static inline void initsection(int section, uintmax_t base, uintmax_t top) { - bmsections[section] = (bmsection_t){ - .base = base, - .top = top, - .searchstart = base - }; - SPINLOCK_INIT(bmsections[section].lock); +static void doalloc(page_t *page) { + __assert(page->refcount == 0); + memset(page, 0, sizeof(page_t)); + page->refcount = 1; + page->flags &= ~PAGE_FLAGS_FREE; } -void *pmm_alloc(size_t size, int section) { +void *pmm_allocpage(int section) { + spinlock_acquire(&freelistlock); + page_t *page = NULL; + for (int i = section; i >= 0; --i) { - spinlock_acquire(&bmsections[i].lock); - uintmax_t page = getfreearea(&bmsections[i], size); - + page = freelists[i]; if (page) { - if (page == bmsections[i].searchstart) - bmsections[i].searchstart += size; - - for (size_t j = 0; j < size; ++j) - bmset((void *)((page + j) * PAGE_SIZE), 0); - - spinlock_release(&bmsections[i].lock); - __atomic_add_fetch(&used, 1, __ATOMIC_SEQ_CST); - return (void *)(page * PAGE_SIZE); - } - spinlock_release(&bmsections[i].lock); - } - return NULL; -} - -void pmm_free(void *addr, size_t size) { - uintmax_t page = (uintptr_t)addr / PAGE_SIZE; - - for (int i = 0; i < PMM_SECTION_COUNT; ++i) { - if (page >= bmsections[i].base && page < bmsections[i].top) { - spinlock_acquire(&bmsections[i].lock); - - if ((uintptr_t)addr / PAGE_SIZE < bmsections[i].searchstart) - bmsections[i].searchstart = (uintptr_t)addr / PAGE_SIZE; - - for (uintmax_t j = 0; j < size; ++j) { - __assert(bmget((void *)((page + j) * PAGE_SIZE)) == false); - bmset((void *)((page + j) * PAGE_SIZE), 1); - } - - spinlock_release(&bmsections[i].lock); - __atomic_sub_fetch(&used, 1, __ATOMIC_SEQ_CST); - return; + removefromfreelist(page); + break; } } + + // TODO get page from standby list and call to remove from cache + + spinlock_release(&freelistlock); + void *address = NULL; + if (page) { + address = (void *)(PAGE_GETID(page) * PAGE_SIZE); + doalloc(page); + } + + return address; } -void pmm_getinfo(size_t *pages, size_t *usedpages) { - if (pages) - *pages = physicalpagecount; - if (usedpages) - *usedpages = used; +void pmm_makefree(void *address, size_t count) { + memorysize += PAGE_SIZE * count; + __assert(((uintptr_t)address % PAGE_SIZE) == 0); + uintmax_t baseid = (uintptr_t)address / PAGE_SIZE; + for (int i = 0; i < count; ++i) { + uintmax_t pageid = baseid + i; + PAGE_BOUNDARYCHECK(pageid); + page_t *page = &pages[pageid]; + insertinfreelist(page); + } } -#define TOP_1MB (0x100000 / PAGE_SIZE) -#define TOP_4GB ((uint64_t)0x100000000 / PAGE_SIZE) - void pmm_init() { __assert(hhdmreq.response); hhdmbase = hhdmreq.response->offset; __assert(pmm_liminemap.response); - uintmax_t topofmemory = 0; - - // get size of memory and print memory map - printf("memory map:\n"); + // get size of memory, top of usable memory, biggest section and print memory map + size_t top = 0; + struct limine_memmap_entry *biggest = NULL; + printf("pmm: ranges:\n"); for (size_t i = 0; i < pmm_liminemap.response->entry_count; ++i) { struct limine_memmap_entry *e = pmm_liminemap.response->entries[i]; - printf("%016p -> %016p: %d\n", e->base, e->base + e->length, e->type); + printf("pmm: %016p -> %016p: %d\n", e->base, e->base + e->length, e->type); + if (e->type == LIMINE_MEMMAP_USABLE || e->type == LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE) { + size_t sectiontop = e->base + e->length; + if (sectiontop > top) + top = sectiontop; + } if (e->type == LIMINE_MEMMAP_USABLE) { - topofmemory = e->base + e->length; + if (biggest == NULL || e->length > biggest->length) + biggest = e; memorysize += e->length; } } - physicalpagecount = topofmemory / PAGE_SIZE; + pages = MAKE_HHDM((page_t *)biggest->base); + pagecount = ROUND_UP(top, PAGE_SIZE) / PAGE_SIZE; + memset(pages, 0, pagecount / sizeof(page_t)); + printf("pmm: %d pages used for page list\n", ROUND_UP(pagecount * sizeof(page_t), PAGE_SIZE) / PAGE_SIZE); - size_t bitmapsize = physicalpagecount / 8; - - // find out where to put the bitmap - for (uintmax_t i = 0; i < pmm_liminemap.response->entry_count; ++i) { + // place pages in free lists + for (size_t i = 0; i < pmm_liminemap.response->entry_count; ++i) { struct limine_memmap_entry *e = pmm_liminemap.response->entries[i]; - if (e->type == LIMINE_MEMMAP_USABLE && e->length >= bitmapsize) - bitmap = (uint64_t *)e->base; - } - __assert(bitmap); - - memset(bitmap, 0, bitmapsize); - - // set free pages - for (uintmax_t i = 0; i < pmm_liminemap.response->entry_count; ++i) { - struct limine_memmap_entry *e = pmm_liminemap.response->entries[i]; - if (e->type != LIMINE_MEMMAP_USABLE) - continue; - - for (uintmax_t offset = 0; offset < e->length; offset += PAGE_SIZE) - bmset((void *)(e->base + offset), 1); + if (e->type == LIMINE_MEMMAP_USABLE) { + int firstusablepage = e == biggest ? ROUND_UP(e->base + pagecount * sizeof(page_t), PAGE_SIZE) / PAGE_SIZE : e->base / PAGE_SIZE; + for (int i = firstusablepage; i < (e->base + e->length) / PAGE_SIZE; ++i) { + pages[i].flags |= PAGE_FLAGS_FREE; + insertinfreelist(&pages[i]); + } + } } - // set bitmap as used - for (uintmax_t offset = 0; offset < bitmapsize; offset += PAGE_SIZE) - bmset((void *)((uintptr_t)bitmap + offset), 0); - - // prepare sections - // for all cases this section will always be present so hardcode its values (otherwise the system wouldn't even get here) - initsection(PMM_SECTION_1MB, 1, TOP_1MB); - initsection(PMM_SECTION_4GB, TOP_1MB, physicalpagecount > TOP_4GB ? TOP_4GB : physicalpagecount); - initsection(PMM_SECTION_DEFAULT, TOP_4GB, physicalpagecount > TOP_4GB ? physicalpagecount : TOP_4GB); - bitmap = MAKE_HHDM(bitmap); + SPINLOCK_INIT(freelistlock); +} + +void *pmm_alloc(size_t size, int section) { + __assert(size); + // pmm_allocpage is more suited for single page allocations, so use that instead + if (size == 1) + return pmm_allocpage(section); + + spinlock_acquire(&freelistlock); + + uintmax_t page = 0; + size_t found = 0; + + for (; section >= 0; --section) { + for (page = sections[section].searchstart; page < sections[section].topid; ++page) { + if (page >= pagecount) + break; + + if (pages[page].flags & PAGE_FLAGS_FREE) { + ++found; + } else + found = 0; + + if (found == size) + goto gotpages; + } + found = 0; + } + + gotpages: + void *addr = NULL; + if (found == size) { + page = page - (found - 1); + addr = (void *)(page * PAGE_SIZE); + for (int i = 0; i < size; ++i) { + removefromfreelist(&pages[page + i]); + doalloc(&pages[page + i]); + } + } + + spinlock_release(&freelistlock); + return addr; +} + +void pmm_free(void *addr, size_t size) { + __assert(size); + // release multiple pages at once + __assert(((uintptr_t)addr % PAGE_SIZE) == 0); + for (int i = 0; i < size; ++i) + pmm_release((void *)((uintptr_t)addr + PAGE_SIZE * i)); } diff --git a/kernel-src/mm/vmm.c b/kernel-src/mm/vmm.c index b2de2e58..71cd1d23 100644 --- a/kernel-src/mm/vmm.c +++ b/kernel-src/mm/vmm.c @@ -10,7 +10,7 @@ #define RANGE_TOP(x) (void *)((uintptr_t)x->start + x->size) static vmmcache_t *newcache() { - vmmcache_t *ptr = pmm_alloc(1, PMM_SECTION_DEFAULT); + vmmcache_t *ptr = pmm_allocpage(PMM_SECTION_DEFAULT); if (ptr == NULL) return NULL; ptr = MAKE_HHDM(ptr); @@ -221,7 +221,7 @@ static void destroyrange(vmmrange_t *range, uintmax_t _offset, size_t size, int if ((range->flags & VMM_FLAGS_FILE) && (range->flags & VMM_FLAGS_SHARED) && (flags & VMM_DESTROY_FLAGS_NOSYNC) == 0) { __assert(VOP_MUNMAP(range->vnode, vaddr, range->offset + offset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0); } else { - pmm_free(physical, 1); + pmm_release(physical); arch_mmu_unmap(_cpu()->vmmctx->pagetable, vaddr); } } @@ -330,7 +330,7 @@ bool vmm_pagefault(void *addr, bool user, int actions) { __assert(VOP_MMAP(range->vnode, addr, range->offset + mapoffset, mmuflagstovnodeflags(range->mmuflags) | (range->flags & VMM_FLAGS_SHARED ? V_FFLAGS_SHARED : 0), cred) == 0); status = true; } else { - void *paddr = pmm_alloc(1, PMM_SECTION_DEFAULT); + void *paddr = pmm_allocpage(PMM_SECTION_DEFAULT); __assert(paddr); // XXX handle these better than an assert once there are signals and such __assert(arch_mmu_map(_cpu()->vmmctx->pagetable, paddr, addr, range->mmuflags)); memset(MAKE_HHDM(paddr), 0, PAGE_SIZE); @@ -399,7 +399,7 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, } else if (flags & VMM_FLAGS_ALLOCATE) { // allocate to virtual memory for (uintmax_t i = 0; i < size; i += PAGE_SIZE) { - void *allocated = pmm_alloc(1, PMM_SECTION_DEFAULT); + void *allocated = pmm_allocpage(PMM_SECTION_DEFAULT); if (allocated == NULL) { retaddr = NULL; goto cleanup; @@ -411,7 +411,7 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags, void *physical = arch_mmu_getphysical(_cpu()->vmmctx->pagetable, virt); if (physical) { - pmm_free(physical, 1); + pmm_release(physical); arch_mmu_unmap(_cpu()->vmmctx->pagetable, virt); } } @@ -529,7 +529,7 @@ vmmcontext_t *vmm_fork(vmmcontext_t *oldcontext) { if (phys == NULL) continue; - void *palloc = pmm_alloc(1, PMM_SECTION_DEFAULT); + void *palloc = pmm_allocpage(PMM_SECTION_DEFAULT); if (palloc == NULL) goto error;