#include #include #include #include #include #include #include #include #include uintptr_t hhdmbase; static size_t memorysize; static size_t pagecount; size_t freepagecount; static mutex_t freelistmutex; static page_t *freelists[PMM_SECTION_COUNT]; static page_t *freetails[PMM_SECTION_COUNT]; static page_t *standbylists[PMM_SECTION_COUNT]; static page_t *standbytails[PMM_SECTION_COUNT]; typedef struct { uintmax_t baseid; uintmax_t topid; uintmax_t searchstart; } section_t; #define TOP_1MB (0x100000 / PAGE_SIZE) #define TOP_4GB ((uint64_t)0x100000000 / PAGE_SIZE) static section_t sections[PMM_SECTION_COUNT] = { {0, TOP_1MB, 0}, {TOP_1MB, TOP_4GB, TOP_1MB}, {TOP_4GB, 0xffffffffffffffffl, TOP_4GB} }; static volatile struct limine_hhdm_request hhdmreq = { .id = LIMINE_HHDM_REQUEST, .revision = 0 }; volatile struct limine_memmap_request pmm_liminemap = { .id = LIMINE_MEMMAP_REQUEST, .revision = 0 }; static page_t* pages; #define PAGE_GETID(page) (((uintptr_t)(page) - (uintptr_t)pages) / sizeof(page_t)) #define PAGE_BOUNDARYCHECK(pageid) \ __assert((pageid) * PAGE_SIZE < (uintptr_t)pages || (pageid) * PAGE_SIZE >= (uintptr_t)&pages[pagecount]) static void insertinfreelist(page_t *page) { uintmax_t pageid = PAGE_GETID(page); PAGE_BOUNDARYCHECK(pageid); struct page_t **list; struct page_t **tail; int section; if (pageid < TOP_1MB) section = PMM_SECTION_1MB; else if (pageid < TOP_4GB) section = PMM_SECTION_4GB; else section = PMM_SECTION_DEFAULT; list = page->backing ? &standbylists[section] : &freelists[section]; tail = page->backing ? &standbytails[section] : &freetails[section]; if (sections[section].searchstart > pageid) sections[section].searchstart = pageid; page->freenext = *list; page->freeprev = NULL; *list = page; if (page->freenext) page->freenext->freeprev = page; else *tail = page; ++freepagecount; } static void removefromfreelist(page_t *page) { uintmax_t pageid = PAGE_GETID(page); PAGE_BOUNDARYCHECK(pageid); struct page_t **list; struct page_t **tail; int section; if (pageid < TOP_1MB) section = PMM_SECTION_1MB; else if (pageid < TOP_4GB) section = PMM_SECTION_4GB; else section = PMM_SECTION_DEFAULT; list = page->backing ? &standbylists[section] : &freelists[section]; tail = page->backing ? &standbytails[section] : &freetails[section]; if (page->freeprev) page->freeprev->freenext = page->freenext; else *list = page->freenext; if (page->freenext) page->freenext->freeprev = page->freeprev; else *tail = page->freeprev; --freepagecount; } static void internalhold(page_t *page) { __atomic_add_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST); if (page->refcount == 1) { // this is only valid on standby pages, in case of free pages its an use after free __assert((page->flags & PAGE_FLAGS_FREE) == 0); removefromfreelist(page); } } page_t *pmm_getpage(void *address) { return &pages[((uintptr_t)address / PAGE_SIZE)]; } void *pmm_getpageaddress(page_t *page) { return (void *)(PAGE_GETID(page) * PAGE_SIZE); } void pmm_hold(void *addr) { page_t *page = &pages[((uintptr_t)addr / PAGE_SIZE)]; MUTEX_ACQUIRE(&freelistmutex); internalhold(page); MUTEX_RELEASE(&freelistmutex); } void pmm_release(void *addr) { page_t *page = &pages[(uintptr_t)addr / PAGE_SIZE]; __assert(page->refcount != 0); uintmax_t newrefcount = __atomic_sub_fetch(&page->refcount, 1, __ATOMIC_SEQ_CST); if (newrefcount == 0) { __assert((page->flags & PAGE_FLAGS_DIRTY) == 0); MUTEX_ACQUIRE(&freelistmutex); insertinfreelist(page); if (page->backing == NULL) page->flags |= PAGE_FLAGS_FREE; MUTEX_RELEASE(&freelistmutex); } } static void doalloc(page_t *page) { __assert(page->refcount == 0); memset(page, 0, sizeof(page_t)); page->refcount = 1; } void *pmm_allocpage(int section) { retry: MUTEX_ACQUIRE(&freelistmutex); page_t *page = NULL; // try to take a free anonymous page for (int i = section; i >= 0; --i) { page = freelists[i]; if (page) { removefromfreelist(page); __assert(page->refcount == 0); break; } } bool cachepage = false; // if that wasn't possible, try to take from the cache standby list if (page == NULL) { for (int i = section; i >= 0; --i) { page = standbytails[i]; if (page) { cachepage = true; internalhold(page); break; } } } MUTEX_RELEASE(&freelistmutex); if (cachepage && vmmcache_takepage(page) == EAGAIN) { // someone already got the page from the cache between us holding it and taking it pmm_release(pmm_getpageaddress(page)); page = NULL; // retry it from the start, as an anonymous page could have been released while the lock was not held goto retry; } else if (cachepage) { // we got the page from the cache, all is good and we hold the only reference to it. // set the refcount to 0, as expected by the doalloc call page->refcount = 0; } void *address = NULL; if (page) { address = (void *)(PAGE_GETID(page) * PAGE_SIZE); doalloc(page); } return address; } void pmm_makefree(void *address, size_t count) { memorysize += PAGE_SIZE * count; __assert(((uintptr_t)address % PAGE_SIZE) == 0); uintmax_t baseid = (uintptr_t)address / PAGE_SIZE; for (int i = 0; i < count; ++i) { uintmax_t pageid = baseid + i; PAGE_BOUNDARYCHECK(pageid); page_t *page = &pages[pageid]; insertinfreelist(page); } } void pmm_init() { __assert(hhdmreq.response); hhdmbase = hhdmreq.response->offset; __assert(pmm_liminemap.response); // get size of memory, top of usable memory, biggest section and print memory map size_t top = 0; struct limine_memmap_entry *biggest = NULL; printf("pmm: ranges:\n"); for (size_t i = 0; i < pmm_liminemap.response->entry_count; ++i) { struct limine_memmap_entry *e = pmm_liminemap.response->entries[i]; printf("pmm: %016p -> %016p: %d\n", e->base, e->base + e->length, e->type); if (e->type == LIMINE_MEMMAP_USABLE || e->type == LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE || e->type == LIMINE_MEMMAP_KERNEL_AND_MODULES) { size_t sectiontop = e->base + e->length; if (sectiontop > top) top = sectiontop; } if (e->type == LIMINE_MEMMAP_USABLE) { if (biggest == NULL || e->length > biggest->length) biggest = e; memorysize += e->length; } } pages = MAKE_HHDM((page_t *)biggest->base); pagecount = ROUND_UP(top, PAGE_SIZE) / PAGE_SIZE; memset(pages, 0, pagecount * sizeof(page_t)); printf("pmm: %d pages used for page list\n", ROUND_UP(pagecount * sizeof(page_t), PAGE_SIZE) / PAGE_SIZE); // place pages in free lists for (size_t i = 0; i < pmm_liminemap.response->entry_count; ++i) { struct limine_memmap_entry *e = pmm_liminemap.response->entries[i]; if (e->type == LIMINE_MEMMAP_USABLE) { int firstusablepage = e == biggest ? ROUND_UP(e->base + pagecount * sizeof(page_t), PAGE_SIZE) / PAGE_SIZE : e->base / PAGE_SIZE; for (int i = firstusablepage; i < (e->base + e->length) / PAGE_SIZE; ++i) { pages[i].flags |= PAGE_FLAGS_FREE; insertinfreelist(&pages[i]); } } } MUTEX_INIT(&freelistmutex); } INIT_ROUTINE_DEFINE(pmm, INIT_ROUTINE_FLAGS_NONE, pmm_init, arch_early); // XXX pmm_alloc won't be able to take pages from the page cache when the allocation size is over 1 page void *pmm_alloc(size_t size, int section) { __assert(size); // pmm_allocpage is more suited for single page allocations, so use that instead if (size == 1) return pmm_allocpage(section); MUTEX_ACQUIRE(&freelistmutex); uintmax_t page = 0; size_t found = 0; for (; section >= 0; --section) { for (page = sections[section].searchstart; page < sections[section].topid; ++page) { if (page >= pagecount) break; if (pages[page].flags & PAGE_FLAGS_FREE) { ++found; } else found = 0; if (found == size) goto gotpages; } found = 0; } gotpages: void *addr = NULL; if (found == size) { page = page - (found - 1); addr = (void *)(page * PAGE_SIZE); for (int i = 0; i < size; ++i) { removefromfreelist(&pages[page + i]); doalloc(&pages[page + i]); } } MUTEX_RELEASE(&freelistmutex); return addr; } void pmm_free(void *addr, size_t size) { __assert(size); // release multiple pages at once __assert(((uintptr_t)addr % PAGE_SIZE) == 0); for (int i = 0; i < size; ++i) pmm_release((void *)((uintptr_t)addr + PAGE_SIZE * i)); } void pmm_getinfo(size_t *total_pages, size_t *free_pages) { *total_pages = pagecount; *free_pages = freepagecount; }