mmu: enable global pages

kernel-src/arch/x86-64/cpu.c

@@ -188,6 +188,8 @@ void arch_cpu_init() {
 		: : : "rax"
 	);
 
+	x86_64_mmu_enable_global_pages();
+
 	// register some exception handlers that give out signals
 	interrupt_register(0, div0isr, NULL, IPL_IGNORE);
 	interrupt_register(6, illisr, NULL, IPL_IGNORE);

kernel-src/arch/x86-64/mmu.c

@@ -192,6 +192,22 @@ void arch_mmu_switch(pagetableptr_t table) {
 	asm volatile("mov %%rax, %%cr3" : : "a"(table));
 }
 
+void x86_64_mmu_enable_global_pages(void) {
+	asm volatile(
+	"mov %%cr4, %%rax;"
+	"or  $0x80,  %%rax;"
+	"mov %%rax, %%cr4;"
+	: : : "rax", "memory");
+}
+
+void x86_64_mmu_disable_global_pages(void) {
+	asm volatile(
+	"mov %%cr4, %%rax;"
+	"and $0xffffffffffffff7f, %%rax;"
+	"mov %%rax, %%cr4;"
+	: : : "rax", "memory");
+}
+
 // hhdm pointer to template to be used for new mappings and smp bootup
 static pagetableptr_t template;
 
@@ -206,8 +222,13 @@ pagetableptr_t arch_mmu_newtable() {
 static inline void do_invalidate(void *page, size_t size) {
 	// if a full reload was requested or we are doing a big release on 
 	if (page == NULL || size >= 32 * PAGE_SIZE) {
-		// TODO if global pages are ever supported, we should disable them in CR4, flush, and then reenable them
+		if (page != NULL && IS_USER_ADDRESS(page))
+			x86_64_mmu_disable_global_pages();
+
 		asm volatile("mov %%cr3, %%rax; mov %%rax, %%cr3;" : : : "rax", "memory");
+
+		if (page != NULL && IS_USER_ADDRESS(page))
+			x86_64_mmu_enable_global_pages();
 	} else {
 		for (uintptr_t i = 0; i < size; i += PAGE_SIZE) {
 			uintptr_t ptr = (uintptr_t)page + i;
@@ -265,7 +286,7 @@ void arch_mmu_invalidate_range(void *page, size_t size) {
 	do_invalidate(page, size);
 
 	if (do_shootdown) {
-		while (__atomic_load_n(&shootdown_done, __ATOMIC_SEQ_CST) != shootdown_total) CPU_PAUSE();
+		while (__atomic_load_n(&shootdown_done, __ATOMIC_RELAXED) != shootdown_total) CPU_PAUSE();
 	}
 }
 
@@ -286,9 +307,9 @@ static void *kerneladdr[6] = {
 };
 
 static mmuflags_t kernelflags[3] = {
-	ARCH_MMU_FLAGS_READ,
-	ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC,
-	ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC
+	ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_GLOBAL,
+	ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_GLOBAL,
+	ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_GLOBAL
 };
 
 extern volatile struct limine_memmap_request pmm_liminemap;
@@ -363,7 +384,7 @@ void arch_mmu_init() {
 			continue;
 
 		for (uint64_t i = 0; i < e->length; i += PAGE_SIZE) {
-			uint64_t entry = ((e->base + i) & ADDRMASK) | ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC;
+			uint64_t entry = ((e->base + i) & ADDRMASK) | ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_GLOBAL;
 			__assert(add_page(FROM_HHDM(template), MAKE_HHDM((void *)(e->base + i)), entry, 0));
 		}
 	}

kernel-src/include/x86-64/arch/mmu.h

@@ -17,6 +17,7 @@
 #define ARCH_MMU_FLAGS_WRITE (uint64_t)2
 #define ARCH_MMU_FLAGS_USER (uint64_t)4
 #define ARCH_MMU_FLAGS_NOEXEC ((uint64_t)1 << 63)
+#define ARCH_MMU_FLAGS_GLOBAL ((uint64_t)1 << 8)
 
 #define ARCH_MMU_FLAGS_WB 0
 #define ARCH_MMU_FLAGS_WT (1 << 3)
@@ -43,4 +44,8 @@ void arch_mmu_apswitch();
 void arch_mmu_invalidate_range(void *page, size_t size);
 bool arch_mmu_getflags(pagetableptr_t table, void *vaddr, mmuflags_t *mmuflagsp);
 
+
+void x86_64_mmu_enable_global_pages(void);
+void x86_64_mmu_disable_global_pages(void);
+
 #endif

kernel-src/mm/vmm.c

@@ -505,6 +505,9 @@ int vmm_changemmuflags(void *base, size_t size, mmuflags_t mmuflags, int flags)
 	if (space == NULL)
 		return ENOMEM;
 
+	if (space == &kernelspace)
+		mmuflags |= ARCH_MMU_FLAGS_GLOBAL;
+
 	MUTEX_ACQUIRE(&space->lock);
 
 	int error = changemap(space, base, size, false, flags, mmuflags);
@@ -699,6 +702,9 @@ void *vmm_map(void *addr, volatile size_t size, int flags, mmuflags_t mmuflags,
 	if (space == NULL)
 		return NULL;
 
+	if (space == &kernelspace)
+		mmuflags |= ARCH_MMU_FLAGS_GLOBAL;
+
 	MUTEX_ACQUIRE(&space->lock);
 	vmmrange_t *range = NULL;
 
@@ -947,14 +953,14 @@ void vmm_init() {
 		if (e->type != LIMINE_MEMMAP_USABLE && e->type != LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE && e->type != LIMINE_MEMMAP_KERNEL_AND_MODULES && e->type != LIMINE_MEMMAP_FRAMEBUFFER)
 			continue;
 
-		mmuflags_t mmuflags = ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC;
+		mmuflags_t mmuflags = ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_GLOBAL;
 		__assert(vmm_map(MAKE_HHDM(e->base), e->length, VMM_FLAGS_EXACT, mmuflags, NULL));
 	}
 
 	// map kernel
-	__assert(vmm_map(&_text_start, (uintptr_t)&_text_end - (uintptr_t)&_text_start, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_READ, NULL));
-	__assert(vmm_map(&_rodata_start, (uintptr_t)&_rodata_end - (uintptr_t)&_rodata_start, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC, NULL));
-	__assert(vmm_map(&_data_start, (uintptr_t)&_data_end - (uintptr_t)&_data_start, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL));
+	__assert(vmm_map(&_text_start, (uintptr_t)&_text_end - (uintptr_t)&_text_start, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_GLOBAL, NULL));
+	__assert(vmm_map(&_rodata_start, (uintptr_t)&_rodata_end - (uintptr_t)&_rodata_start, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_GLOBAL, NULL));
+	__assert(vmm_map(&_data_start, (uintptr_t)&_data_end - (uintptr_t)&_data_start, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC | ARCH_MMU_FLAGS_GLOBAL, NULL));
 
 	// null page
 	vmm_map(MAKE_HHDM(NULL), PAGE_SIZE, VMM_FLAGS_EXACT, ARCH_MMU_FLAGS_NOEXEC, NULL);