x86-64: enable smep and smap

kernel-src/arch/x86-64/cpu.c

@@ -12,6 +12,11 @@
 #include <arch/prof.h>
 
 #define EFER_SYSCALLENABLE 1
+#define CR4_SMEP (1 << 20)
+#define CR4_SMAP (1 << 21)
+
+static bool has_smap;
+static bool has_smap_detected;
 
 void arch_syscall_entry();
 static void illisr(isr_t *self, context_t *ctx) {
@@ -61,6 +66,55 @@ static void reload_ldt(isr_t *self, context_t *ctx) {
 	arch_ldt_invalidate();
 }
 
+static void enable_smep_if_available(void) {
+	if (cpuid_base_max_leaf() < 7)
+		return;
+
+	cpuid_results_t results;
+	cpuid_with_ecx(7, 0, &results);
+	if ((results.ebx & CPUID_LEAF_7_EBX_SMEP) == 0)
+		return;
+
+	uint64_t cr4;
+	asm volatile("mov %%cr4, %0" : "=r"(cr4));
+	cr4 |= CR4_SMEP;
+	asm volatile("mov %0, %%cr4" : : "r"(cr4) : "memory");
+}
+
+static bool detect_smap(void) {
+	if (cpuid_base_max_leaf() < 7)
+		return false;
+
+	cpuid_results_t results;
+	cpuid_with_ecx(7, 0, &results);
+	return results.ebx & CPUID_LEAF_7_EBX_SMAP;
+}
+
+static void enable_smap_if_available(void) {
+	if (!has_smap_detected) {
+		has_smap = detect_smap();
+		has_smap_detected = true;
+	}
+
+	if (!has_smap)
+		return;
+
+	uint64_t cr4;
+	asm volatile("mov %%cr4, %0" : "=r"(cr4));
+	cr4 |= CR4_SMAP;
+	asm volatile("mov %0, %%cr4" : : "r"(cr4) : "memory");
+}
+
+void arch_cpu_user_access_begin(void) {
+	if (has_smap)
+		asm volatile("stac" : : : "memory");
+}
+
+void arch_cpu_user_access_end(void) {
+	if (has_smap)
+		asm volatile("clac" : : : "memory");
+}
+
 void arch_nmi_isr(context_t *ctx) {
 	// we are in an NMI context, running on IST0 
 	// first load the proper gsbase to get access to cpu-local variables. 
@@ -227,6 +281,8 @@ void arch_cpu_init() {
 	);
 
 	x86_64_mmu_enable_global_pages();
+	enable_smep_if_available();
+	enable_smap_if_available();
 
 	// register some exception handlers that give out signals
 	interrupt_register(0, sigfpeisr, NULL, IPL_IGNORE);

kernel-src/arch/x86-64/mmu.c

@@ -324,7 +324,14 @@ static volatile struct limine_executable_address_request kaddrreq = {
 #define ERROR_FETCH   16
 
 static void pfisr(isr_t *self, context_t *ctx) {
+	if ((ctx->rflags & ARCH_CONTEXT_RFLAGS_AC) == 0 && IS_USER_ADDRESS((void *)ctx->cr2) && !IS_USER_ADDRESS((void *)ctx->rip)) {
+		_panic("SMAP/SMEP violation", ctx);
+	}
+
 	thread_t *thread = current_thread();
+	if (thread && thread->usercopyctx)
+		arch_cpu_user_access_end();
+
 	interrupt_set(true);
 	int vmmerror = 0;
 
@@ -338,6 +345,7 @@ static void pfisr(isr_t *self, context_t *ctx) {
 	if (vmm_pagefault((void *)ctx->cr2, ctx->cs != 8, vmmerror) == false) {
 		if (thread && thread->usercopyctx) {
 			memcpy(ctx, thread->usercopyctx, sizeof(context_t));
+			ctx->rflags &= ~ARCH_CONTEXT_RFLAGS_AC;
 			thread->usercopyctx = NULL;
 			CTX_RET(ctx) = EFAULT;
 		} else if (ARCH_CONTEXT_ISUSER(ctx)) {
@@ -361,7 +369,9 @@ static void gpfisr(isr_t *self, context_t *ctx) {
 
 	thread_t *thread = current_thread();
 	if (thread && thread->usercopyctx) {
+		arch_cpu_user_access_end();
 		memcpy(ctx, thread->usercopyctx, sizeof(context_t));
+		ctx->rflags &= ~ARCH_CONTEXT_RFLAGS_AC;
 		thread->usercopyctx = NULL;
 		CTX_RET(ctx) = EFAULT;
 	} else if (ARCH_CONTEXT_ISUSER(ctx)) {

kernel-src/include/x86-64/arch/context.h

@@ -67,6 +67,8 @@ extern size_t arch_xsave_size;
 #define CTX_ARG1(x) (x)->rsi
 #define CTX_ARG2(x) (x)->rdx
 
+#define ARCH_CONTEXT_RFLAGS_AC (1 << 18)
+
 #define PRINT_CTX(x) { \
 	printf("cr2: %016lx  gs: %04x\nrax: %016lx  fs: %04x\nrbx: %016lx  es: %04x\nrcx: %016lx  ds: %04x\nrdx: %016lx  cs: %04x\n r8: %016lx  ss: %04x\n r9: %016lx r10: %016lx\nr11: %016lx r12: %016lx\nr13: %016lx r14: %016lx\nr15: %016lx rdi: %016lx\nrsi: %016lx rbp: %016lx\nrip: %016lx rsp: %016lx\nerr: %016lx rfl: %016lx\n", \
 	x->cr2, x->gs, x->rax, x->fs, x->rbx, x->es, x->rcx, x->ds, x->rdx, x->cs, x->r8, x->ss, x->r9, x->r10, x->r11, x->r12, x->r13, x->r14, x->r15, x->rdi, x->rsi, x->rbp, x->rip, x->rsp, x->error, x->rflags); \

kernel-src/include/x86-64/arch/cpu.h

@@ -114,6 +114,8 @@ static inline uint16_t be_to_cpu_w(uint16_t w) {
 }
 
 cpu_t *get_bsp(void);
+void arch_cpu_user_access_begin(void);
+void arch_cpu_user_access_end(void);
 
 static inline thread_t *current_thread(void) {
 	thread_t *thread;

kernel-src/include/x86-64/arch/cpuid.h

@@ -20,6 +20,8 @@ typedef struct {
 #define CPUID_LEAF_0x80000001_EDX_SYSCALL (1 << 11)
 #define CPUID_LEAF_1_EDX_TSC (1 << 4)
 #define CPUID_LEAF_1_EDX_HTT (1 << 28)
+#define CPUID_LEAF_7_EBX_SMEP (1 << 7)
+#define CPUID_LEAF_7_EBX_SMAP (1 << 20)
 
 static inline void cpuid_with_ecx(uint32_t leaf, uint32_t ecx, cpuid_results_t *cpuid_results) {
 	asm volatile ("cpuid"

kernel-src/io/iovec.c

@@ -112,7 +112,7 @@ int iovec_iterator_copy_from_buffer(iovec_iterator_t *iovec_iterator, void *buff
 		size_t remaining_current = iovec_iterator->current->len - iovec_iterator->current_offset;
 		size_t copy_current = min(remaining_current, remaining_total);
 
-		error = USERCOPY_POSSIBLY_FROM_USER((void *)((uintptr_t)iovec_iterator->current->addr + iovec_iterator->current_offset), (void *)((uintptr_t)buffer + total_done), copy_current);
+		error = USERCOPY_POSSIBLY_TO_USER((void *)((uintptr_t)iovec_iterator->current->addr + iovec_iterator->current_offset), (void *)((uintptr_t)buffer + total_done), copy_current);
 		if (error)
 			break;
 

kernel-src/mm/usercopy.c

@@ -31,7 +31,9 @@ static void memcpyinternal(context_t *ctx, void *arg) {
 	// this context with CTX_RET set to EFAULT
 	current_thread()->usercopyctx = ctx;
 
+	arch_cpu_user_access_begin();
 	memcpy(desc->dst, desc->src, desc->size);
+	arch_cpu_user_access_end();
 
 	// no errors!
 	CTX_RET(ctx) = 0;
@@ -45,7 +47,9 @@ static void strleninternal(context_t *ctx, void *arg) {
 	// this context with CTX_RET set to EFAULT
 	current_thread()->usercopyctx = ctx;
 
+	arch_cpu_user_access_begin();
 	*desc->size = strlen(desc->str);
+	arch_cpu_user_access_end();
 
 	// no errors!
 	CTX_RET(ctx) = 0;
@@ -59,7 +63,9 @@ static void atomic32internal(context_t *ctx, void *arg) {
 	// this context with CTX_RET set to EFAULT
 	current_thread()->usercopyctx = ctx;
 
+	arch_cpu_user_access_begin();
 	__atomic_load(desc->address, desc->value, __ATOMIC_SEQ_CST);
+	arch_cpu_user_access_end();
 
 	// no errors!
 	CTX_RET(ctx) = 0;
@@ -73,7 +79,9 @@ static void memset_internal(context_t *ctx, void *arg) {
 	// this context with CTX_RET set to EFAULT
 	current_thread()->usercopyctx = ctx;
 
+	arch_cpu_user_access_begin();
 	memset(desc->dst, desc->byte, desc->size);
+	arch_cpu_user_access_end();
 
 	// no errors!
 	CTX_RET(ctx) = 0;

kernel-src/sys/elf.c

@@ -4,6 +4,8 @@
 #include <kernel/alloc.h>
 #include <util.h>
 #include <kernel/vmm.h>
+#include <kernel/usercopy.h>
+#include <arch/cpu.h>
 
 #ifdef	__x86_64__
 	#define ARCH_ELF_BITS 2
@@ -72,7 +74,10 @@ static int load(vnode_t *vnode, elfph64_t *ph) {
 		if (error)
 			return error;
 
-		memset(page, 0, firstpageoffset);
+		error = usercopy_memset(page, 0, firstpageoffset);
+		if (error)
+			return error;
+
 		mempos += firstpagecount;
 		msize -= firstpagecount;
 		fsize -= firstpagecount;
@@ -85,7 +90,10 @@ static int load(vnode_t *vnode, elfph64_t *ph) {
 			if (remainingmsize > ROUND_UP(mempos, PAGE_SIZE) - mempos)
 				remainingmsize = pagediff;
 
-			memset((void *)mempos, 0, remainingmsize);
+			error = usercopy_memset((void *)mempos, 0, remainingmsize);
+			if (error)
+				return error;
+
 			msize -= remainingmsize;
 			mempos += remainingmsize;
 		}
@@ -121,7 +129,10 @@ static int load(vnode_t *vnode, elfph64_t *ph) {
 		if (error)
 			return error;
 
-		memset((void *)(mempos + lastpagecount), 0, PAGE_SIZE - lastpagecount);
+		error = usercopy_memset((void *)(mempos + lastpagecount), 0, PAGE_SIZE - lastpagecount);
+		if (error)
+			return error;
+
 		msize -= msize > PAGE_SIZE ? PAGE_SIZE : msize;
 
 		error = vmm_changemmuflags((void *)mempos, PAGE_SIZE, mmuflags, 0);
@@ -266,6 +277,7 @@ void *elf_preparestack(void *top, auxv64list_t *auxv64, char **argv, char **envp
 	char **argstart = (char **)envstart - argc - 1;
 	size_t *argcptr = (size_t *)argstart - 1;
 
+	arch_cpu_user_access_begin();
 	auxv64->execfn.type = AT_EXECFN;
 	auxv64->execfn.val = (uint64_t)pathdatastart;
 
@@ -289,5 +301,6 @@ void *elf_preparestack(void *top, auxv64list_t *auxv64, char **argv, char **envp
 	*envstart = NULL;
 	*argcptr = argc;
 
+	arch_cpu_user_access_end();
 	return argcptr;
 }

kernel-src/sys/syscalls/sigreturn.c

@@ -29,7 +29,10 @@ __attribute__((noreturn)) void syscall_sigreturn(context_t *context) {
 
 	__assert(ARCH_CONTEXT_ISUSER(&sigframe->context));
 
+	// XXX TODO FIXME there is a bug here that needs this temporarily.
+	arch_cpu_user_access_begin();
 	arch_extracontext_copy(&current_thread()->extracontext, &sigframe->extracontext);
+	arch_cpu_user_access_end();
 	ARCH_CONTEXT_THREADLOAD(current_thread(), context);
 	arch_context_switch(&sigframe->context);