kernel: add xsave support

kernel-src/arch/x86-64/context.c

@@ -0,0 +1,130 @@
+#include <arch/context.h>
+#include <stdbool.h>
+#include <arch/cpuid.h>
+#include <kernel/slab.h>
+#include <logging.h>
+
+#define XCR0_X87 (1ull << 0)
+#define XCR0_SSE (1ull << 1)
+#define XCR0_AVX (1ull << 2)
+#define XCR0_AVX512 (7ull << 5)
+
+#define XCR0_MASK (XCR0_AVX512 | XCR0_AVX | XCR0_SSE | XCR0_X87)
+
+static enum {
+    FXSAVE,
+    XSAVE,
+    XSAVEOPT,
+} xsave_method;
+size_t arch_xsave_size;
+static uint64_t xcr0_value;
+static scache_t *xsave_cache;
+
+typedef struct {
+    uint16_t fcw;
+    uint16_t fsw;
+    uint8_t ftw;
+    uint8_t reserved1;
+    uint16_t fop;
+    uint64_t fip;
+    uint64_t fdp;
+    uint32_t mxcsr;
+    uint32_t mxcsr_mask;
+    struct {
+        uint64_t data[2];
+    } mm[8];
+    struct {
+        uint64_t data[2];
+    } xmm[16];
+    uint64_t reserved2[6];
+    uint64_t unused[6];
+} fxsave_area_t;
+
+void arch_extracontext_detect(void) {
+    if (!arch_xsave_size) {
+        if (cpuid_base_max_leaf() >= 0x0d) {
+            xsave_method = XSAVE;
+
+            cpuid_results_t results;
+            cpuid_with_ecx(0x0d, 0, &results);
+            xcr0_value = ((uint64_t)results.edx << 32) | results.eax;
+            xcr0_value &= XCR0_MASK;
+        } else {
+            xsave_method = FXSAVE;
+            arch_xsave_size = 512;
+        }
+    }
+
+    if (xsave_method == FXSAVE) return;
+
+    size_t cr4;
+    asm volatile("mov %%cr4, %0" : "=r" (cr4));
+    cr4 |= (1u << 18);
+    asm volatile("mov %0, %%cr4" :: "r" (cr4));
+
+    uint32_t low = xcr0_value;
+    uint32_t high = xcr0_value >> 32;
+    asm volatile("xsetbv" :: "a" (low), "d" (high), "c" (0u));
+
+    if (!arch_xsave_size) {
+        // after xsetbv ebx is updated to the size of the context for the enabled features
+        cpuid_results_t results;
+        cpuid_with_ecx(0x0d, 0, &results);
+        arch_xsave_size = results.ebx;
+
+        cpuid_with_ecx(0x0d, 1, &results);
+        if (results.eax & 1) xsave_method = XSAVEOPT;
+    }
+}
+
+int arch_extracontext_init(extracontext_t *context) {
+    if (!xsave_cache) {
+        xsave_cache = slab_newcache(arch_xsave_size, xsave_method == FXSAVE ? 16 : 64, NULL, NULL);
+        __assert(xsave_cache);
+    }
+
+    context->xsave = slab_allocate(xsave_cache);
+    if (!context->xsave) return 1;
+
+    // all sse exceptions masked
+    // initialise the x87 FPU state as it would be after the FNINIT instruction
+    memset(context->xsave, 0, arch_xsave_size);
+    fxsave_area_t *area = context->xsave;
+    area->mxcsr = 0x1f80;
+    area->fcw = 0x37f;
+
+    return 0;
+}
+
+void arch_extracontext_free(extracontext_t *context) {
+    slab_free(xsave_cache, context->xsave);
+}
+
+void arch_extracontext_save(extracontext_t *context) {
+    context->gsbase = rdmsr(MSR_KERNELGSBASE);
+    context->fsbase = rdmsr(MSR_FSBASE);
+
+    switch (xsave_method) {
+        case FXSAVE: asm("fxsaveq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave) : "memory"); break;
+        case XSAVE: asm("xsaveq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
+        case XSAVEOPT: asm("xsaveoptq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
+    }
+}
+
+void arch_extracontext_load(extracontext_t *context) {
+    wrmsr(MSR_KERNELGSBASE, context->gsbase);
+    wrmsr(MSR_FSBASE, context->fsbase);
+
+    switch (xsave_method) {
+        case FXSAVE: asm("fxrstorq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave) : "memory"); break;
+        case XSAVE:
+        case XSAVEOPT: asm("xrstorq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
+    }
+}
+
+void arch_extracontext_copy(extracontext_t *dst, const extracontext_t *src) {
+    // XXX is this behavior correct? ucontext doesnt seem to save it on linux and so it will not be saved
+    dst->gsbase = src->gsbase;
+    dst->fsbase = src->fsbase;
+    memcpy(dst->xsave, src->xsave, arch_xsave_size);
+}

kernel-src/arch/x86-64/cpu.c

@@ -8,6 +8,7 @@
 #include <arch/idt.h>
 #include <arch/gdt.h>
 #include <kernel/dpc.h>
+#include <arch/context.h>
 
 #define EFER_SYSCALLENABLE 1
 
@@ -251,6 +252,7 @@ static void arch_early(void) {
 	arch_idt_reload();
 	interrupt_register(0xfd, (void *)infinite_loop, NULL, IPL_IGNORE);
 	dpc_init();
+	arch_extracontext_detect();
 }
 
 INIT_ROUTINE_DEFINE(arch_early, INIT_ROUTINE_FLAGS_NONE, arch_early)

kernel-src/arch/x86-64/smp.c

@@ -8,6 +8,7 @@
 #include <arch/smp.h>
 #include <kernel/alloc.h>
 #include <kernel/init.h>
+#include <arch/context.h>
 
 DEFINE_KERNEL_ARGUMENT(nosmp, bool);
 
@@ -35,6 +36,7 @@ static void cpuwakeup(struct limine_smp_info *info) {
 	arch_idt_reload();
 	interrupt_register(0xfd, (void *)cpuwakeuphalt, NULL, IPL_IGNORE);
 	dpc_init();
+	arch_extracontext_detect();
 	arch_mmu_apswitch();
 	vmm_apinit();
 

kernel-src/include/x86-64/arch/context.h

@@ -41,10 +41,11 @@ typedef struct {
 typedef struct {
 	uint64_t gsbase;
 	uint64_t fsbase;
-	__attribute__((aligned(16))) uint8_t fx[512];
-	uint32_t mxcsr;
+	void *xsave;
 } extracontext_t;
 
+extern size_t arch_xsave_size;
+
 #define CTX_INIT(x,u,interrupts) \
 	if (u) { \
 		(x)->cs = 0x23; \
@@ -55,15 +56,6 @@ typedef struct {
 	} \
 	(x)->rflags = interrupts ? 0x200 : 0;
 
-// all sse exceptions masked
-// initialise the x87 FPU state as it would be after the FNINIT instruction
-#define CTX_XINIT(x, u) {\
-	(x)->mxcsr = 0x1f80; \
-	*((uint16_t *)&(x)->fx[0]) = 0x37f; \
-	*((uint16_t *)&(x)->fx[2]) = 0; \
-	(x)->fx[4] = 0; \
-	}
-
 #define CTX_SP(x) (x)->rsp
 #define CTX_IP(x) (x)->rip
 #define CTX_RET(x) (x)->rax
@@ -81,43 +73,36 @@ typedef struct {
 void arch_context_switch(context_t *context);
 int arch_context_saveandcall(void (*fn)(context_t *context, void *argument), void *stack, void *argument);
 
+void arch_extracontext_detect(void);
+
+int arch_extracontext_init(extracontext_t *context);
+void arch_extracontext_free(extracontext_t *context);
+void arch_extracontext_save(extracontext_t *context);
+void arch_extracontext_load(extracontext_t *context);
+void arch_extracontext_copy(extracontext_t *dst, const extracontext_t *src);
+
 #include <string.h>
 
 // kernelgsbase is set as user because they'll be swapped in the context switch
 #define ARCH_CONTEXT_SWITCHTHREAD(x) \
 	current_cpu()->ist.rsp0 = (uint64_t)(x)->kernelstacktop; \
-	wrmsr(MSR_KERNELGSBASE, (x)->extracontext.gsbase); \
-	wrmsr(MSR_FSBASE, (x)->extracontext.fsbase); \
-	asm("fxrstor (%%rax)" : : "a"(&(x)->extracontext.fx[0])); \
-	asm("ldmxcsr (%%rax)" : : "a"(&(x)->extracontext.mxcsr)); \
+	arch_extracontext_load(&(x)->extracontext); \
 	arch_context_switch(&thread->context);
 
 #define ARCH_CONTEXT_THREADSAVE(t, c) \
 	memcpy(&(t)->context, c, sizeof(context_t)); \
-	(t)->extracontext.gsbase = rdmsr(MSR_KERNELGSBASE); \
-	(t)->extracontext.fsbase = rdmsr(MSR_FSBASE); \
-	asm("fxsave (%%rax)" : : "a"(&(t)->extracontext.fx[0])); \
-	asm("stmxcsr (%%rax)" : : "a"(&(t)->extracontext.mxcsr));
+	arch_extracontext_save(&(t)->extracontext);
 
 #define ARCH_CONTEXT_THREADLOAD(t, c) \
 	memcpy(c, &(t)->context, sizeof(context_t)); \
 	current_cpu()->ist.rsp0 = (uint64_t)(t)->kernelstacktop; \
-	wrmsr(MSR_KERNELGSBASE, (t)->extracontext.gsbase); \
-	wrmsr(MSR_FSBASE, (t)->extracontext.fsbase); \
-	asm("fxrstor (%%rax)" : : "a"(&(t)->extracontext.fx[0])); \
-	asm("ldmxcsr (%%rax)" : : "a"(&(t)->extracontext.mxcsr));
+	arch_extracontext_load(&(t)->extracontext);
 
 #define ARCH_EXTRACONTEXT_LOAD(c) \
-	wrmsr(MSR_KERNELGSBASE, (c)->gsbase); \
-	wrmsr(MSR_FSBASE, (c)->fsbase); \
-	asm("fxrstor (%%rax)" : : "a"(&(c)->fx[0])); \
-	asm("ldmxcsr (%%rax)" : : "a"(&(c)->mxcsr));
+	arch_extracontext_load(c);
 
 #define ARCH_EXTRACONTEXT_SAVE(c) \
-	(c)->gsbase = rdmsr(MSR_KERNELGSBASE); \
-	(c)->fsbase = rdmsr(MSR_FSBASE); \
-	asm("fxsave (%%rax)" : : "a"(&(c)->fx[0])); \
-	asm("stmxcsr (%%rax)" : : "a"(&(c)->mxcsr));
+	arch_extracontext_save(c);
 
 #define ARCH_CONTEXT_INTSTATUS(x) ((x)->rflags & 0x200 ? true : false)
 #define ARCH_CONTEXT_ISUSER(x) ((x)->cs == 0x23)

kernel-src/include/x86-64/arch/signal.h

@@ -56,9 +56,15 @@ typedef struct {
 	context_t context;
 	extracontext_t extracontext;
 	siginfo_t siginfo;
+	uint8_t xsave[];
 } sigframe_t;
 
 #define ARCH_SIGFRAME_GET_UCONTEXT_POINTER(x) (&(x)->uc_flags)
+#define ARCH_SIGFRAME_SIZE (sizeof(sigframe_t) + arch_xsave_size)
+
+static inline void arch_sigframe_init(sigframe_t *sigframe) {
+	sigframe->extracontext.xsave = sigframe->xsave;
+}
 
 static inline void arch_sigframe_prepare_mcontext(void *stack, sigframe_t *sigframe) {
 	context_t *context = &sigframe->context;
@@ -89,7 +95,8 @@ static inline void arch_sigframe_prepare_mcontext(void *stack, sigframe_t *sigfr
 	gregs[MCONTEXT_REG_OLDMASK] = 0;
 	gregs[MCONTEXT_REG_CR2] = context->cr2;
 
-	sigframe->mcontext.fpu_state_p = &((sigframe_t *)stack)->extracontext.fx;
+	extracontext->xsave = ((sigframe_t *)stack)->xsave;
+	sigframe->mcontext.fpu_state_p = extracontext->xsave;
 }
 
 static inline void arch_sigframe_get_context_from_mcontext(sigframe_t *sigframe) {

kernel-src/sys/signal.c

@@ -540,7 +540,7 @@ bool signal_check(struct thread_t *thread, context_t *context, bool syscall, uin
 		// get where in memory to put the frame
 		#if ARCH_SIGNAL_STACK_GROWS_DOWNWARDS == 1
 		void *stack = altstack ? (void *)((uintptr_t)altstack + thread->signals.stack.size) : (void *)CTX_SP(context);
-		stack = (void *)((((uintptr_t)stack - ARCH_SIGNAL_REDZONE_SIZE - sizeof(sigframe_t)) & ~0xfl) - 0x8);
+		stack = (void *)((((uintptr_t)stack - ARCH_SIGNAL_REDZONE_SIZE - ARCH_SIGFRAME_SIZE) & ~0xfl) - 0x8);
 		#else
 			#error unsupported
 		#endif
@@ -553,35 +553,36 @@ bool signal_check(struct thread_t *thread, context_t *context, bool syscall, uin
 		}
 
 		// configure stack frame
-		sigframe_t sigframe;
-		sigframe.restorer = action->restorer;
+		sigframe_t *sigframe = __builtin_alloca(ARCH_SIGFRAME_SIZE);
+		arch_sigframe_init(sigframe);
+		sigframe->restorer = action->restorer;
 		if (altstack) {
-			memcpy(&sigframe.oldstack, &thread->signals.stack, sizeof(stack_t));
+			memcpy(&sigframe->oldstack, &thread->signals.stack, sizeof(stack_t));
 			memset(&thread->signals.stack, 0, sizeof(stack_t));
 		} else {
-			memset(&sigframe.oldstack, 0, sizeof(stack_t));
+			memset(&sigframe->oldstack, 0, sizeof(stack_t));
 		}
 
 		if (thread->signals.hasreturnmask) {
 			// if we were waiting in signal_suspend, push the old mask into the return frame
 			// of the first signal and set it to not act this way the next check
-			memcpy(&sigframe.oldmask, &thread->signals.returnmask, sizeof(sigset_t));
+			memcpy(&sigframe->oldmask, &thread->signals.returnmask, sizeof(sigset_t));
 			thread->signals.hasreturnmask = false;
 		} else {
 			// else, just push the current mask
-			memcpy(&sigframe.oldmask, &thread->signals.mask, sizeof(sigset_t));
+			memcpy(&sigframe->oldmask, &thread->signals.mask, sizeof(sigset_t));
 		}
-		memcpy(&sigframe.context, context, sizeof(context_t));
-		memcpy(&sigframe.extracontext, &thread->extracontext, sizeof(extracontext_t));
+		memcpy(&sigframe->context, context, sizeof(context_t));
+		arch_extracontext_copy(&sigframe->extracontext, &thread->extracontext);
 
-		arch_sigframe_prepare_mcontext(stack, &sigframe);
+		arch_sigframe_prepare_mcontext(stack, sigframe);
 
-		memset(&sigframe.siginfo, 0, sizeof(siginfo_t));
+		memset(&sigframe->siginfo, 0, sizeof(siginfo_t));
 		if (signal == SIGSEGV)
-			sigframe.siginfo.__si_fields.__sigfault.si_addr = CTX_TRAP_ADDR(context);
+			sigframe->siginfo.__si_fields.__sigfault.si_addr = CTX_TRAP_ADDR(context);
 
-		if (usercopy_touser(stack, &sigframe, sizeof(sigframe_t))) {
-			printf("signal: bad user stack %p (altstack %p) handling signal %d trapno %lu\n", stack, altstack, signal, sigframe.mcontext.gregs[MCONTEXT_REG_TRAPNO]);
+		if (usercopy_touser(stack, sigframe, ARCH_SIGFRAME_SIZE)) {
+			printf("signal: bad user stack %p (altstack %p) handling signal %d trapno %lu\n", stack, altstack, signal, sigframe->mcontext.gregs[MCONTEXT_REG_TRAPNO]);
 			THREAD_LEAVE(thread);
 			PROCESS_LEAVE(proc);
 			interrupt_set(true);

kernel-src/sys/syscalls/sigreturn.c

@@ -9,31 +9,26 @@
 
 // this expects CTX_SP to be pointing to the signal frame
 __attribute__((noreturn)) void syscall_sigreturn(context_t *context) {
-	sigframe_t sigframe;
-	int error = usercopy_fromuser(&sigframe, (void *)CTX_SP(context), sizeof(sigframe_t));
-	if (error || ARCH_CONTEXT_ISUSER(&sigframe.context) == false) {
+	sigframe_t *sigframe = __builtin_alloca(ARCH_SIGFRAME_SIZE);
+	int error = usercopy_fromuser(sigframe, (void *)CTX_SP(context), ARCH_SIGFRAME_SIZE);
+	if (error || ARCH_CONTEXT_ISUSER(&sigframe->context) == false) {
 		printf("syscall_sigreturn: bad return stack or bad return information\n");
 		proc_terminate(SIGSEGV);
 	}
 
 	interrupt_set(false);
-	signal_altstack(current_thread(), &sigframe.oldstack, NULL);
-	signal_changemask(current_thread(), SIG_SETMASK, &sigframe.oldmask, NULL);
+	signal_altstack(current_thread(), &sigframe->oldstack, NULL);
+	signal_changemask(current_thread(), SIG_SETMASK, &sigframe->oldmask, NULL);
 
-	arch_sigframe_get_context_from_mcontext(&sigframe);
+	arch_sigframe_get_context_from_mcontext(sigframe);
 
-	__assert(ARCH_CONTEXT_ISUSER(&sigframe.context));
+	__assert(ARCH_CONTEXT_ISUSER(&sigframe->context));
 
-	#ifdef __x86_64__
-	// XXX is this behavior correct? ucontext doesnt seem to save it on linux and so it will not be saved
-	sigframe.extracontext.gsbase = rdmsr(MSR_KERNELGSBASE);
-	sigframe.extracontext.fsbase = rdmsr(MSR_FSBASE);
-	#endif
-	memcpy(&current_thread()->extracontext, &sigframe.extracontext, sizeof(extracontext_t));
+	arch_extracontext_copy(&current_thread()->extracontext, &sigframe->extracontext);
 	ARCH_CONTEXT_THREADLOAD(current_thread(), context);
 
 	interrupt_set(true);
-	arch_context_switch(&sigframe.context);
+	arch_context_switch(&sigframe->context);
 
 	// if we were to return normally, CTX_RET and CTX_ERRNO would be corrupted,
 	// so we will manually call arch_context_switch, which will jump to the right userland code

kernel-src/sys/thread.c

@@ -2,6 +2,7 @@
 #include <kernel/proc.h>
 #include <kernel/slab.h>
 #include <logging.h>
+#include <arch/context.h>
 
 static scache_t *thread_cache;
 
@@ -17,8 +18,14 @@ thread_t *sched_newthread(void *ip, size_t kstacksize, int nice, proc_t *proc, v
 
 	memset(thread, 0, sizeof(thread_t));
 
+	if (arch_extracontext_init(&thread->extracontext)) {
+		slab_free(thread_cache, thread);
+		return NULL;
+	}
+
 	thread->kernelstack = vmm_map(NULL, kstacksize, VMM_FLAGS_ALLOCATE, ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_NOEXEC, NULL);
 	if (thread->kernelstack == NULL) {
+		arch_extracontext_free(&thread->extracontext);
 		slab_free(thread_cache, thread);
 		return NULL;
 	}
@@ -37,7 +44,6 @@ thread_t *sched_newthread(void *ip, size_t kstacksize, int nice, proc_t *proc, v
 	}
 
 	CTX_INIT(&thread->context, proc != NULL, true);
-	CTX_XINIT(&thread->extracontext, proc != NULL);
 	CTX_SP(&thread->context) = proc ? (ctxreg_t)ustack : (ctxreg_t)thread->kernelstacktop;
 	CTX_IP(&thread->context) = (ctxreg_t)ip;
 	SPINLOCK_INIT(thread->sleeplock);
@@ -52,6 +58,7 @@ thread_t *sched_newthread(void *ip, size_t kstacksize, int nice, proc_t *proc, v
 
 void sched_destroythread(thread_t *thread) {
 	vmm_unmap(thread->kernelstack, thread->kernelstacksize, 0);
+	arch_extracontext_free(&thread->extracontext);
 	slab_free(thread_cache, thread);
 }