kernel: ldt support

kernel-src/arch/x86-64/archctl.c

@@ -8,11 +8,17 @@
 #define ARCH_CTL_GET_GSBASE 2
 #define ARCH_CTL_GET_FSBASE 3
 #define ARCH_CTL_SET_SYSTRACE 4
+#define ARCH_CTL_SET_LDT_ENTRY 5
 
 #define ARCH_CTL_SYSTRACE_OFF 0
 #define ARCH_CTL_SYSTRACE_SELF 1
 #define ARCH_CTL_SYSTRACE_ALL 2
 
+typedef struct {
+	uint64_t entry;
+	uint16_t seg;
+} archctl_ldt_request_t;
+
 static inline int copy_u64_to_user(void *ptr, uint64_t v) {
 	return usercopy_touser(ptr, &v, sizeof(v));
 }
@@ -25,9 +31,11 @@ syscallret_t syscall_archctl(context_t *context, int func, void *arg) {
 
 	if (!IS_USER_ADDRESS(arg)) {
 		ret.errno = EFAULT;
+		ret.ret = -1;
 		return ret;
 	}
 
+	uint64_t scratch;
 	switch (func) {
 		case ARCH_CTL_SET_GSBASE:
 			// kernelgsbase because it will be switched out to user
@@ -42,6 +50,14 @@ syscallret_t syscall_archctl(context_t *context, int func, void *arg) {
 		case ARCH_CTL_GET_FSBASE:
 			ret.errno = copy_u64_to_user(arg, rdmsr(MSR_FSBASE));
 			break;
+		case ARCH_CTL_SET_LDT_ENTRY:
+			archctl_ldt_request_t req;
+			ret.errno = usercopy_fromuser(&req, arg, sizeof(req));
+			if (ret.errno)
+				break;
+
+			ret.errno = arch_ldt_set_entry(req.seg, req.entry);
+			break;
 		case ARCH_CTL_SET_SYSTRACE:
 #ifdef SYSCALL_LOGGING
 			switch ((uint64_t)arg) {
@@ -69,5 +85,6 @@ syscallret_t syscall_archctl(context_t *context, int func, void *arg) {
 			break;
 	}
 
+	ret.ret = ret.errno ? -1 : 0;
 	return ret;
 }

kernel-src/arch/x86-64/context.asm

@@ -32,8 +32,8 @@ arch_context_switch:
 	add rsp,8 ; remove error code
 
 	; check if swapgs is needed
-	cmp qword [rsp+8], 0x23
-	jne .notneeded2
+	cmp qword [rsp+8], 0x8
+	je .notneeded2
 	swapgs
 	.notneeded2:
 	o64 iret

kernel-src/arch/x86-64/context.c

@@ -3,6 +3,7 @@
 #include <arch/cpuid.h>
 #include <kernel/slab.h>
 #include <logging.h>
+#include <arch/gdt.h>
 
 #define XCR0_X87 (1ull << 0)
 #define XCR0_SSE (1ull << 1)
@@ -12,120 +13,135 @@
 #define XCR0_MASK (XCR0_AVX512 | XCR0_AVX | XCR0_SSE | XCR0_X87)
 
 static enum {
-    FXSAVE,
-    XSAVE,
-    XSAVEOPT,
+	FXSAVE,
+	XSAVE,
+	XSAVEOPT,
 } xsave_method;
 size_t arch_xsave_size;
 static uint64_t xcr0_value;
 static scache_t *xsave_cache;
 
 typedef struct {
-    uint16_t fcw;
-    uint16_t fsw;
-    uint8_t ftw;
-    uint8_t reserved1;
-    uint16_t fop;
-    uint64_t fip;
-    uint64_t fdp;
-    uint32_t mxcsr;
-    uint32_t mxcsr_mask;
-    struct {
-        uint64_t data[2];
-    } mm[8];
-    struct {
-        uint64_t data[2];
-    } xmm[16];
-    uint64_t reserved2[6];
-    uint64_t unused[6];
+	uint16_t fcw;
+	uint16_t fsw;
+	uint8_t ftw;
+	uint8_t reserved1;
+	uint16_t fop;
+	uint64_t fip;
+	uint64_t fdp;
+	uint32_t mxcsr;
+	uint32_t mxcsr_mask;
+	struct {
+		uint64_t data[2];
+	} mm[8];
+	struct {
+		uint64_t data[2];
+	} xmm[16];
+	uint64_t reserved2[6];
+	uint64_t unused[6];
 } fxsave_area_t;
 
 void arch_extracontext_detect(void) {
-    if (!arch_xsave_size) {
-        cpuid_results_t results;
-	cpuid(1, &results);
-        if (cpuid_base_max_leaf() >= 0x0d && (results.ecx & (1 << 26))) {
-            xsave_method = XSAVE;
+	if (!arch_xsave_size) {
+		cpuid_results_t results;
+		cpuid(1, &results);
+		if (cpuid_base_max_leaf() >= 0x0d && (results.ecx & (1 << 26))) {
+			xsave_method = XSAVE;
 
-            cpuid_with_ecx(0x0d, 0, &results);
-            xcr0_value = ((uint64_t)results.edx << 32) | results.eax;
-            xcr0_value &= XCR0_MASK;
-        } else {
-            xsave_method = FXSAVE;
-            arch_xsave_size = 512;
-        }
-    }
+			cpuid_with_ecx(0x0d, 0, &results);
+			xcr0_value = ((uint64_t)results.edx << 32) | results.eax;
+			xcr0_value &= XCR0_MASK;
+		} else {
+			xsave_method = FXSAVE;
+			arch_xsave_size = 512;
+		}
+	}
 
-    if (xsave_method == FXSAVE) return;
+	if (xsave_method == FXSAVE) return;
 
-    size_t cr4;
-    asm volatile("mov %%cr4, %0" : "=r" (cr4));
-    cr4 |= (1u << 18);
-    asm volatile("mov %0, %%cr4" :: "r" (cr4));
+	size_t cr4;
+	asm volatile("mov %%cr4, %0" : "=r" (cr4));
+	cr4 |= (1u << 18);
+	asm volatile("mov %0, %%cr4" :: "r" (cr4));
 
-    uint32_t low = xcr0_value;
-    uint32_t high = xcr0_value >> 32;
-    asm volatile("xsetbv" :: "a" (low), "d" (high), "c" (0u));
+	uint32_t low = xcr0_value;
+	uint32_t high = xcr0_value >> 32;
+	asm volatile("xsetbv" :: "a" (low), "d" (high), "c" (0u));
 
-    if (!arch_xsave_size) {
-        // after xsetbv ebx is updated to the size of the context for the enabled features
-        cpuid_results_t results;
-        cpuid_with_ecx(0x0d, 0, &results);
-        arch_xsave_size = results.ebx;
+	if (!arch_xsave_size) {
+		// after xsetbv ebx is updated to the size of the context for the enabled features
+		cpuid_results_t results;
+		cpuid_with_ecx(0x0d, 0, &results);
+		arch_xsave_size = results.ebx;
 
-        cpuid_with_ecx(0x0d, 1, &results);
-        if (results.eax & 1) xsave_method = XSAVEOPT;
-    }
+		cpuid_with_ecx(0x0d, 1, &results);
+		if (results.eax & 1) xsave_method = XSAVEOPT;
+	}
 }
 
 int arch_extracontext_init(extracontext_t *context) {
-    if (!xsave_cache) {
-        xsave_cache = slab_newcache(arch_xsave_size, xsave_method == FXSAVE ? 16 : 64, NULL, NULL);
-        __assert(xsave_cache);
-    }
+	if (!xsave_cache) {
+		xsave_cache = slab_newcache(arch_xsave_size, xsave_method == FXSAVE ? 16 : 64, NULL, NULL);
+		__assert(xsave_cache);
+	}
 
-    context->xsave = slab_allocate(xsave_cache);
-    if (!context->xsave) return 1;
+	context->xsave = slab_allocate(xsave_cache);
+	if (!context->xsave) return 1;
 
-    // all sse exceptions masked
-    // initialise the x87 FPU state as it would be after the FNINIT instruction
-    memset(context->xsave, 0, arch_xsave_size);
-    fxsave_area_t *area = context->xsave;
-    area->mxcsr = 0x1f80;
-    area->fcw = 0x37f;
+	// all sse exceptions masked
+	// initialise the x87 FPU state as it would be after the FNINIT instruction
+	memset(context->xsave, 0, arch_xsave_size);
+	fxsave_area_t *area = context->xsave;
+	area->mxcsr = 0x1f80;
+	area->fcw = 0x37f;
 
-    return 0;
+	return 0;
 }
 
 void arch_extracontext_free(extracontext_t *context) {
-    slab_free(xsave_cache, context->xsave);
+	slab_free(xsave_cache, context->xsave);
 }
 
 void arch_extracontext_save(extracontext_t *context) {
-    context->gsbase = rdmsr(MSR_KERNELGSBASE);
-    context->fsbase = rdmsr(MSR_FSBASE);
+	context->gsbase = rdmsr(MSR_KERNELGSBASE);
+	context->fsbase = rdmsr(MSR_FSBASE);
 
-    switch (xsave_method) {
-        case FXSAVE: asm("fxsaveq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave) : "memory"); break;
-        case XSAVE: asm("xsaveq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
-        case XSAVEOPT: asm("xsaveoptq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
-    }
+	asm volatile("mov %%gs, %0" : "=r"(context->gs));
+	asm volatile("mov %%fs, %0" : "=r"(context->fs));
+
+	switch (xsave_method) {
+		case FXSAVE: asm("fxsaveq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave) : "memory"); break;
+		case XSAVE: asm("xsaveq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
+		case XSAVEOPT: asm("xsaveoptq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
+	}
 }
 
 void arch_extracontext_load(extracontext_t *context) {
-    wrmsr(MSR_KERNELGSBASE, context->gsbase);
-    wrmsr(MSR_FSBASE, context->fsbase);
+	if (current_thread()->proc && current_thread()->proc->ldt)
+		arch_gdt_set_ldt(current_thread()->proc->ldt, 0xffff);
+	else
+		arch_gdt_set_ldt(NULL, 0);
 
-    switch (xsave_method) {
-        case FXSAVE: asm("fxrstorq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave) : "memory"); break;
-        case XSAVE:
-        case XSAVEOPT: asm("xrstorq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
-    }
+	void *old_base = current_cpu();
+	if (!load_seg_gs(context->gs)) load_seg_gs(0);
+	wrmsr(MSR_GSBASE, (uint64_t)old_base);
+	wrmsr(MSR_KERNELGSBASE, context->gsbase);
+
+	if (!load_seg_fs(context->fs)) load_seg_fs(0);	
+	wrmsr(MSR_FSBASE, context->fsbase);
+
+	switch (xsave_method) {
+		case FXSAVE: asm("fxrstorq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave) : "memory"); break;
+		case XSAVE:
+		case XSAVEOPT: asm("xrstorq %0" ::"m"(*(char(*)[arch_xsave_size])context->xsave), "d"(-1), "a"(-1) : "memory"); break;
+	}
 }
 
 void arch_extracontext_copy(extracontext_t *dst, const extracontext_t *src) {
-    // XXX is this behavior correct? ucontext doesnt seem to save it on linux and so it will not be saved
-    dst->gsbase = src->gsbase;
-    dst->fsbase = src->fsbase;
-    memcpy(dst->xsave, src->xsave, arch_xsave_size);
+	// XXX is this behavior correct? ucontext doesnt seem to save it on linux and so it will not be saved
+	dst->gsbase = src->gsbase;
+	dst->fsbase = src->fsbase;
+	dst->fs = dst->fs;
+	dst->gs = dst->gs;
+	memcpy(dst->xsave, src->xsave, arch_xsave_size);
 }

kernel-src/arch/x86-64/cpu.c

@@ -46,6 +46,13 @@ static void x87isr(isr_t *self, context_t *ctx) {
 	}
 }
 
+static void reload_ldt(isr_t *self, context_t *ctx) {
+	if (current_thread() == NULL || current_thread()->proc == NULL || current_thread()->proc->ldt == NULL)
+		return;
+
+	arch_ldt_invalidate();
+}
+
 void arch_nmi_isr(context_t *ctx) {
 	// we are in an NMI context, running on IST0 
 	// first load the proper gsbase to get access to cpu-local variables. 
@@ -219,6 +226,9 @@ void arch_cpu_init() {
 	interrupt_register(16, x87isr, NULL, IPL_IGNORE);
 	interrupt_register(19, simdisr, NULL, IPL_IGNORE);
 
+	// and the ldt reload one
+	interrupt_register(0xfc, (void *)reload_ldt, NULL, IPL_MAX);
+
 	// get vendor string for cpu and max cpuid eax
 	__get_cpuid(0, &eax, &ebx, &ecx, &edx);
 	int *vendor_ptr = (int *)current_cpu()->vendor;

kernel-src/arch/x86-64/desc.asm

@@ -0,0 +1,23 @@
+; segment is in rdi
+; returns 1 for success or 0 for failure
+load_seg_gs:
+global load_seg_gs
+	xor rax, rax
+	mov gs, rdi
+	inc rax
+	ret
+load_seg_gs_fallback:
+global load_seg_gs_fallback
+	ret
+
+; segment is in rdi
+; returns 1 for success or 0 for failure
+load_seg_fs:
+global load_seg_fs
+	xor rax, rax
+	mov fs, rdi
+	inc rax
+	ret
+load_seg_fs_fallback:
+global load_seg_fs_fallback
+	ret

kernel-src/arch/x86-64/gdt.c

@@ -5,16 +5,36 @@
 #include <string.h>
 #include <printf.h>
 
-static uint64_t template[7] = {
+static uint64_t template[9] = {
 	0, // NULL 0x0
 	0x00af9b000000ffff, // code64 0x8
 	0x00af93000000ffff, // data64 0x10
-	0x00aff3000000ffff, // udata64 0x18
+	0x00eff3000000ffff, // udata64 0x18
 	0x00affb000000ffff, // ucode64 0x20
 	0x0020890000000000, // low ist 0x28
-	0x0000000000000000  // high ist
+	0x0000000000000000, // high ist
+	0x0000000000000000, // LDT low
+	0x0000000000000000  // LDT high
 };
 
+// should be called with preemption DISABLED
+void arch_gdt_set_ldt(void *base, size_t limit) {
+	if (base == NULL) {
+		current_cpu()->gdt[7] = 0;
+		current_cpu()->gdt[8] = 0;
+		asm volatile("lldt %%ax" : : "a"(0));
+		return;
+	}
+
+	uint64_t base_low = (uint64_t)base & 0xffffffff;
+	current_cpu()->gdt[7] = 0x82000000ffff; // present, ldt, limit 64k
+	current_cpu()->gdt[7] |= (base_low & 0xffff) << 16;
+	current_cpu()->gdt[7] |= ((base_low >> 16) & 0xff) << 32;
+	current_cpu()->gdt[7] |= ((base_low >> 24) & 0xff) << 56;
+	current_cpu()->gdt[8] = ((uint64_t)base >> 32) & 0xffffffff;
+	asm volatile("lldt %%ax" : : "a"(0x38));
+}
+
 typedef struct {
 	uint16_t size;
 	uint64_t address;

kernel-src/arch/x86-64/isr.asm

@@ -17,8 +17,8 @@ isr_%+ i:
 
 isr_common:
 	; check if swapgs is needed
-	cmp qword [rsp+24], 0x23
-	jne .notneeded1
+	cmp qword [rsp+24], 0x8
+	je .notneeded1
 	swapgs
 	.notneeded1:
 	; push context
@@ -90,8 +90,8 @@ isr_common:
 	add rsp,8 ; remove error code
 
 	; check if swapgs is needed
-	cmp qword [rsp+8], 0x23
-	jne .notneeded2
+	cmp qword [rsp+8], 0x8
+	je .notneeded2
 	swapgs
 	.notneeded2:
 	o64 iret

kernel-src/arch/x86-64/ldt.c

@@ -0,0 +1,72 @@
+#include <arch/ldt.h>
+#include <arch/gdt.h>
+#include <kernel/alloc.h>
+#include <errno.h>
+#include <arch/cpu.h>
+#include <arch/smp.h>
+#include <spinlock.h>
+
+static spinlock_t ldt_invalidate_lock;
+static size_t done;
+
+void arch_ldt_invalidate(void) {
+	arch_gdt_set_ldt(current_thread()->proc->ldt, 0xffff);
+	__atomic_add_fetch(&done, 1, __ATOMIC_SEQ_CST);
+}
+
+int arch_ldt_set_entry(int which, ldt_entry_t entry) {
+	// is an ldt already loaded?
+	if (__atomic_load_n(&current_thread()->proc->ldt, __ATOMIC_RELAXED) == NULL) {
+		// allocate one and atomically set it
+		ldt_entry_t *ldt = alloc(sizeof(ldt_entry_t) * 8192);
+		if (ldt == NULL)
+			return ENOMEM;
+
+		ldt_entry_t *expected = NULL;
+
+		bool swap_ok = __atomic_compare_exchange_n(&current_thread()->proc->ldt, &expected, ldt, false, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED);
+		if (!swap_ok)
+			free(ldt);
+
+		bool do_shootdown = // do shootdown if
+		   	swap_ok // we actually need to
+			&& arch_smp_cpusawake >= 2 // there are multiple cpus in the system
+			&& current_thread()->proc->runningthreadcount > 1; // in a process which has multiple threads running
+
+		if (do_shootdown) {
+			long old_ipl = spinlock_acquire_raise_ipl(&ldt_invalidate_lock, IPL_DPC);
+			size_t shootdown_total = 1;
+			done = 0;
+
+			for (int i = 0; i < arch_smp_cpusawake; ++i) {
+				// reading the vmm context like this is racey but there are no ill side effects other than spurious shootdowns
+				// if a cpu changes to this vmmctx, they will already have done a ldt invalidation.
+				// if it changes out of it, same thing
+				if (smp_cpus[i] == current_cpu() || (smp_cpus[i]->vmmctx != current_thread()->vmmctx))
+					continue;
+
+				++shootdown_total;
+				arch_smp_send_ipi(smp_cpus[i], &smp_cpus[i]->isr[0xfc], ARCH_SMP_IPI_TARGET, false);
+			}
+
+			arch_ldt_invalidate();
+
+			while (__atomic_load_n(&done, __ATOMIC_SEQ_CST) != shootdown_total) asm("pause");
+
+			spinlock_release_lower_ipl(&ldt_invalidate_lock, old_ipl);
+		}
+	}
+
+	__atomic_store_n(&current_thread()->proc->ldt[which], entry, __ATOMIC_RELAXED);
+
+	return 0;
+}
+
+int arch_ldt_fork(ldt_entry_t **dst, ldt_entry_t *src) {
+	*dst = alloc(sizeof(ldt_entry_t) * 8192);
+	if (!*dst)
+		return ENOMEM;
+
+	memcpy(*dst, src, sizeof(ldt_entry_t) * 8192);
+	return 0;
+}

kernel-src/arch/x86-64/mmu.c

@@ -349,6 +349,16 @@ static void pfisr(isr_t *self, context_t *ctx) {
 }
 
 static void gpfisr(isr_t *self, context_t *ctx) {
+	if (ctx->rip >= (uintptr_t)load_seg_gs && ctx->rip <= (uintptr_t)load_seg_gs_fallback) {
+		ctx->rip = (uintptr_t)load_seg_gs_fallback;
+		return;
+	}
+
+	if (ctx->rip >= (uintptr_t)load_seg_fs && ctx->rip <= (uintptr_t)load_seg_fs_fallback) {
+		ctx->rip = (uintptr_t)load_seg_fs_fallback;
+		return;
+	}
+
 	thread_t *thread = current_thread();
 	if (thread && thread->usercopyctx) {
 		memcpy(ctx, thread->usercopyctx, sizeof(context_t));