kernel: profiling infrastructure

kernel-src/arch/x86-64/apic.c

@@ -129,6 +129,10 @@ static void writeiored(void *ioapic, uint8_t entry, uint8_t vector, uint8_t deli
 	writeioapic(ioapic, 0x11 + entry * 2, (uint32_t)dest << 24);
 }
 
+void arch_apic_init_perf(void) {
+	writelapic(APIC_LVT_PERFORMANCE, LVT_DELIVERY_NMI);
+}
+
 void arch_ioapic_setirq(uint8_t irq, uint8_t vector, uint8_t proc, bool masked) {
 	// default settings for ISA irqs
 	uint8_t polarity = 1; // active high

kernel-src/arch/x86-64/arch_prof.c

@@ -0,0 +1,144 @@
+#include <kernel/init.h>
+#include <string.h>
+#include <arch/cpu.h>
+#include <arch/cpuid.h>
+#include <kernel/prof.h>
+#include <logging.h>
+
+#ifdef ENABLE_PROFILING
+
+#define INTEL_TSC_INDEX 2
+#define INTEL_TSC_INDEX_OFFSET (1lu << 34)
+#define INTEL_TSC_FIXED_CTR_CTRL_OFFSET 8
+#define INTEL_TSC_FIXED_CTR_CTRL_ENABLE (1 << (11))
+#define INTEL_PERF_FIXED_CTR_CTRL_USER 1
+#define INTEL_PERF_FIXED_CTR_CTRL_SUPERVISOR 2
+
+#define TICK_COUNT 3200000
+
+static int intel_get_max_size(void) {
+	cpuid_results_t cpuid_results;
+	cpuid(0xa, &cpuid_results);
+	return 1 << ((cpuid_results.eax >> 16) & 0xff);
+}
+
+static int intel_get_version(void) {
+	cpuid_results_t cpuid_results;
+	cpuid(0xa, &cpuid_results);
+	return cpuid_results.eax & 0xff;
+}
+
+static bool intel_check_tsc_fixed(void) {
+	cpuid_results_t cpuid_results;
+	cpuid(0xa, &cpuid_results);
+
+	int fixed_count = cpuid_results.edx & 0x1f;
+	return fixed_count > INTEL_TSC_INDEX;
+}
+
+static void reset_intel_pmc(void) {
+	// disable PMCs
+	wrmsr(MSR_PERF_FIXED_CTR_CTRL, 0);
+
+	// clear overflow
+	wrmsr(MSR_PERF_GLOBAL_OVF_CTRL, INTEL_TSC_INDEX_OFFSET);
+
+	// program PMC
+	wrmsr(MSR_PERF_FIXED_CTR2, intel_get_max_size() - TICK_COUNT);
+
+	// enable the PMC
+	wrmsr(MSR_PERF_FIXED_CTR_CTRL,
+		((INTEL_PERF_FIXED_CTR_CTRL_USER | INTEL_PERF_FIXED_CTR_CTRL_SUPERVISOR) << INTEL_TSC_FIXED_CTR_CTRL_OFFSET) |
+		INTEL_TSC_FIXED_CTR_CTRL_ENABLE);
+}
+
+static void init_intel_pmc(void) {
+	// check for version 2 PMC
+	if (intel_get_version() < 2) {
+		__assert(!"Profiling is not supported on Intel CPUs without at least a version 2 PMC. Recompile without profiling.");
+	}
+
+	// check for support for the tsc fixed counter
+	__assert(intel_check_tsc_fixed());
+
+	// enable fixed PMC
+	wrmsr(MSR_PERF_GLOBAL_CTRL, rdmsr(MSR_PERF_GLOBAL_CTRL) | INTEL_TSC_INDEX_OFFSET);
+
+	reset_intel_pmc();
+	arch_apic_init_perf();
+}
+
+static uint8_t get_backtrace(context_t *context, uintptr_t *data) {
+	if (IS_USER_ADDRESS(context->rip))
+		return 0;
+
+	uint64_t *rbp = (uint64_t *)context->rbp;
+	uint8_t done = 0;
+
+	for (;;) {
+		if (IS_USER_ADDRESS(rbp) || done == 255)
+			break;
+
+		uint64_t rip = *(rbp + 1);
+		if (IS_USER_ADDRESS(rip))
+			break;
+
+		data[done++] = rip;
+		rbp = (uint64_t *)*rbp;
+	}
+
+	return done;
+}
+
+static bool intel_irq(context_t *context) {
+	if ((rdmsr(MSR_PERF_GLOBAL_STATUS) & INTEL_TSC_INDEX_OFFSET) == 0)
+		return false;
+
+	uint8_t data[1 + sizeof(uintptr_t) * 256];
+
+	uint8_t size = get_backtrace(context, (uintptr_t *)(data + 1));
+	if (size) {
+		*data = size;
+		profiling_insert(size, data);
+	}
+
+	arch_apic_init_perf(); // SDM LVT figure says a PMC irq masks itself, so unmask it
+	reset_intel_pmc();
+	return true;
+}
+
+static bool amd_irq(context_t *context) {
+	__assert(!"Unimplemented");
+}
+
+
+static void init_amd_pmc(void) {
+	__assert(!"Unimplemented");
+}
+
+
+
+bool arch_profiling_irq(context_t *context) {
+	if (!strcmp(current_cpu()->vendor, "AuthenticAMD"))
+		return amd_irq(context);
+	else if (!strcmp(current_cpu()->vendor, "GenuineIntel"))
+		return intel_irq(context);
+
+	__assert(!"unreachable");
+}
+
+
+void arch_profiling_init(void) {
+	arch_apic_init_perf();
+
+	if (!strcmp(current_cpu()->vendor, "AuthenticAMD"))
+		init_amd_pmc();
+	else if (!strcmp(current_cpu()->vendor, "GenuineIntel"))
+		init_intel_pmc();
+	else
+		printf("arch_prof: no PMC for this vendor\n");
+}
+
+INIT_ROUTINE_DEFINE(arch_profiling, INIT_ROUTINE_FLAGS_NONE, arch_profiling_init, profiling, cpu);
+
+#endif

kernel-src/arch/x86-64/smp.c

@@ -49,6 +49,10 @@ static void cpuwakeup(struct limine_smp_info *info) {
 	
 	timekeeper_init();
 
+#ifdef ENABLE_PROFILING
+	arch_profiling_init();
+#endif
+
 	__atomic_add_fetch(&arch_smp_cpusawake, 1, __ATOMIC_SEQ_CST);
 	sched_ap_entry();
 }

kernel-src/include/kernel/devfs.h

@@ -20,6 +20,7 @@
 #define DEV_MAJOR_MOUSE 11
 #define DEV_MAJOR_PTY 12
 #define DEV_MAJOR_ACPI 13
+#define DEV_MAJOR_PROF 14
 
 typedef struct {
 	int (*open)(int minor, vnode_t **vnode, int flags);

kernel-src/include/kernel/prof.h

@@ -0,0 +1,6 @@
+#ifndef _PROF_H
+#define _PROF_H
+
+void profiling_insert(uint8_t size, void *data);
+
+#endif

kernel-src/include/x86-64/arch/apic.h

@@ -7,6 +7,7 @@ void arch_apic_timerinit();
 void arch_apic_init();
 void arch_apic_initap();
 void arch_apic_eoi();
+void arch_apic_init_perf(void);
 void arch_ioapic_setirq(uint8_t gsi, uint8_t vector, uint8_t proc, bool masked);
 void arch_apic_sendipi(uint8_t cpu, uint8_t vec, uint8_t dest, uint8_t mode, uint8_t level);
 

kernel-src/include/x86-64/arch/cpu.h

@@ -13,6 +13,7 @@
 #include <kernel/dpc.h>
 #include <arch/apic.h>
 #include <kernel/timekeeper.h>
+#include <ringbuffer.h>
 
 #define ARCH_EOI arch_apic_eoi
 
@@ -68,6 +69,10 @@ typedef struct cpu_t {
 	spinlock_t shootdown_lock;
 	int *shootdown_done;
 
+#ifdef ENABLE_PROFILING
+	ringbuffer_t *prof_ringbuffer;
+#endif
+
 	// architecture specific, does not need to be exposed
 
 	uint64_t gdt[7];

kernel-src/include/x86-64/arch/msr.h

@@ -4,6 +4,11 @@
 #include <stdint.h>
 
 #define MSR_IA32APICBASE 0x1B
+#define MSR_PERF_FIXED_CTR2 0x30b
+#define MSR_PERF_FIXED_CTR_CTRL 0x38d
+#define MSR_PERF_GLOBAL_STATUS 0x38e
+#define MSR_PERF_GLOBAL_CTRL 0x38f
+#define MSR_PERF_GLOBAL_OVF_CTRL 0x390
 #define MSR_EFER 0xC0000080
 #define MSR_STAR 0xC0000081
 #define MSR_LSTAR 0xC0000082

kernel-src/include/x86-64/arch/prof.h

@@ -0,0 +1,9 @@
+#ifndef ARCH_PROF_H
+#define ARCH_PROF_H
+
+#include <arch/context.h>
+#include <stdbool.h>
+
+bool arch_profiling_irq(context_t *ctx);
+
+#endif

kernel-src/sys/prof.c

@@ -0,0 +1,197 @@
+#include <kernel/init.h>
+#include <kernel/devfs.h>
+#include <mutex.h>
+#include <ringbuffer.h>
+#include <semaphore.h>
+#include <kernel/thread.h>
+#include <kernel/scheduler.h>
+#include <kernel/vfs.h>
+#include <arch/smp.h>
+#include <logging.h>
+
+#ifdef ENABLE_PROFILING
+
+#define PROFILING_GLOBAL_BUFFER_SIZE (256 * 1024)
+#define PROFILING_PRIVATE_BUFFER_SIZE (256 * 1024) // the size of each cpu's private ring buffer
+
+static ringbuffer_t global_ringbuffer;
+static MUTEX_DEFINE(mutex);
+static pollheader_t pollheader;
+static semaphore_t wait_sem;
+static int remaining;
+
+// called from a possibly NMI context!
+void profiling_insert(uint8_t size, void *data) {
+	size_t write_size = sizeof(uint8_t) + sizeof(uintptr_t) * size;
+	if (current_cpu()->prof_ringbuffer && RINGBUFFER_FREESPACE(current_cpu()->prof_ringbuffer) >= write_size) {
+		size_t written = ringbuffer_write(current_cpu()->prof_ringbuffer, data, write_size);
+		if (write_size != written) for (;;);
+	}
+}
+
+static void profiling_thread() {
+	// this will be shared between the cpu and this thread
+	ringbuffer_t ringbuffer;
+	cpu_t *cpu = current_thread()->kernelarg;
+	__assert(ringbuffer_init(&ringbuffer, PROFILING_PRIVATE_BUFFER_SIZE) == 0);
+	sched_target_cpu(cpu);
+	cpu->prof_ringbuffer = &ringbuffer;
+
+	if (__atomic_sub_fetch(&remaining, 1, __ATOMIC_SEQ_CST) == 0)
+		semaphore_signal(&wait_sem);
+
+	for (;;) {
+		// profiling uses a NMI, which means that the handler cannot call into the scheduler
+		// without the possibility of a deadlock. for this reason, this thread will sleep for a set
+		// amount of time and only wake up to insert the data into a more reasonable place.
+		// by default, sleep for 10 ms
+		sched_sleep_us(10000);
+
+		MUTEX_ACQUIRE(&mutex);
+
+		int read = 0;
+
+		for (;;) {
+			uint8_t data[1 + sizeof(uintptr_t) * 256];
+			size_t read_count = ringbuffer_read(&ringbuffer, data, 1);
+			if (read_count == 0)
+				break;
+
+			read_count = ringbuffer_read(&ringbuffer, data + 1, data[0] * sizeof(uintptr_t));
+			__assert(read_count == data[0] * sizeof(uintptr_t));
+			size_t write_size = 1 + data[0] * sizeof(uintptr_t);
+
+			if (RINGBUFFER_FREESPACE(&global_ringbuffer) >= write_size) {
+				size_t written = ringbuffer_write(&global_ringbuffer, data, write_size);
+				__assert(write_size == written);
+				read = 1;
+			}
+
+		}
+
+		if (read)
+			poll_event(&pollheader, POLLIN);
+
+		MUTEX_RELEASE(&mutex);
+	}
+}
+
+static int internal_poll(polldata_t *data, int events) {
+	int revents = 0;
+
+	if (events & POLLOUT)
+		revents |= POLLERR;
+
+	if ((events & POLLIN) && RINGBUFFER_DATACOUNT(&global_ringbuffer))
+		revents |= POLLIN;
+
+	if (revents == 0 && data)
+		poll_add(&pollheader, data, events);
+
+	return revents;
+}
+
+static int device_poll(int minor, polldata_t *data, int events) {
+	(void)minor;
+	MUTEX_ACQUIRE(&mutex);
+
+	int revents = internal_poll(data, events);
+
+	MUTEX_RELEASE(&mutex);
+
+	return revents;
+}
+
+static int device_read(int minor, iovec_iterator_t *iovec_iterator, size_t size, uintmax_t offset, int flags, size_t *readc) {
+	(void)minor;
+	if (unlikely(size == 0)) {
+		*readc = 0;
+		return 0;
+	}
+
+	polldesc_t desc;
+	int error = poll_initdesc(&desc, 1);
+	if (error)
+		return error;
+
+	MUTEX_ACQUIRE(&mutex);
+
+	for (;;) {
+		int revents = internal_poll(&desc.data[0], POLLIN);
+		if (revents)
+			break;
+
+		if (flags & V_FFLAGS_NONBLOCKING) {
+			error = EAGAIN;
+			goto leave;
+		}
+
+		MUTEX_RELEASE(&mutex);
+		error = poll_dowait(&desc, 0);
+
+		if (error)
+			goto leave_nolock;
+		poll_leave(&desc);
+		MUTEX_ACQUIRE(&mutex);
+	}
+
+	*readc = iovec_iterator_read_from_ringbuffer(iovec_iterator, &global_ringbuffer, size);
+	if (*readc == RINGBUFFER_USER_COPY_FAILED)
+		error = EFAULT;
+
+	__assert(*readc);
+
+	leave:
+	MUTEX_RELEASE(&mutex);
+	leave_nolock:
+	poll_leave(&desc);
+	poll_destroydesc(&desc);
+	return error;
+}
+
+static int device_ioctl(int, unsigned long request, void *, int *result, cred_t *cred) {
+	if (request != 12345678)
+		return ENOTTY;
+
+	// only supported ioctl is a truncation one: remove all previous data
+
+	MUTEX_ACQUIRE(&mutex);
+
+	ringbuffer_truncate(&global_ringbuffer, PROFILING_GLOBAL_BUFFER_SIZE);
+
+	MUTEX_RELEASE(&mutex);
+
+	*result = 0;
+	return 0;
+}
+
+static devops_t devops = {
+	.read = device_read,
+	.poll = device_poll,
+	.ioctl = device_ioctl
+};
+
+static void profiling_init() {
+	// initialize global profiling data
+	POLL_INITHEADER(&pollheader);
+	__assert(ringbuffer_init(&global_ringbuffer, PROFILING_GLOBAL_BUFFER_SIZE) == 0);
+	__assert(devfs_register(&devops, "prof", V_TYPE_CHDEV, DEV_MAJOR_PROF, 0, 0400, NULL) == 0);
+
+	// start a thread for each cpu
+	SEMAPHORE_INIT(&wait_sem, 0);
+	remaining = arch_smp_cpusawake;
+
+	for (int i = 0; i < arch_smp_cpusawake; ++i) {
+		thread_t *thread = sched_newthread(profiling_thread, PAGE_SIZE * 2, -100, NULL, NULL);
+		__assert(thread);
+
+		thread->kernelarg = smp_cpus[i];
+		sched_queue(thread);
+	}
+
+	semaphore_wait(&wait_sem, false);
+}
+
+INIT_ROUTINE_DEFINE(profiling, INIT_ROUTINE_FLAGS_NONE, profiling_init, devfs, smp);
+
+#endif