kcov: initial implementation

Implement a device (/dev/kcov) that exposes coverage info for the
kernel. This will be especially useful for coverage-based fuzzing (e.g.
syzkaller).

Exclude a lot of functions which would crash/otherwise interfere in the
coverage, including only processing coverage callbacks when not handling
an irq.

kernel-src/sys/interrupt.c

@@ -2,6 +2,7 @@
 #include <logging.h>
 #include <arch/cpu.h>
 #include <arch/context.h>
+#include <kernel/kcov.h>
 #include <panic.h>
 
 void arch_interrupt_disable();
@@ -9,7 +10,7 @@ void arch_interrupt_enable();
 
 #define DOPENDING_SAVE() arch_context_saveandcall(dopending, NULL, NULL)
 
-static void removefromqueue(isr_t *isr) {
+KCOV_DISABLED static void removefromqueue(isr_t *isr) {
 	if (isr->prev)
 		isr->prev->next = isr->next;
 	else
@@ -19,7 +20,7 @@ static void removefromqueue(isr_t *isr) {
 		isr->next->prev = isr->prev;
 }
 
-static void insertinqueue(isr_t *isr) {
+KCOV_DISABLED static void insertinqueue(isr_t *isr) {
 	if (isr->pending)
 		return;
 
@@ -29,7 +30,7 @@ static void insertinqueue(isr_t *isr) {
 	current_cpu()->isrqueue = isr;
 }
 
-static void runisr(isr_t *isr, context_t *ctx) {
+KCOV_DISABLED static void runisr(isr_t *isr, context_t *ctx) {
 		long oldipl = -1;
 		if (isr->priority != IPL_IGNORE)
 			oldipl = interrupt_raiseipl(isr->priority);
@@ -41,7 +42,7 @@ static void runisr(isr_t *isr, context_t *ctx) {
 			interrupt_loweripl(oldipl);
 }
 
-static void dopending(context_t *ctx, void *) {
+KCOV_DISABLED static void dopending(context_t *ctx, void *) {
 	arch_interrupt_disable();
 	bool entrystatus = current_cpu()->intstatus;
 	current_cpu()->intstatus = false;
@@ -100,7 +101,7 @@ __attribute__((no_caller_saved_registers)) void sched_userspacecheck(context_t *
 								     uint64_t syscallret,
 								     uint64_t syscallerrno);
 
-void interrupt_isr(int vec, context_t *ctx) {
+KCOV_DISABLED void interrupt_isr(int vec, context_t *ctx) {
 	isr_t *isr = &current_cpu()->isr[vec];
 	current_cpu()->intstatus = false;
 
@@ -133,7 +134,7 @@ void interrupt_isr(int vec, context_t *ctx) {
 	current_cpu()->intstatus = ARCH_CONTEXT_INTSTATUS(ctx);
 }
 
-void interrupt_raise(isr_t *isr) {
+KCOV_DISABLED void interrupt_raise(isr_t *isr) {
 	bool entrystate = interrupt_set(false);
 
 	if (isr->pending)
@@ -146,7 +147,8 @@ void interrupt_raise(isr_t *isr) {
 	interrupt_set(entrystate);
 }
 
-void interrupt_register(int vector, void (*func)(isr_t *self, context_t *ctx), void (*eoi)(isr_t *self), long priority) {
+KCOV_DISABLED void interrupt_register(int vector, void (*func)(isr_t *self, context_t *ctx), 
+			void (*eoi)(isr_t *self), long priority) {
 	bool intstatus = interrupt_set(false);
 
 	isr_t *isr = &current_cpu()->isr[vector];
@@ -159,7 +161,8 @@ void interrupt_register(int vector, void (*func)(isr_t *self, context_t *ctx), v
 	interrupt_set(intstatus);
 }
 
-isr_t *interrupt_allocate(void (*func)(isr_t *self, context_t *ctx), void (*eoi)(isr_t *self), long priority) {
+KCOV_DISABLED isr_t *interrupt_allocate(void (*func)(isr_t *self, context_t *ctx), void (*eoi)(isr_t *self),
+			  long priority) {
 	bool intstatus = interrupt_set(false);
 	isr_t *isr = NULL;
 
@@ -175,7 +178,7 @@ isr_t *interrupt_allocate(void (*func)(isr_t *self, context_t *ctx), void (*eoi)
 	return isr;
 }
 
-long interrupt_loweripl(long ipl) {
+KCOV_DISABLED long interrupt_loweripl(long ipl) {
 	bool oldintstatus = interrupt_set(false);
 	long oldipl = current_cpu()->ipl;
 	if (oldipl < ipl)
@@ -185,7 +188,7 @@ long interrupt_loweripl(long ipl) {
 	return oldipl;
 }
 
-long interrupt_raiseipl(long ipl) {
+KCOV_DISABLED long interrupt_raiseipl(long ipl) {
 	bool oldintstatus = interrupt_set(false);
 	long oldipl = current_cpu()->ipl;
 
@@ -197,7 +200,7 @@ long interrupt_raiseipl(long ipl) {
 	return oldipl;
 }
 
-bool interrupt_set(bool status) {
+KCOV_DISABLED bool interrupt_set(bool status) {
 	arch_interrupt_disable();
 	bool old = current_cpu()->intstatus;
 

kernel-src/sys/kcov.c

@@ -0,0 +1,129 @@
+#include <kernel/kcov.h>
+#include <kernel/alloc.h>
+#include <kernel/devfs.h>
+#include <kernel/init.h>
+#include <kernel/usercopy.h>
+#include <arch/cpu.h>
+#include <logging.h>
+
+#ifdef KCOV_ENABLED
+
+#define MAX_ENTRIES (1 << 24)
+
+KCOV_DISABLED void kcov_publish_pc(uint64_t pc) {
+	kcov_t *kcov = current_thread()->kcov;
+
+	uint64_t entry = kcov->buffer[0];
+	if (entry >= kcov->entry_count)
+		return;
+
+	kcov->buffer[entry + 1] = pc;
+	kcov->buffer[0] = entry + 1;
+	__atomic_thread_fence(__ATOMIC_RELEASE);
+}
+
+KCOV_DISABLED void kcov_publish_cmp(uint64_t pc, uint64_t v1, uint64_t v2, uint64_t flags) {
+	kcov_t *kcov = current_thread()->kcov;
+
+	uint64_t entry = kcov->buffer[0];
+	if (entry >= kcov->entry_count / 4)
+		return;
+
+	kcov->buffer[entry * 4 + 1] = flags;
+	kcov->buffer[entry * 4 + 2] = v1;
+	kcov->buffer[entry * 4 + 3] = v2;
+	kcov->buffer[entry * 4 + 4] = pc;
+	kcov->buffer[0] = entry + 1;
+	__atomic_thread_fence(__ATOMIC_RELEASE);
+}
+
+// expects the mutex to be held OR this to be the last reference
+KCOV_DISABLED void kcov_free_buffer(kcov_t *kcov) {
+	if (kcov->buffer == NULL)
+		return;
+
+	mm_unmap(kcov->buffer, sizeof(uint64_t) * (kcov->entry_count + 1), 0);
+	kcov->entry_count = 0;
+	kcov->buffer = NULL;
+}
+
+KCOV_DISABLED static int resize_buffer(kcov_t *kcov, unsigned int entry_count) {
+	if (entry_count == 0) {
+		kcov_free_buffer(kcov);
+		return 0;
+	}
+
+	void *tmp = mm_map(NULL, sizeof(uint64_t) * (entry_count + 1), MM_RANGE_FLAGS_ALLOCATE, 
+			   ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
+	if (tmp == NULL)
+		return ENOMEM;
+
+	void *old = kcov->buffer;
+	size_t old_entries = kcov->entry_count;
+
+	kcov->buffer = tmp;
+	kcov->entry_count = entry_count;
+
+	mm_unmap(old, (old_entries + 1) * sizeof(uint64_t), 0);
+
+	return 0;
+}
+
+KCOV_DISABLED int kcov_set_mode(unsigned int mode) {
+	kcov_t *kcov = current_thread()->kcov;
+	__assert(kcov);
+
+	if (mode != KCOV_MODE_DISABLED &&
+	    mode != KCOV_MODE_PC &&
+	    mode != KCOV_MODE_CMP)
+		return EINVAL;
+
+	if (mode == KCOV_MODE_DISABLED && kcov->mode == KCOV_MODE_DISABLED)
+		return EINVAL;
+
+	if (mode != KCOV_MODE_DISABLED && kcov->mode != KCOV_MODE_DISABLED)
+		return EINVAL;
+
+	if (kcov->buffer == NULL)
+		return EINVAL;
+
+	kcov->mode = mode;
+	return 0;
+}
+
+KCOV_DISABLED int kcov_set_buffer_size(unsigned int entry_count) {
+	if (entry_count < 0 || entry_count > MAX_ENTRIES)
+		return EINVAL;
+
+	kcov_t *kcov = current_thread()->kcov;
+	__assert(kcov);
+
+	MUTEX_ACQUIRE(&kcov->mapping_mutex);
+	int error = 0;
+	if (kcov->mappings || kcov->mode != KCOV_MODE_DISABLED) {
+		error = EBUSY;
+		goto cleanup;
+	}
+
+	if (kcov->buffer) {
+		error = resize_buffer(kcov, entry_count);
+		goto cleanup;
+	}
+
+	if (entry_count == 0)
+		goto cleanup;
+
+	kcov->buffer = mm_map(NULL, sizeof(uint64_t) * (entry_count + 1), MM_RANGE_FLAGS_ALLOCATE, 
+			      ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, NULL);
+	if (kcov->buffer == NULL) {
+		error = ENOMEM;
+		goto cleanup;
+	}
+	kcov->entry_count = entry_count;
+
+	cleanup:
+	MUTEX_RELEASE(&kcov->mapping_mutex);
+	return error;
+}
+
+#endif

kernel-src/sys/kcov_dev.c

@@ -0,0 +1,255 @@
+#ifdef KCOV_ENABLED
+#include <kernel/kcov.h>
+#include <kernel/alloc.h>
+#include <kernel/devfs.h>
+#include <kernel/init.h>
+#include <kernel/usercopy.h>
+#include <arch/cpu.h>
+#include <logging.h>
+
+#define KCOV_MAX 256
+
+static MUTEX_DEFINE(map_mutex);
+static kcov_t *kcov_map[KCOV_MAX];
+
+static int allocate_kcov(int *out) {
+	kcov_t *kcov = alloc(sizeof(kcov_t));
+	if (kcov == NULL)
+		return ENOMEM;
+
+	MUTEX_INIT(&kcov->mapping_mutex);
+	__atomic_store_n(&kcov->refcount, 1, __ATOMIC_RELAXED);
+
+	MUTEX_ACQUIRE(&map_mutex);
+	int minor;
+	for (minor = 0; minor < KCOV_MAX; ++minor) {
+		if (kcov_map[minor] == NULL) {
+			kcov_map[minor] = kcov;
+			break;
+		}
+	}
+
+	if (minor < KCOV_MAX)
+		kcov->minor = minor;
+
+	MUTEX_RELEASE(&map_mutex);
+
+	if (minor == KCOV_MAX) {
+		free(kcov);
+		return EAGAIN;
+	}
+
+	*out = minor;
+	return 0;
+}
+
+static void free_kcov(int minor) {
+	MUTEX_ACQUIRE(&map_mutex);
+	kcov_t *kcov = kcov_map[minor];
+	kcov_map[minor] = NULL;
+	MUTEX_RELEASE(&map_mutex);
+
+	__assert(kcov);
+	free(kcov);
+}
+
+void kcov_hold(kcov_t *kcov) {
+	__atomic_add_fetch(&kcov->refcount, 1, __ATOMIC_RELAXED);
+}
+
+void kcov_release(kcov_t *kcov) {
+	if (__atomic_sub_fetch(&kcov->refcount, 1, __ATOMIC_RELAXED) == 0) {
+		__atomic_thread_fence(__ATOMIC_ACQ_REL);
+		// this is safe to do as we are the only ones holding a reference
+		kcov_free_buffer(kcov);
+		free_kcov(kcov->minor);
+	}
+}
+
+static kcov_t *get_kcov(int minor) {
+	if (minor >= KCOV_MAX)
+		return NULL;
+
+	MUTEX_ACQUIRE(&map_mutex);
+	kcov_t *kcov = kcov_map[minor];
+	MUTEX_RELEASE(&map_mutex);
+
+	return kcov;
+}
+
+static devops_t ops;
+
+static int create_node_for_kcov(kcov_t *kcov, int minor, vnode_t **vnodep) {
+	vnode_t *vnode;
+	int error = devfs_register_anonymous(&ops, V_TYPE_CHDEV, DEV_MAJOR_KCOV, minor,
+					     0, NULL, &vnode);
+	if (error)
+		return error;
+
+	kcov->vnode_handle = vnode;
+	devfs_remove_anonymous(DEV_MAJOR_KCOV, minor);
+
+	// a kcov will have two references initially:
+	// - one for the thread holding it
+	// - one for the vnode
+	kcov_hold(kcov);
+	*vnodep = vnode;
+	return 0;
+}
+
+static int open(int oldminor, vnode_t **vnodep, int flags) {
+	vnode_t *old = *vnodep;
+
+	if (oldminor != KCOV_MAX)
+		return ENODEV;
+
+	// if the thread already has a kcov attached, return it
+	if (current_thread()->kcov) {
+		kcov_t *kcov = current_thread()->kcov;
+
+		int error = create_node_for_kcov(kcov, kcov->minor, vnodep);
+		if (error)
+			return error;
+
+		VOP_RELEASE(old);
+		return 0;
+	}
+
+	int minor;
+	int error = allocate_kcov(&minor);
+	if (error)
+		return error;
+
+	kcov_t *kcov = get_kcov(minor);
+	error = create_node_for_kcov(kcov, minor, vnodep);
+	if (error) {
+		kcov_release(kcov);
+		return error;
+	}
+
+	current_thread()->kcov = kcov;
+	VOP_RELEASE(old);
+	return 0;
+}
+
+#define IOCTL_REQUEST_START 0xF1177000
+#define IOCTL_REQUEST_STOP 0xF1177100
+#define IOCTL_REQUEST_SET_SIZE 0xF1177200
+
+static int ioctl(int minor, unsigned long request, void *arg, int *result, cred_t *cred) {
+	kcov_t *kcov = current_thread()->kcov;
+	if (kcov == NULL || kcov->minor != minor)
+		return EINVAL;
+
+	switch (request) {
+		case IOCTL_REQUEST_SET_SIZE: {
+			int size;
+			int error = USERCOPY_POSSIBLY_FROM_USER(&size, arg, sizeof(size));
+			if (error)
+				return error;
+
+			return kcov_set_buffer_size(size);
+		}
+		case IOCTL_REQUEST_START: {
+			if (kcov->mode != KCOV_MODE_DISABLED)
+				return EBUSY;
+
+			if (kcov->buffer == NULL)
+				return EINVAL;
+
+			int mode;
+			int error = USERCOPY_POSSIBLY_FROM_USER(&mode, arg, sizeof(mode));
+			if (error)
+				return error;
+
+			return kcov_set_mode(mode);
+		}
+		case IOCTL_REQUEST_STOP:
+			return kcov_set_mode(KCOV_MODE_DISABLED);
+		default:
+			return ENOTTY;
+	}
+}
+
+static int mmap(int minor, void *addr, uintmax_t offset, int flags) {
+	kcov_t *kcov = get_kcov(minor);
+	__assert(kcov); // mapped kcov that doesn't exist...?
+
+	if (offset % PAGE_SIZE)
+		return EINVAL;
+
+	// why would anyone do this?
+	if ((flags & V_FFLAGS_SHARED) == 0)
+		return EINVAL;
+
+	MUTEX_ACQUIRE(&kcov->mapping_mutex);
+	int error = EINVAL;
+	if (kcov->buffer == NULL)
+		goto cleanup;
+
+	size_t start_entry = offset / sizeof(uint64_t);
+	if (kcov->entry_count < start_entry)
+		goto cleanup;
+
+	void *phys = arch_mmu_getphysical(current_mm_context()->pagetable,
+					  &kcov->buffer[start_entry]);
+	__assert(phys);
+	// this code is called holding the mm lock, so this is the way of doing this
+	if (arch_mmu_map(current_mm_context()->pagetable, phys, addr,
+			 mm_vnode_flags_to_mmu_flags(flags)) == false) {
+		error = ENOMEM;
+		goto cleanup;
+	}
+
+	++kcov->mappings;
+	error = 0;
+	cleanup:
+	MUTEX_RELEASE(&kcov->mapping_mutex);
+	return error;
+}
+
+static int munmap(int minor, void *addr, uintmax_t offset, int flags) {
+	kcov_t *kcov = get_kcov(minor);
+	__assert(kcov); // mapped kcov that doesn't exist...?
+
+	if (offset % PAGE_SIZE)
+		return EINVAL;
+
+	// why would anyone do this?
+	if ((flags & V_FFLAGS_SHARED) == 0)
+		return EINVAL;
+
+	// this code is called holding the mm lock, so this is the way of doing this
+	arch_mmu_unmap(current_mm_context()->pagetable, addr);
+	arch_mmu_invalidate_range(addr, PAGE_SIZE);
+
+	MUTEX_ACQUIRE(&kcov->mapping_mutex);
+	--kcov->mappings;
+	MUTEX_RELEASE(&kcov->mapping_mutex);
+
+	return 0;
+}
+
+static void inactive(int minor) {
+	kcov_t *kcov = get_kcov(minor);
+	if (kcov == NULL)
+		return;
+
+	kcov_release(kcov);
+}
+
+static devops_t ops = {
+	.open = open,
+	.inactive = inactive,
+	.mmap = mmap,
+	.munmap = munmap,
+	.ioctl = ioctl
+};
+
+static void kcov_init(void) {
+	__assert(devfs_register(&ops, "kcov", V_TYPE_CHDEV, DEV_MAJOR_KCOV, KCOV_MAX, 0600,
+				NULL) == 0);
+}
+
+INIT_ROUTINE_DEFINE(kcov, INIT_ROUTINE_FLAGS_NONE, kcov_init, devfs);
+#endif

kernel-src/sys/kcov_gcc.c

@@ -0,0 +1,101 @@
+#include <kernel/kcov.h>
+#include <arch/cpu.h>
+
+#ifdef KCOV_ENABLED
+
+KCOV_DISABLED void __sanitizer_cov_trace_pc(void) {
+	if (current_thread() == NULL || current_thread()->kcov == NULL ||
+	    current_thread()->kcov->mode != KCOV_MODE_PC)
+		return;
+
+	if (current_irq_depth() > 0)
+		return;
+
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	kcov_publish_pc(pc);
+}
+
+KCOV_DISABLED static void publish_cmp(uint64_t pc, uint64_t v1, uint64_t v2, uint64_t flags) {
+	if (current_thread() == NULL || current_thread()->kcov == NULL ||
+	    current_thread()->kcov->mode != KCOV_MODE_CMP)
+		return;
+
+	if (current_irq_depth() > 0)
+		return;
+
+	kcov_publish_cmp(pc, v1, v2, flags);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_const_cmp1(uint8_t a1, uint8_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_CONST | KCOV_CMP_FLAGS_8BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_const_cmp2(uint16_t a1, uint16_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_CONST | KCOV_CMP_FLAGS_16BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_const_cmp4(uint32_t a1, uint32_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_CONST | KCOV_CMP_FLAGS_32BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_const_cmp8(uint64_t a1, uint64_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_CONST | KCOV_CMP_FLAGS_64BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_cmp1(uint8_t a1, uint8_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_8BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_cmp2(uint16_t a1, uint16_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_16BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_cmp4(uint32_t a1, uint32_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_32BIT);
+
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_cmp8(uint64_t a1, uint64_t a2) {
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	publish_cmp(pc, a1, a2, KCOV_CMP_FLAGS_64BIT);
+}
+
+KCOV_DISABLED void __sanitizer_cov_trace_switch(uint64_t v, uint64_t *cases) {
+	if (current_thread() == NULL || current_thread()->kcov == NULL ||
+	    current_thread()->kcov->mode != KCOV_MODE_CMP)
+		return;
+
+	if (current_irq_depth() > 0)
+		return;
+
+	uint64_t pc = (uint64_t)__builtin_return_address(0);
+	uint64_t flags = KCOV_CMP_FLAGS_CONST;
+	switch (cases[1]) {
+		case 8:
+			flags |= KCOV_CMP_FLAGS_8BIT;
+			break;
+		case 16:
+			flags |= KCOV_CMP_FLAGS_16BIT;
+			break;
+		case 32:
+			flags |= KCOV_CMP_FLAGS_32BIT;
+			break;
+		case 64:
+			flags |= KCOV_CMP_FLAGS_64BIT;
+			break;
+		default:
+			return;
+	}
+
+	for (int i = 0; i < cases[0]; ++i)
+		kcov_publish_cmp(pc, cases[2 + i], v, flags);
+}
+
+#endif

kernel-src/sys/thread.c

@@ -57,6 +57,10 @@ thread_t *sched_newthread(void *ip, size_t kstacksize, int nice, proc_t *proc, v
 }
 
 void sched_destroythread(thread_t *thread) {
+#ifdef KCOV_ENABLED
+	if (thread->kcov)
+		kcov_release(thread->kcov);
+#endif
 	mm_unmap(thread->kernelstack, thread->kernelstacksize, 0);
 	arch_extracontext_free(&thread->extracontext);
 	slab_free(thread_cache, thread);
@@ -101,6 +105,11 @@ __attribute__((noreturn)) void sched_threadexit() {
 		PROC_RELEASE(proc);
 	}
 
+#ifdef KCOV_ENABLED
+	if (current_thread()->kcov)
+		kcov_set_mode(KCOV_MODE_DISABLED);
+#endif
+
 	interrupt_set(false);
 	arch_context_saveandcall(threadexit_internal, current_cpu()->schedulerstack, NULL);
 	__builtin_unreachable();