timekeeper: redo timekeeping

kernel-src/arch/x86-64/apic.c

@@ -178,8 +178,6 @@ void arch_apic_initap() {
 		}
 	}
 
-	printf("processor id: %lu (APIC) %lu (ACPI)\n", current_cpu()->id, current_cpu()->acpiid);
-
 	isr_t *nmiisr = interrupt_allocate(nmi, NULL, IPL_MAX); // MAX IPL because NMI
 	__assert(nmiisr);
 
@@ -225,16 +223,9 @@ void arch_apic_timerinit() {
 
 	writelapic(APIC_TIMER_DIVIDE, 11);
 
-	void (*uswait)(time_t) = NULL;
-
-	if (arch_hpet_exists())
-		uswait = arch_hpet_waitus;
-
-	__assert(uswait);
-
 	writelapic(APIC_TIMER_INITIALCOUNT, 0xffffffff);
 
-	uswait(50000);
+	timekeeper_wait_us(50000);
 
 	time_t ticksperus = (0xffffffff - readlapic(APIC_TIMER_COUNT)) / 50000;
 

kernel-src/arch/x86-64/hpet.c

@@ -27,84 +27,84 @@
 #define HPET_CAP_LEGACYCAPABLE (1 << 15)
 #define HPET_CAP_FSPERTICK(x) (((x) >> 32) & 0xffffffff)
 
-static volatile uint64_t *hpet;
-static time_t ticksperus;
-static uint64_t tickspassed;
-
 // in order of preference
 #define TYPE_64BIT 0
 #define TYPE_LEGACY 1
-static int type;
+
+
+static MUTEX_DEFINE(init_mutex);
+static struct {
+	volatile uint64_t *hpet;
+	uint64_t tickspassed;
+	int type;
+} hpet_private;
+
+static timekeeper_source_info_t timekeeper_source_info = {
+	.private = &hpet_private
+};
 
 static uint64_t read64(int reg) {
-	return hpet[reg];
+	return hpet_private.hpet[reg];
 }
 
 static void write64(int reg, uint64_t v) {
-	hpet[reg] = v;
+	hpet_private.hpet[reg] = v;
-}
-
-time_t arch_hpet_ticks() {
-	if (type == TYPE_64BIT) {
-		return read64(HPET_REG_COUNTER);
-	} else {
-		uint64_t ticks;
-		do {
-			ticks = __atomic_load_n(&tickspassed, __ATOMIC_SEQ_CST) + (read64(HPET_REG_COUNTER) & 0xffffffff);
-		} while (ticks < __atomic_load_n(&tickspassed, __ATOMIC_SEQ_CST));
-		return ticks;
-	}
-}
-
-void arch_hpet_waitticks(time_t ticks) {
-	uint64_t target = arch_hpet_ticks() + ticks;
-	while (target > arch_hpet_ticks()) asm volatile("pause");
-}
-
-void arch_hpet_waitus(time_t us) {
-	arch_hpet_waitticks(us * ticksperus);
 }
 
 static void counterirq(isr_t *isr, context_t *context) {
 	write64(HPET_REG_CONFIG, 0);
 
-	__atomic_add_fetch(&tickspassed, read64(HPET_REG_COUNTER), __ATOMIC_SEQ_CST);
+	__atomic_add_fetch(&hpet_private.tickspassed, read64(HPET_REG_COUNTER), __ATOMIC_SEQ_CST);
 	write64(HPET_REG_COUNTER, 0);
 
-	write64(HPET_REG_CONFIG, 1 + ((type == TYPE_LEGACY) ? 2 : 0));
+	write64(HPET_REG_CONFIG, 1 + ((hpet_private.type == TYPE_LEGACY) ? 2 : 0));
 }
 
-bool arch_hpet_exists() {
+static bool hpet_probe(void) {
-	// This is assumed by arch_hpet_init below
+	bool usable = false;
-	return true;
-}
-
-time_t arch_hpet_init() {
-	uacpi_table tbl;
 	struct acpi_hpet *table;
+	uacpi_table tbl;
 
-	if (uacpi_table_find_by_signature("HPET", &tbl) != UACPI_STATUS_OK) {
+	if (uacpi_table_find_by_signature("HPET", &tbl) != UACPI_STATUS_OK)
-		printf("hpet: no timers\n");
+		return false;
-		return 0;
-	}
 
 	table = tbl.ptr;
 
+	if (table->address.address_space_id != ACPI_AS_ID_SYS_MEM)
+		goto leave;
+
+	usable = true;
+	leave:
+	uacpi_table_unref(&tbl);
+	return usable;
+}
+
+static timekeeper_source_info_t *hpet_init(void) {
+	uacpi_table tbl;
+	struct acpi_hpet *table;
+
+	MUTEX_ACQUIRE(&init_mutex, false);
+	if (timekeeper_source_info.ticks_per_us) {
+		// already initialized by someone else
+		goto leave;
+	}
+
+	__assert(uacpi_table_find_by_signature("HPET", &tbl) == UACPI_STATUS_OK);
+	table = tbl.ptr;
 	printf("hpet%lu: %lu bits %lu comparators\n", table->number,
 		(table->block_id & ACPI_HPET_COUNT_SIZE_CAP) ? 64 : 32,
 		(table->block_id >> ACPI_HPET_NUMBER_OF_COMPARATORS_SHIFT) & ACPI_HPET_NUMBER_OF_COMPARATORS_MASK);
 	__assert(table->address.address_space_id == ACPI_AS_ID_SYS_MEM);
 
 	uint64_t page_offset = table->address.address % PAGE_SIZE;
-	hpet = vmm_map(NULL, (PAGE_SIZE + page_offset), VMM_FLAGS_PHYSICAL, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, (void *)(table->address.address - page_offset));
+	hpet_private.hpet = vmm_map(NULL, (PAGE_SIZE + page_offset), VMM_FLAGS_PHYSICAL, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, (void *)(table->address.address - page_offset));
-	__assert(hpet);
+	__assert(hpet_private.hpet);
-	hpet = (void *)((uintptr_t)hpet + page_offset);
+	hpet_private.hpet = (void *)((uintptr_t)hpet_private.hpet + page_offset);
 
 	uint64_t capabilities = read64(HPET_REG_CAPS);
-
+	timekeeper_source_info.ticks_per_us = 1000000000 / HPET_CAP_FSPERTICK(capabilities);
-	ticksperus = 1000000000 / HPET_CAP_FSPERTICK(capabilities);
+	printf("hpet%lu: %lu ticks per us (%lu fs per tick)\n", table->number, timekeeper_source_info.ticks_per_us, HPET_CAP_FSPERTICK(capabilities));
-	printf("hpet%lu: %lu ticks per us (%lu fs per tick)\n", table->number, ticksperus, HPET_CAP_FSPERTICK(capabilities));
+	__assert(timekeeper_source_info.ticks_per_us);
-	__assert(ticksperus);
 
 	write64(HPET_REG_CONFIG, 0);
 	write64(HPET_REG_COUNTER, 0);
@@ -120,7 +120,7 @@ time_t arch_hpet_init() {
 		// TODO FSB mapping
 		// timer 0 will be used with the timer irq replacing the PIT's as irq0.
 		if (capabilities & HPET_CAP_LEGACYCAPABLE) {
-			type = TYPE_LEGACY;
+			hpet_private.type = TYPE_LEGACY;
 
 			arch_ioapic_setirq(0, isr->id & 0xff, current_cpu_id(), false);
 
@@ -139,11 +139,37 @@ time_t arch_hpet_init() {
 		write64(HPET_TIMER_COMPARATOR(selectedtimer), 0x80000000);
 		printf("hpet%d: timer %d on isr %d using %s\n", table->number, selectedtimer, isr->id & 0xff, "legacy replacement");
 	} else {
-		type = TYPE_64BIT;
+		hpet_private.type = TYPE_64BIT;
 	}
 
-	write64(HPET_REG_CONFIG, 1 + ((type == TYPE_LEGACY) ? 2 : 0));
+	write64(HPET_REG_CONFIG, 1 + ((hpet_private.type == TYPE_LEGACY) ? 2 : 0));
 
 	uacpi_table_unref(&tbl);
-	return ticksperus;
+
+	leave:
+	MUTEX_RELEASE(&init_mutex);
+	return &timekeeper_source_info;
+
 }
+
+time_t hpet_ticks(timekeeper_source_info_t *) {
+	if (hpet_private.type == TYPE_64BIT) {
+		return read64(HPET_REG_COUNTER);
+	} else {
+		uint64_t ticks;
+		do {
+			ticks = __atomic_load_n(&hpet_private.tickspassed, __ATOMIC_SEQ_CST) + (read64(HPET_REG_COUNTER) & 0xffffffff);
+		} while (ticks < __atomic_load_n(&hpet_private.tickspassed, __ATOMIC_SEQ_CST));
+		return ticks;
+	}
+}
+
+TIMEKEEPER_SOURCE(
+	hpet_source,
+	.name = "HPET",
+	.priority = 100,
+	.probe = hpet_probe,
+	.init = hpet_init,
+	.ticks = hpet_ticks,
+	.flags = TIMEKEEPER_SOURCE_FLAGS_EARLY
+);

kernel-src/arch/x86-64/kernel.ld

@@ -26,6 +26,13 @@ SECTIONS
 		*(.rodata .rodata.*);
 	} :rodata
 
+	. = ALIGN(8);
+	.timekeeper_sources : {
+		timekeeper_sources = .;
+		*(.timekeeper_sources);
+		timekeeper_sources_end = .;
+	} :rodata
+
 	. = ALIGN(4K);
 	_rodata_end = .;
 	_data_start = .;
@@ -33,13 +40,13 @@ SECTIONS
 	data : {
 		*(.data .data.*);
 	} :data
-	
+
+	. = ALIGN(8);
 	.bss : {
 		*(COMMON)
 		*(.bss .bss.*)
 	} :data
 	
 	. = ALIGN(4K);
-
 	_data_end = .;
 }

kernel-src/arch/x86-64/main.c

@@ -47,6 +47,7 @@ void kernel_entry() {
 	cpu_set(&bsp_cpu);
 	logging_init();
 	logging_sethook(arch_e9_putc);
+
 	arch_gdt_reload();
 	arch_idt_setup();
 	arch_idt_reload();
@@ -60,16 +61,17 @@ void kernel_entry() {
 	cmdline_parse();
 	acpi_early_init();
 	arch_apic_init();
+
+	timekeeper_early_init(0);
+
 	cpu_initstate();
 
-	// XXX fall back to another clock source
-	time_t hpet_ticks_per_us = arch_hpet_init();
-	__assert(hpet_ticks_per_us);
-
-	timekeeper_init(arch_hpet_ticks, hpet_ticks_per_us);
-
 	arch_apic_timerinit();
+
 	sched_init();
+
+	timekeeper_init();
+
 	arch_smp_wakeup();
 	pci_init();
 

kernel-src/arch/x86-64/smp.c

@@ -21,6 +21,8 @@ static void cpuwakeuphalt(struct limine_smp_info *info) {
 	for (;;) CPU_HALT();
 }
 
+static time_t us_offset;
+
 static void cpuwakeup(struct limine_smp_info *info) {
 	cpu_set((cpu_t *)info->extra_argument);
 
@@ -31,11 +33,14 @@ static void cpuwakeup(struct limine_smp_info *info) {
 	arch_mmu_apswitch();
 	vmm_apinit();
 
+	timekeeper_early_init(us_offset);
+
 	cpu_initstate();
 	arch_apic_timerinit();
 
-	__atomic_add_fetch(&arch_smp_cpusawake, 1, __ATOMIC_SEQ_CST);
+	timekeeper_init();
 
+	__atomic_add_fetch(&arch_smp_cpusawake, 1, __ATOMIC_SEQ_CST);
 	sched_ap_entry();
 }
 
@@ -72,6 +77,8 @@ void arch_smp_wakeup() {
 
 	void (*wakeupfn)(struct limine_smp_info *) = cmdline_get("nosmp") ? cpuwakeuphalt : cpuwakeup;
 
+	us_offset = timespec_us(timekeeper_timefromboot());
+
 	// make the other processors jump to cpuwakeup()
 	for (int i = 0; i < response->cpu_count; ++i) {
 		// skip the bootstrap processor

kernel-src/include/kernel/timekeeper.h

@@ -2,9 +2,40 @@
 #define _TIMEKEEPER_H
 
 #include <time.h>
+#include <stdbool.h>
 
-void timekeeper_init(time_t (*tick)(), time_t ticksperus);
+#define TIMEKEEPER_SOURCE_FLAGS_EARLY 1 // can be used as an early timer
-timespec_t timekeeper_timefromboot();
+
-timespec_t timekeeper_time();
+typedef struct {
+	time_t ticks_per_us;
+	void *private;
+} timekeeper_source_info_t;
+
+typedef struct {
+	const char *name;
+	int priority;
+	bool (*probe)(void); // returns true if the timer can currently be used
+	timekeeper_source_info_t *(*init)(void); // initializes the timer, returning the info structure
+	time_t (*ticks)(timekeeper_source_info_t *private); // returns a monotonic number of ticks
+	int flags;
+} timekeeper_source_t;
+
+// initializes the highest priority source that can be used as an early timer and switches the cpu to use it
+void timekeeper_early_init(time_t us_offset);
+
+// initializes the highest priority source and switches the cpu to use it. uses the early timer for calibration if nescessary
+void timekeeper_init(void);
+
+// spins waiting for us to pass
+void timekeeper_wait_us(time_t us);
+
+timespec_t timekeeper_timefromboot(void);
+timespec_t timekeeper_time(void);
+
+#define TIMEKEEPER_SOURCE(x, ...) \
+	static timekeeper_source_t x = { \
+		__VA_ARGS__ \
+	}; \
+	__attribute__((section(".timekeeper_sources"), used)) static timekeeper_source_t *timekeeper_source_##x = &x;
 
 #endif

kernel-src/include/mutex.h

@@ -20,4 +20,6 @@ typedef semaphore_t mutex_t;
 #define MUTEX_TRY(m) \
 	semaphore_test(m)
 
+#define MUTEX_DEFINE(x) SEMAPHORE_DEFINE(x, 1)
+
 #endif

kernel-src/include/semaphore.h

@@ -23,6 +23,11 @@ typedef struct semaphore_t{
 		(x)->head = NULL; \
 	}
 
+#define SEMAPHORE_DEFINE(x, v) semaphore_t x = { \
+	.i = v, \
+	.lock = SPINLOCK_INIT_VALUE \
+	}
+
 
 int semaphore_wait(semaphore_t *sem, bool interruptible);
 int semaphore_timedwait(semaphore_t *sem, time_t timeoutusec, bool interruptible);

kernel-src/include/spinlock.h

@@ -7,6 +7,7 @@
 typedef int spinlock_t;
 
 #define SPINLOCK_INIT(x) x = 0;
+#define SPINLOCK_INIT_VALUE 0
 
 static inline void spinlock_acquire(spinlock_t *lock){
 	while(!__sync_bool_compare_and_swap(lock, 0, 1)) asm("pause");

kernel-src/include/time.h

@@ -2,6 +2,7 @@
 #define _TIME_H
 
 #include <util.h>
+#include <stdbool.h>
 
 typedef long time_t;
 typedef long suseconds_t;
@@ -39,4 +40,21 @@ static inline time_t timespec_ns(timespec_t a) {
 	return a.ns + a.s * 1000000000;
 }
 
+static inline time_t timespec_us(timespec_t a) {
+	return a.ns / 1000 + a.s * 1000000;
+}
+
+static inline timespec_t timespec_from_us(time_t us) {
+	timespec_t ts = {
+		.s = us / 1000000,
+		.ns = (us % 1000000) * 100
+	};
+
+	return ts;
+}
+
+static inline bool timespec_bigger(timespec_t t1, timespec_t t2) {
+	return t1.s > t2.s || (t1.s == t2.s && t1.ns > t2.ns);
+}
+
 #endif

kernel-src/include/x86-64/arch/cpu.h

@@ -12,6 +12,7 @@
 #include <kernel/scheduler.h>
 #include <kernel/dpc.h>
 #include <arch/apic.h>
+#include <kernel/timekeeper.h>
 
 #define ARCH_EOI arch_apic_eoi
 
@@ -23,7 +24,12 @@ typedef struct cpu_t {
 
 	vmmcontext_t *vmmctx; // expected to be here by other code
 
-	long id;
+	long id; // expected to be here by other code
+
+	timekeeper_source_t *timekeeper_source;
+	timekeeper_source_info_t *timekeeper_source_info;
+	time_t timekeeper_source_base_ticks;
+	time_t timekeeper_source_tick_offset;
 
 	isr_t isr[MAX_ISR_COUNT];
 	bool intstatus;

kernel-src/sys/timekeeper.c

@@ -7,35 +7,123 @@ static volatile struct limine_boot_time_request timereq = {
 	.revision = 0
 };
 
-static time_t bootunix;
+static time_t boot_unix;
-static time_t ticksperus;
+
-static time_t (*clockticks)();
+timespec_t timekeeper_timefromboot(void) {
-static time_t initclockticks;
+	long old_ipl = interrupt_raiseipl(IPL_DPC);
+
+	time_t ticks = current_cpu()->timekeeper_source->ticks(current_cpu()->timekeeper_source_info);
+	time_t base_ticks = current_cpu()->timekeeper_source_base_ticks;
+	time_t ticks_per_us = current_cpu()->timekeeper_source_info->ticks_per_us;
+	time_t tick_offset = current_cpu()->timekeeper_source_tick_offset;
+
+	interrupt_loweripl(old_ipl);
 
-timespec_t timekeeper_timefromboot() {
 	timespec_t ts;
-	time_t ticks = clockticks() - initclockticks;
+	ticks = ticks - base_ticks + tick_offset;
-	time_t uspassed = ticks / ticksperus;
+	time_t us_passed = ticks / ticks_per_us;
-	ts.s = uspassed / 1000000;
+
-	ts.ns = (uspassed * 1000) % 1000000000;
+	ts.s = us_passed / 1000000;
+	ts.ns = (us_passed * 1000) % 1000000000;
+
 	return ts;
 }
 
 timespec_t timekeeper_time() {
 	timespec_t fromboot = timekeeper_timefromboot();
-	timespec_t unix;
+	timespec_t unix = {
-	unix.s = bootunix;
+		.s = boot_unix,
-	unix.ns = 0;
+		.ns = 0
+	};
+
 	return timespec_add(unix, fromboot);
 }
 
-// tick is a function that returns the amount of ticks since the timer's initialisation
+extern timekeeper_source_t *timekeeper_sources;
-void timekeeper_init(time_t (*tick)(), time_t _ticksperus) {
+extern timekeeper_source_t *timekeeper_sources_end;
-	__assert(timereq.response);
+
-	bootunix = timereq.response->boot_time;
+static timekeeper_source_t *get_source(int priority, bool early) {
-	printf("timekeeper: unix time at boot: %lu\n", bootunix);
+	timekeeper_source_t *to_use = NULL;
-	initclockticks = tick();
+
-	ticksperus = _ticksperus;
+	for (timekeeper_source_t **sourcep = &timekeeper_sources; sourcep < &timekeeper_sources_end; ++sourcep) {
-	clockticks = tick;
+		timekeeper_source_t *source = *sourcep;
-	printf("timekeeper: %lu clock ticks at init (%lu ticks per us)\n", initclockticks, ticksperus);
+		if (source->priority <= priority || (early && (source->flags & TIMEKEEPER_SOURCE_FLAGS_EARLY) == 0) || source->probe() == false)
+			continue;
+
+		to_use = source;
+		priority = source->priority;
+	}
+
+	return to_use;
+}
+
+void timekeeper_wait_us(time_t us) {
+	timespec_t target = timespec_add(timespec_from_us(us), timekeeper_timefromboot());
+	while (timespec_bigger(target, timekeeper_timefromboot()))
+		CPU_PAUSE();
+}
+
+void timekeeper_early_init(time_t us_offset) {
+	__assert(timereq.response);
+	boot_unix = timereq.response->boot_time;
+
+	timekeeper_source_t *early_source = get_source(-1, true);
+	if (early_source == NULL) {
+		// we cannot continue without any timekeeper source
+		__assert(!"Unable to find an early timekeeper source");
+	}
+
+	timekeeper_source_info_t *early_source_info = early_source->init();
+	__assert(early_source_info);
+
+	// no need to raise IPL, this is called before the scheduler is initialised.
+	current_cpu()->timekeeper_source_base_ticks = early_source->ticks(early_source_info);
+	current_cpu()->timekeeper_source_info = early_source_info;
+	current_cpu()->timekeeper_source = early_source;
+
+	// this tick offset is to account for any possible time before early_init.
+	// for example, on SMP init the early init will have the time passed on the ap as an offset
+	current_cpu()->timekeeper_source_tick_offset = us_offset * early_source_info->ticks_per_us;
+
+	printf("cpu%d: timekeeper: \"%s\" selected as early source. %lu ticks at early init (%lu ticks per us)\n",
+			current_cpu_id(), early_source->name, current_cpu()->timekeeper_source_base_ticks, early_source_info->ticks_per_us);
+}
+
+// this will do a re-init per cpu
+// the initialization threads will not switch cpus so we dont have to worry about it here
+void timekeeper_init(void) {
+	timekeeper_source_t *old_source = current_cpu()->timekeeper_source;
+	timekeeper_source_info_t *old_source_info = current_cpu()->timekeeper_source_info;
+
+	timekeeper_source_t *new_source = get_source(old_source->priority, false);
+	if (new_source == old_source || new_source == NULL) {
+		// there is no better source in the system available than the current one
+		printf("cpu%d: timekeeper: keeping early source as main source\n", current_cpu_id());
+		return;
+	}
+
+	timekeeper_source_info_t *new_source_info = new_source->init();
+	__assert(new_source_info);
+
+	// we do need to raise IPL to IPL_DPC to not get preempted here as the
+	// timekeeper data will be in an invalid state
+	long old_ipl = interrupt_raiseipl(IPL_DPC);
+
+	time_t old_base = current_cpu()->timekeeper_source_base_ticks;
+
+	current_cpu()->timekeeper_source_base_ticks = new_source->ticks(new_source_info);
+	current_cpu()->timekeeper_source_info = new_source_info;
+	current_cpu()->timekeeper_source = new_source;
+
+	// we need a way to account for the time between early init and init in the main source
+	// we will calculate how much time passed and have a tick offset
+	time_t old_tick_offset = current_cpu()->timekeeper_source_tick_offset;
+	time_t old_ticks = old_source->ticks(old_source_info);
+	time_t old_us = (old_ticks - old_base + old_tick_offset) / old_source_info->ticks_per_us; // XXX would a division with rounding be better for precision here?
+	current_cpu()->timekeeper_source_tick_offset = old_us * new_source_info->ticks_per_us;
+
+	interrupt_loweripl(old_ipl);
+
+	printf("cpu%d: timekeeper: \"%s\" selected as main source. %lu ticks at init (%lu ticks per us)\n",
+			current_cpu_id(), new_source->name, current_cpu()->timekeeper_source_base_ticks, new_source_info->ticks_per_us);
 }