diff --git a/kernel-src/arch/x86-64/apic.c b/kernel-src/arch/x86-64/apic.c index 104319ee..8069b86b 100644 --- a/kernel-src/arch/x86-64/apic.c +++ b/kernel-src/arch/x86-64/apic.c @@ -178,8 +178,6 @@ void arch_apic_initap() { } } - printf("processor id: %lu (APIC) %lu (ACPI)\n", current_cpu()->id, current_cpu()->acpiid); - isr_t *nmiisr = interrupt_allocate(nmi, NULL, IPL_MAX); // MAX IPL because NMI __assert(nmiisr); @@ -225,16 +223,9 @@ void arch_apic_timerinit() { writelapic(APIC_TIMER_DIVIDE, 11); - void (*uswait)(time_t) = NULL; - - if (arch_hpet_exists()) - uswait = arch_hpet_waitus; - - __assert(uswait); - writelapic(APIC_TIMER_INITIALCOUNT, 0xffffffff); - uswait(50000); + timekeeper_wait_us(50000); time_t ticksperus = (0xffffffff - readlapic(APIC_TIMER_COUNT)) / 50000; diff --git a/kernel-src/arch/x86-64/hpet.c b/kernel-src/arch/x86-64/hpet.c index d1cd3e1a..05db9fc1 100644 --- a/kernel-src/arch/x86-64/hpet.c +++ b/kernel-src/arch/x86-64/hpet.c @@ -27,84 +27,84 @@ #define HPET_CAP_LEGACYCAPABLE (1 << 15) #define HPET_CAP_FSPERTICK(x) (((x) >> 32) & 0xffffffff) -static volatile uint64_t *hpet; -static time_t ticksperus; -static uint64_t tickspassed; - // in order of preference #define TYPE_64BIT 0 #define TYPE_LEGACY 1 -static int type; + + +static MUTEX_DEFINE(init_mutex); +static struct { + volatile uint64_t *hpet; + uint64_t tickspassed; + int type; +} hpet_private; + +static timekeeper_source_info_t timekeeper_source_info = { + .private = &hpet_private +}; static uint64_t read64(int reg) { - return hpet[reg]; + return hpet_private.hpet[reg]; } static void write64(int reg, uint64_t v) { - hpet[reg] = v; -} - -time_t arch_hpet_ticks() { - if (type == TYPE_64BIT) { - return read64(HPET_REG_COUNTER); - } else { - uint64_t ticks; - do { - ticks = __atomic_load_n(&tickspassed, __ATOMIC_SEQ_CST) + (read64(HPET_REG_COUNTER) & 0xffffffff); - } while (ticks < __atomic_load_n(&tickspassed, __ATOMIC_SEQ_CST)); - return ticks; - } -} - -void arch_hpet_waitticks(time_t ticks) { - uint64_t target = arch_hpet_ticks() + ticks; - while (target > arch_hpet_ticks()) asm volatile("pause"); -} - -void arch_hpet_waitus(time_t us) { - arch_hpet_waitticks(us * ticksperus); + hpet_private.hpet[reg] = v; } static void counterirq(isr_t *isr, context_t *context) { write64(HPET_REG_CONFIG, 0); - __atomic_add_fetch(&tickspassed, read64(HPET_REG_COUNTER), __ATOMIC_SEQ_CST); + __atomic_add_fetch(&hpet_private.tickspassed, read64(HPET_REG_COUNTER), __ATOMIC_SEQ_CST); write64(HPET_REG_COUNTER, 0); - write64(HPET_REG_CONFIG, 1 + ((type == TYPE_LEGACY) ? 2 : 0)); + write64(HPET_REG_CONFIG, 1 + ((hpet_private.type == TYPE_LEGACY) ? 2 : 0)); } -bool arch_hpet_exists() { - // This is assumed by arch_hpet_init below - return true; -} - -time_t arch_hpet_init() { - uacpi_table tbl; +static bool hpet_probe(void) { + bool usable = false; struct acpi_hpet *table; + uacpi_table tbl; - if (uacpi_table_find_by_signature("HPET", &tbl) != UACPI_STATUS_OK) { - printf("hpet: no timers\n"); - return 0; - } + if (uacpi_table_find_by_signature("HPET", &tbl) != UACPI_STATUS_OK) + return false; table = tbl.ptr; + if (table->address.address_space_id != ACPI_AS_ID_SYS_MEM) + goto leave; + + usable = true; + leave: + uacpi_table_unref(&tbl); + return usable; +} + +static timekeeper_source_info_t *hpet_init(void) { + uacpi_table tbl; + struct acpi_hpet *table; + + MUTEX_ACQUIRE(&init_mutex, false); + if (timekeeper_source_info.ticks_per_us) { + // already initialized by someone else + goto leave; + } + + __assert(uacpi_table_find_by_signature("HPET", &tbl) == UACPI_STATUS_OK); + table = tbl.ptr; printf("hpet%lu: %lu bits %lu comparators\n", table->number, (table->block_id & ACPI_HPET_COUNT_SIZE_CAP) ? 64 : 32, (table->block_id >> ACPI_HPET_NUMBER_OF_COMPARATORS_SHIFT) & ACPI_HPET_NUMBER_OF_COMPARATORS_MASK); __assert(table->address.address_space_id == ACPI_AS_ID_SYS_MEM); uint64_t page_offset = table->address.address % PAGE_SIZE; - hpet = vmm_map(NULL, (PAGE_SIZE + page_offset), VMM_FLAGS_PHYSICAL, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, (void *)(table->address.address - page_offset)); - __assert(hpet); - hpet = (void *)((uintptr_t)hpet + page_offset); + hpet_private.hpet = vmm_map(NULL, (PAGE_SIZE + page_offset), VMM_FLAGS_PHYSICAL, ARCH_MMU_FLAGS_READ | ARCH_MMU_FLAGS_WRITE | ARCH_MMU_FLAGS_NOEXEC, (void *)(table->address.address - page_offset)); + __assert(hpet_private.hpet); + hpet_private.hpet = (void *)((uintptr_t)hpet_private.hpet + page_offset); uint64_t capabilities = read64(HPET_REG_CAPS); - - ticksperus = 1000000000 / HPET_CAP_FSPERTICK(capabilities); - printf("hpet%lu: %lu ticks per us (%lu fs per tick)\n", table->number, ticksperus, HPET_CAP_FSPERTICK(capabilities)); - __assert(ticksperus); + timekeeper_source_info.ticks_per_us = 1000000000 / HPET_CAP_FSPERTICK(capabilities); + printf("hpet%lu: %lu ticks per us (%lu fs per tick)\n", table->number, timekeeper_source_info.ticks_per_us, HPET_CAP_FSPERTICK(capabilities)); + __assert(timekeeper_source_info.ticks_per_us); write64(HPET_REG_CONFIG, 0); write64(HPET_REG_COUNTER, 0); @@ -120,7 +120,7 @@ time_t arch_hpet_init() { // TODO FSB mapping // timer 0 will be used with the timer irq replacing the PIT's as irq0. if (capabilities & HPET_CAP_LEGACYCAPABLE) { - type = TYPE_LEGACY; + hpet_private.type = TYPE_LEGACY; arch_ioapic_setirq(0, isr->id & 0xff, current_cpu_id(), false); @@ -139,11 +139,37 @@ time_t arch_hpet_init() { write64(HPET_TIMER_COMPARATOR(selectedtimer), 0x80000000); printf("hpet%d: timer %d on isr %d using %s\n", table->number, selectedtimer, isr->id & 0xff, "legacy replacement"); } else { - type = TYPE_64BIT; + hpet_private.type = TYPE_64BIT; } - write64(HPET_REG_CONFIG, 1 + ((type == TYPE_LEGACY) ? 2 : 0)); + write64(HPET_REG_CONFIG, 1 + ((hpet_private.type == TYPE_LEGACY) ? 2 : 0)); uacpi_table_unref(&tbl); - return ticksperus; + + leave: + MUTEX_RELEASE(&init_mutex); + return &timekeeper_source_info; + } + +time_t hpet_ticks(timekeeper_source_info_t *) { + if (hpet_private.type == TYPE_64BIT) { + return read64(HPET_REG_COUNTER); + } else { + uint64_t ticks; + do { + ticks = __atomic_load_n(&hpet_private.tickspassed, __ATOMIC_SEQ_CST) + (read64(HPET_REG_COUNTER) & 0xffffffff); + } while (ticks < __atomic_load_n(&hpet_private.tickspassed, __ATOMIC_SEQ_CST)); + return ticks; + } +} + +TIMEKEEPER_SOURCE( + hpet_source, + .name = "HPET", + .priority = 100, + .probe = hpet_probe, + .init = hpet_init, + .ticks = hpet_ticks, + .flags = TIMEKEEPER_SOURCE_FLAGS_EARLY +); diff --git a/kernel-src/arch/x86-64/kernel.ld b/kernel-src/arch/x86-64/kernel.ld index b2a2d575..28043a36 100644 --- a/kernel-src/arch/x86-64/kernel.ld +++ b/kernel-src/arch/x86-64/kernel.ld @@ -26,6 +26,13 @@ SECTIONS *(.rodata .rodata.*); } :rodata + . = ALIGN(8); + .timekeeper_sources : { + timekeeper_sources = .; + *(.timekeeper_sources); + timekeeper_sources_end = .; + } :rodata + . = ALIGN(4K); _rodata_end = .; _data_start = .; @@ -33,13 +40,13 @@ SECTIONS data : { *(.data .data.*); } :data - + + . = ALIGN(8); .bss : { *(COMMON) *(.bss .bss.*) } :data . = ALIGN(4K); - _data_end = .; } diff --git a/kernel-src/arch/x86-64/main.c b/kernel-src/arch/x86-64/main.c index b0a27434..26e04673 100644 --- a/kernel-src/arch/x86-64/main.c +++ b/kernel-src/arch/x86-64/main.c @@ -47,6 +47,7 @@ void kernel_entry() { cpu_set(&bsp_cpu); logging_init(); logging_sethook(arch_e9_putc); + arch_gdt_reload(); arch_idt_setup(); arch_idt_reload(); @@ -60,16 +61,17 @@ void kernel_entry() { cmdline_parse(); acpi_early_init(); arch_apic_init(); + + timekeeper_early_init(0); + cpu_initstate(); - // XXX fall back to another clock source - time_t hpet_ticks_per_us = arch_hpet_init(); - __assert(hpet_ticks_per_us); - - timekeeper_init(arch_hpet_ticks, hpet_ticks_per_us); - arch_apic_timerinit(); + sched_init(); + + timekeeper_init(); + arch_smp_wakeup(); pci_init(); diff --git a/kernel-src/arch/x86-64/smp.c b/kernel-src/arch/x86-64/smp.c index b4fe06de..f7c7b3bd 100644 --- a/kernel-src/arch/x86-64/smp.c +++ b/kernel-src/arch/x86-64/smp.c @@ -21,6 +21,8 @@ static void cpuwakeuphalt(struct limine_smp_info *info) { for (;;) CPU_HALT(); } +static time_t us_offset; + static void cpuwakeup(struct limine_smp_info *info) { cpu_set((cpu_t *)info->extra_argument); @@ -31,11 +33,14 @@ static void cpuwakeup(struct limine_smp_info *info) { arch_mmu_apswitch(); vmm_apinit(); + timekeeper_early_init(us_offset); + cpu_initstate(); arch_apic_timerinit(); - __atomic_add_fetch(&arch_smp_cpusawake, 1, __ATOMIC_SEQ_CST); + timekeeper_init(); + __atomic_add_fetch(&arch_smp_cpusawake, 1, __ATOMIC_SEQ_CST); sched_ap_entry(); } @@ -72,6 +77,8 @@ void arch_smp_wakeup() { void (*wakeupfn)(struct limine_smp_info *) = cmdline_get("nosmp") ? cpuwakeuphalt : cpuwakeup; + us_offset = timespec_us(timekeeper_timefromboot()); + // make the other processors jump to cpuwakeup() for (int i = 0; i < response->cpu_count; ++i) { // skip the bootstrap processor diff --git a/kernel-src/include/kernel/timekeeper.h b/kernel-src/include/kernel/timekeeper.h index 1d863f5f..3396e71e 100644 --- a/kernel-src/include/kernel/timekeeper.h +++ b/kernel-src/include/kernel/timekeeper.h @@ -2,9 +2,40 @@ #define _TIMEKEEPER_H #include +#include -void timekeeper_init(time_t (*tick)(), time_t ticksperus); -timespec_t timekeeper_timefromboot(); -timespec_t timekeeper_time(); +#define TIMEKEEPER_SOURCE_FLAGS_EARLY 1 // can be used as an early timer + +typedef struct { + time_t ticks_per_us; + void *private; +} timekeeper_source_info_t; + +typedef struct { + const char *name; + int priority; + bool (*probe)(void); // returns true if the timer can currently be used + timekeeper_source_info_t *(*init)(void); // initializes the timer, returning the info structure + time_t (*ticks)(timekeeper_source_info_t *private); // returns a monotonic number of ticks + int flags; +} timekeeper_source_t; + +// initializes the highest priority source that can be used as an early timer and switches the cpu to use it +void timekeeper_early_init(time_t us_offset); + +// initializes the highest priority source and switches the cpu to use it. uses the early timer for calibration if nescessary +void timekeeper_init(void); + +// spins waiting for us to pass +void timekeeper_wait_us(time_t us); + +timespec_t timekeeper_timefromboot(void); +timespec_t timekeeper_time(void); + +#define TIMEKEEPER_SOURCE(x, ...) \ + static timekeeper_source_t x = { \ + __VA_ARGS__ \ + }; \ + __attribute__((section(".timekeeper_sources"), used)) static timekeeper_source_t *timekeeper_source_##x = &x; #endif diff --git a/kernel-src/include/mutex.h b/kernel-src/include/mutex.h index 38d033c4..0029cca3 100644 --- a/kernel-src/include/mutex.h +++ b/kernel-src/include/mutex.h @@ -20,4 +20,6 @@ typedef semaphore_t mutex_t; #define MUTEX_TRY(m) \ semaphore_test(m) +#define MUTEX_DEFINE(x) SEMAPHORE_DEFINE(x, 1) + #endif diff --git a/kernel-src/include/semaphore.h b/kernel-src/include/semaphore.h index eb2bbf6e..a4f7abd7 100644 --- a/kernel-src/include/semaphore.h +++ b/kernel-src/include/semaphore.h @@ -23,6 +23,11 @@ typedef struct semaphore_t{ (x)->head = NULL; \ } +#define SEMAPHORE_DEFINE(x, v) semaphore_t x = { \ + .i = v, \ + .lock = SPINLOCK_INIT_VALUE \ + } + int semaphore_wait(semaphore_t *sem, bool interruptible); int semaphore_timedwait(semaphore_t *sem, time_t timeoutusec, bool interruptible); diff --git a/kernel-src/include/spinlock.h b/kernel-src/include/spinlock.h index f81f108f..e9514f59 100644 --- a/kernel-src/include/spinlock.h +++ b/kernel-src/include/spinlock.h @@ -7,6 +7,7 @@ typedef int spinlock_t; #define SPINLOCK_INIT(x) x = 0; +#define SPINLOCK_INIT_VALUE 0 static inline void spinlock_acquire(spinlock_t *lock){ while(!__sync_bool_compare_and_swap(lock, 0, 1)) asm("pause"); diff --git a/kernel-src/include/time.h b/kernel-src/include/time.h index c95645d3..4527e7fe 100644 --- a/kernel-src/include/time.h +++ b/kernel-src/include/time.h @@ -2,6 +2,7 @@ #define _TIME_H #include +#include typedef long time_t; typedef long suseconds_t; @@ -39,4 +40,21 @@ static inline time_t timespec_ns(timespec_t a) { return a.ns + a.s * 1000000000; } +static inline time_t timespec_us(timespec_t a) { + return a.ns / 1000 + a.s * 1000000; +} + +static inline timespec_t timespec_from_us(time_t us) { + timespec_t ts = { + .s = us / 1000000, + .ns = (us % 1000000) * 100 + }; + + return ts; +} + +static inline bool timespec_bigger(timespec_t t1, timespec_t t2) { + return t1.s > t2.s || (t1.s == t2.s && t1.ns > t2.ns); +} + #endif diff --git a/kernel-src/include/x86-64/arch/cpu.h b/kernel-src/include/x86-64/arch/cpu.h index 17d6c408..b6679b25 100644 --- a/kernel-src/include/x86-64/arch/cpu.h +++ b/kernel-src/include/x86-64/arch/cpu.h @@ -12,6 +12,7 @@ #include #include #include +#include #define ARCH_EOI arch_apic_eoi @@ -23,7 +24,12 @@ typedef struct cpu_t { vmmcontext_t *vmmctx; // expected to be here by other code - long id; + long id; // expected to be here by other code + + timekeeper_source_t *timekeeper_source; + timekeeper_source_info_t *timekeeper_source_info; + time_t timekeeper_source_base_ticks; + time_t timekeeper_source_tick_offset; isr_t isr[MAX_ISR_COUNT]; bool intstatus; diff --git a/kernel-src/sys/timekeeper.c b/kernel-src/sys/timekeeper.c index 8090723b..5e00c7b2 100644 --- a/kernel-src/sys/timekeeper.c +++ b/kernel-src/sys/timekeeper.c @@ -7,35 +7,123 @@ static volatile struct limine_boot_time_request timereq = { .revision = 0 }; -static time_t bootunix; -static time_t ticksperus; -static time_t (*clockticks)(); -static time_t initclockticks; +static time_t boot_unix; + +timespec_t timekeeper_timefromboot(void) { + long old_ipl = interrupt_raiseipl(IPL_DPC); + + time_t ticks = current_cpu()->timekeeper_source->ticks(current_cpu()->timekeeper_source_info); + time_t base_ticks = current_cpu()->timekeeper_source_base_ticks; + time_t ticks_per_us = current_cpu()->timekeeper_source_info->ticks_per_us; + time_t tick_offset = current_cpu()->timekeeper_source_tick_offset; + + interrupt_loweripl(old_ipl); -timespec_t timekeeper_timefromboot() { timespec_t ts; - time_t ticks = clockticks() - initclockticks; - time_t uspassed = ticks / ticksperus; - ts.s = uspassed / 1000000; - ts.ns = (uspassed * 1000) % 1000000000; + ticks = ticks - base_ticks + tick_offset; + time_t us_passed = ticks / ticks_per_us; + + ts.s = us_passed / 1000000; + ts.ns = (us_passed * 1000) % 1000000000; + return ts; } timespec_t timekeeper_time() { timespec_t fromboot = timekeeper_timefromboot(); - timespec_t unix; - unix.s = bootunix; - unix.ns = 0; + timespec_t unix = { + .s = boot_unix, + .ns = 0 + }; + return timespec_add(unix, fromboot); } -// tick is a function that returns the amount of ticks since the timer's initialisation -void timekeeper_init(time_t (*tick)(), time_t _ticksperus) { - __assert(timereq.response); - bootunix = timereq.response->boot_time; - printf("timekeeper: unix time at boot: %lu\n", bootunix); - initclockticks = tick(); - ticksperus = _ticksperus; - clockticks = tick; - printf("timekeeper: %lu clock ticks at init (%lu ticks per us)\n", initclockticks, ticksperus); +extern timekeeper_source_t *timekeeper_sources; +extern timekeeper_source_t *timekeeper_sources_end; + +static timekeeper_source_t *get_source(int priority, bool early) { + timekeeper_source_t *to_use = NULL; + + for (timekeeper_source_t **sourcep = &timekeeper_sources; sourcep < &timekeeper_sources_end; ++sourcep) { + timekeeper_source_t *source = *sourcep; + if (source->priority <= priority || (early && (source->flags & TIMEKEEPER_SOURCE_FLAGS_EARLY) == 0) || source->probe() == false) + continue; + + to_use = source; + priority = source->priority; + } + + return to_use; +} + +void timekeeper_wait_us(time_t us) { + timespec_t target = timespec_add(timespec_from_us(us), timekeeper_timefromboot()); + while (timespec_bigger(target, timekeeper_timefromboot())) + CPU_PAUSE(); +} + +void timekeeper_early_init(time_t us_offset) { + __assert(timereq.response); + boot_unix = timereq.response->boot_time; + + timekeeper_source_t *early_source = get_source(-1, true); + if (early_source == NULL) { + // we cannot continue without any timekeeper source + __assert(!"Unable to find an early timekeeper source"); + } + + timekeeper_source_info_t *early_source_info = early_source->init(); + __assert(early_source_info); + + // no need to raise IPL, this is called before the scheduler is initialised. + current_cpu()->timekeeper_source_base_ticks = early_source->ticks(early_source_info); + current_cpu()->timekeeper_source_info = early_source_info; + current_cpu()->timekeeper_source = early_source; + + // this tick offset is to account for any possible time before early_init. + // for example, on SMP init the early init will have the time passed on the ap as an offset + current_cpu()->timekeeper_source_tick_offset = us_offset * early_source_info->ticks_per_us; + + printf("cpu%d: timekeeper: \"%s\" selected as early source. %lu ticks at early init (%lu ticks per us)\n", + current_cpu_id(), early_source->name, current_cpu()->timekeeper_source_base_ticks, early_source_info->ticks_per_us); +} + +// this will do a re-init per cpu +// the initialization threads will not switch cpus so we dont have to worry about it here +void timekeeper_init(void) { + timekeeper_source_t *old_source = current_cpu()->timekeeper_source; + timekeeper_source_info_t *old_source_info = current_cpu()->timekeeper_source_info; + + timekeeper_source_t *new_source = get_source(old_source->priority, false); + if (new_source == old_source || new_source == NULL) { + // there is no better source in the system available than the current one + printf("cpu%d: timekeeper: keeping early source as main source\n", current_cpu_id()); + return; + } + + timekeeper_source_info_t *new_source_info = new_source->init(); + __assert(new_source_info); + + // we do need to raise IPL to IPL_DPC to not get preempted here as the + // timekeeper data will be in an invalid state + long old_ipl = interrupt_raiseipl(IPL_DPC); + + time_t old_base = current_cpu()->timekeeper_source_base_ticks; + + current_cpu()->timekeeper_source_base_ticks = new_source->ticks(new_source_info); + current_cpu()->timekeeper_source_info = new_source_info; + current_cpu()->timekeeper_source = new_source; + + // we need a way to account for the time between early init and init in the main source + // we will calculate how much time passed and have a tick offset + time_t old_tick_offset = current_cpu()->timekeeper_source_tick_offset; + time_t old_ticks = old_source->ticks(old_source_info); + time_t old_us = (old_ticks - old_base + old_tick_offset) / old_source_info->ticks_per_us; // XXX would a division with rounding be better for precision here? + current_cpu()->timekeeper_source_tick_offset = old_us * new_source_info->ticks_per_us; + + interrupt_loweripl(old_ipl); + + printf("cpu%d: timekeeper: \"%s\" selected as main source. %lu ticks at init (%lu ticks per us)\n", + current_cpu_id(), new_source->name, current_cpu()->timekeeper_source_base_ticks, new_source_info->ticks_per_us); }