X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=gettime.c;h=1a0f827460c9da6562678733f041be73c2eec484;hp=c0d27e4cd568fbddaca2600764e01733d01620bc;hb=fdc0f3b646e417497849d4398029f780b0e5262f;hpb=d481e00608dc118e646b6fee9c9292c1ab35448f diff --git a/gettime.c b/gettime.c index c0d27e4c..1a0f8274 100644 --- a/gettime.c +++ b/gettime.c @@ -3,20 +3,36 @@ */ #include +#include #include +#include #include "fio.h" #include "smalloc.h" #include "hash.h" +#include "os/os.h" -static int clock_gettime_works; -static struct timeval last_tv; -static int last_tv_valid; +#ifdef ARCH_HAVE_CPU_CLOCK +static unsigned long cycles_per_usec; +static unsigned long inv_cycles_per_usec; +#endif +int tsc_reliable = 0; + +struct tv_valid { + struct timeval last_tv; + uint64_t last_cycles; + int last_tv_valid; +}; +#ifdef CONFIG_TLS_THREAD +static __thread struct tv_valid static_tv_valid; +#else +static pthread_key_t tv_tls_key; +#endif -static struct timeval *fio_tv; -int fio_gtod_offload = 0; -int fio_gtod_cpu = -1; +enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE; +int fio_clock_source_set = 0; +enum fio_cs fio_clock_source_inited = CS_INVAL; #ifdef FIO_DEBUG_TIME @@ -109,12 +125,80 @@ static void fio_init gtod_init(void) #endif /* FIO_DEBUG_TIME */ +#ifdef CONFIG_CLOCK_GETTIME +static int fill_clock_gettime(struct timespec *ts) +{ +#ifdef CONFIG_CLOCK_MONOTONIC + return clock_gettime(CLOCK_MONOTONIC, ts); +#else + return clock_gettime(CLOCK_REALTIME, ts); +#endif +} +#endif + +static void *__fio_gettime(struct timeval *tp) +{ + struct tv_valid *tv; + +#ifdef CONFIG_TLS_THREAD + tv = &static_tv_valid; +#else + tv = pthread_getspecific(tv_tls_key); +#endif + + switch (fio_clock_source) { +#ifdef CONFIG_GETTIMEOFDAY + case CS_GTOD: + gettimeofday(tp, NULL); + break; +#endif +#ifdef CONFIG_CLOCK_GETTIME + case CS_CGETTIME: { + struct timespec ts; + + if (fill_clock_gettime(&ts) < 0) { + log_err("fio: clock_gettime fails\n"); + assert(0); + } + + tp->tv_sec = ts.tv_sec; + tp->tv_usec = ts.tv_nsec / 1000; + break; + } +#endif +#ifdef ARCH_HAVE_CPU_CLOCK + case CS_CPUCLOCK: { + uint64_t usecs, t; + + t = get_cpu_clock(); + if (tv && t < tv->last_cycles) { + dprint(FD_TIME, "CPU clock going back in time\n"); + t = tv->last_cycles; + } else if (tv) + tv->last_cycles = t; + + usecs = (t * inv_cycles_per_usec) / 16777216UL; + tp->tv_sec = usecs / 1000000; + tp->tv_usec = usecs % 1000000; + break; + } +#endif + default: + log_err("fio: invalid clock source %d\n", fio_clock_source); + break; + } + + return tv; +} + #ifdef FIO_DEBUG_TIME void fio_gettime(struct timeval *tp, void *caller) #else void fio_gettime(struct timeval *tp, void fio_unused *caller) #endif { + struct tv_valid *tv; + #ifdef FIO_DEBUG_TIME if (!caller) caller = __builtin_return_address(0); @@ -124,43 +208,419 @@ void fio_gettime(struct timeval *tp, void fio_unused *caller) if (fio_tv) { memcpy(tp, fio_tv, sizeof(*tp)); return; - } else if (!clock_gettime_works) { -gtod: - gettimeofday(tp, NULL); - } else { - struct timespec ts; + } + + tv = __fio_gettime(tp); - if (clock_gettime(CLOCK_REALTIME, &ts) < 0) { - clock_gettime_works = 0; - goto gtod; + /* + * If Linux is using the tsc clock on non-synced processors, + * sometimes time can appear to drift backwards. Fix that up. + */ + if (tv) { + if (tv->last_tv_valid) { + if (tp->tv_sec < tv->last_tv.tv_sec) + tp->tv_sec = tv->last_tv.tv_sec; + else if (tv->last_tv.tv_sec == tp->tv_sec && + tp->tv_usec < tv->last_tv.tv_usec) + tp->tv_usec = tv->last_tv.tv_usec; } + tv->last_tv_valid = 1; + memcpy(&tv->last_tv, tp, sizeof(*tp)); + } +} - tp->tv_sec = ts.tv_sec; - tp->tv_usec = ts.tv_nsec / 1000; +#ifdef ARCH_HAVE_CPU_CLOCK +static unsigned long get_cycles_per_usec(void) +{ + struct timeval s, e; + uint64_t c_s, c_e; + enum fio_cs old_cs = fio_clock_source; + +#ifdef CONFIG_CLOCK_GETTIME + fio_clock_source = CS_CGETTIME; +#else + fio_clock_source = CS_GTOD; +#endif + __fio_gettime(&s); + + c_s = get_cpu_clock(); + do { + uint64_t elapsed; + + __fio_gettime(&e); + + elapsed = utime_since(&s, &e); + if (elapsed >= 1280) { + c_e = get_cpu_clock(); + break; + } + } while (1); + + fio_clock_source = old_cs; + return (c_e - c_s + 127) >> 7; +} + +#define NR_TIME_ITERS 50 + +static int calibrate_cpu_clock(void) +{ + double delta, mean, S; + uint64_t avg, cycles[NR_TIME_ITERS]; + int i, samples; + + cycles[0] = get_cycles_per_usec(); + S = delta = mean = 0.0; + for (i = 0; i < NR_TIME_ITERS; i++) { + cycles[i] = get_cycles_per_usec(); + delta = cycles[i] - mean; + if (delta) { + mean += delta / (i + 1.0); + S += delta * (cycles[i] - mean); + } } /* - * If Linux is using the tsc clock on non-synced processors, - * sometimes time can appear to drift backwards. Fix that up. + * The most common platform clock breakage is returning zero + * indefinitely. Check for that and return failure. + */ + if (!cycles[0] && !cycles[NR_TIME_ITERS - 1]) + return 1; + + S = sqrt(S / (NR_TIME_ITERS - 1.0)); + + samples = avg = 0; + for (i = 0; i < NR_TIME_ITERS; i++) { + double this = cycles[i]; + + if ((fmax(this, mean) - fmin(this, mean)) > S) + continue; + samples++; + avg += this; + } + + S /= (double) NR_TIME_ITERS; + mean /= 10.0; + + for (i = 0; i < NR_TIME_ITERS; i++) + dprint(FD_TIME, "cycles[%d]=%llu\n", i, + (unsigned long long) cycles[i] / 10); + + avg /= samples; + avg = (avg + 5) / 10; + dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg); + dprint(FD_TIME, "mean=%f, S=%f\n", mean, S); + + cycles_per_usec = avg; + inv_cycles_per_usec = 16777216UL / cycles_per_usec; + dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec); + return 0; +} +#else +static int calibrate_cpu_clock(void) +{ + return 1; +} +#endif + +#ifndef CONFIG_TLS_THREAD +void fio_local_clock_init(int is_thread) +{ + struct tv_valid *t; + + t = calloc(sizeof(*t), 1); + if (pthread_setspecific(tv_tls_key, t)) + log_err("fio: can't set TLS key\n"); +} + +static void kill_tv_tls_key(void *data) +{ + free(data); +} +#else +void fio_local_clock_init(int is_thread) +{ +} +#endif + +void fio_clock_init(void) +{ + if (fio_clock_source == fio_clock_source_inited) + return; + +#ifndef CONFIG_TLS_THREAD + if (pthread_key_create(&tv_tls_key, kill_tv_tls_key)) + log_err("fio: can't create TLS key\n"); +#endif + + fio_clock_source_inited = fio_clock_source; + + if (calibrate_cpu_clock()) + tsc_reliable = 0; + + /* + * If the arch sets tsc_reliable != 0, then it must be good enough + * to use as THE clock source. For x86 CPUs, this means the TSC + * runs at a constant rate and is synced across CPU cores. + */ + if (tsc_reliable) { + if (!fio_clock_source_set) + fio_clock_source = CS_CPUCLOCK; + } else if (fio_clock_source == CS_CPUCLOCK) + log_info("fio: clocksource=cpu may not be reliable\n"); +} + +uint64_t utime_since(struct timeval *s, struct timeval *e) +{ + long sec, usec; + uint64_t ret; + + sec = e->tv_sec - s->tv_sec; + usec = e->tv_usec - s->tv_usec; + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + /* + * time warp bug on some kernels? */ - if (last_tv_valid) { - if (tp->tv_sec < last_tv.tv_sec) - tp->tv_sec = last_tv.tv_sec; - else if (last_tv.tv_sec == tp->tv_sec && - tp->tv_usec < last_tv.tv_usec) - tp->tv_usec = last_tv.tv_usec; + if (sec < 0 || (sec == 0 && usec < 0)) + return 0; + + ret = sec * 1000000ULL + usec; + + return ret; +} + +uint64_t utime_since_now(struct timeval *s) +{ + struct timeval t; + + fio_gettime(&t, NULL); + return utime_since(s, &t); +} + +uint64_t mtime_since(struct timeval *s, struct timeval *e) +{ + long sec, usec, ret; + + sec = e->tv_sec - s->tv_sec; + usec = e->tv_usec - s->tv_usec; + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + if (sec < 0 || (sec == 0 && usec < 0)) + return 0; + + sec *= 1000UL; + usec /= 1000UL; + ret = sec + usec; + + return ret; +} + +uint64_t mtime_since_now(struct timeval *s) +{ + struct timeval t; + void *p = __builtin_return_address(0); + + fio_gettime(&t, p); + return mtime_since(s, &t); +} + +uint64_t time_since_now(struct timeval *s) +{ + return mtime_since_now(s) / 1000; +} + +#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \ + defined(CONFIG_SFAA) + +#define CLOCK_ENTRIES 100000 + +struct clock_entry { + uint32_t seq; + uint32_t cpu; + uint64_t tsc; +}; + +struct clock_thread { + pthread_t thread; + int cpu; + pthread_mutex_t lock; + pthread_mutex_t started; + uint32_t *seq; + struct clock_entry *entries; +}; + +static inline uint32_t atomic32_inc_return(uint32_t *seq) +{ + return 1 + __sync_fetch_and_add(seq, 1); +} + +static void *clock_thread_fn(void *data) +{ + struct clock_thread *t = data; + struct clock_entry *c; + os_cpu_mask_t cpu_mask; + uint32_t last_seq; + int i; + + memset(&cpu_mask, 0, sizeof(cpu_mask)); + fio_cpu_set(&cpu_mask, t->cpu); + + if (fio_setaffinity(gettid(), cpu_mask) == -1) { + log_err("clock setaffinity failed\n"); + return (void *) 1; } - last_tv_valid = 1; - memcpy(&last_tv, tp, sizeof(*tp)); + + pthread_mutex_lock(&t->lock); + pthread_mutex_unlock(&t->started); + + last_seq = 0; + c = &t->entries[0]; + for (i = 0; i < CLOCK_ENTRIES; i++, c++) { + uint32_t seq; + uint64_t tsc; + + c->cpu = t->cpu; + do { + seq = atomic32_inc_return(t->seq); + if (seq < last_seq) + break; + tsc = get_cpu_clock(); + } while (seq != *t->seq); + + c->seq = seq; + c->tsc = tsc; + } + + log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu, + (unsigned long long) t->entries[i - 1].tsc - t->entries[0].tsc); + + /* + * The most common platform clock breakage is returning zero + * indefinitely. Check for that and return failure. + */ + if (!t->entries[i - 1].tsc && !t->entries[0].tsc) + return (void *) 1; + + return NULL; } -void fio_gtod_init(void) +static int clock_cmp(const void *p1, const void *p2) { - fio_tv = smalloc(sizeof(struct timeval)); - assert(fio_tv); + const struct clock_entry *c1 = p1; + const struct clock_entry *c2 = p2; + + if (c1->seq == c2->seq) + log_err("cs: bug in atomic sequence!\n"); + + return c1->seq - c2->seq; +} + +int fio_monotonic_clocktest(void) +{ + struct clock_thread *threads; + unsigned int nr_cpus = cpus_online(); + struct clock_entry *entries; + unsigned long tentries, failed; + struct clock_entry *prev, *this; + uint32_t seq = 0; + int i; + + log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no"); + + fio_debug |= 1U << FD_TIME; + calibrate_cpu_clock(); + fio_debug &= ~(1U << FD_TIME); + + threads = malloc(nr_cpus * sizeof(struct clock_thread)); + tentries = CLOCK_ENTRIES * nr_cpus; + entries = malloc(tentries * sizeof(struct clock_entry)); + + log_info("cs: Testing %u CPUs\n", nr_cpus); + + for (i = 0; i < nr_cpus; i++) { + struct clock_thread *t = &threads[i]; + + t->cpu = i; + t->seq = &seq; + t->entries = &entries[i * CLOCK_ENTRIES]; + pthread_mutex_init(&t->lock, NULL); + pthread_mutex_init(&t->started, NULL); + pthread_mutex_lock(&t->lock); + pthread_create(&t->thread, NULL, clock_thread_fn, t); + } + + for (i = 0; i < nr_cpus; i++) { + struct clock_thread *t = &threads[i]; + + pthread_mutex_lock(&t->started); + } + + for (i = 0; i < nr_cpus; i++) { + struct clock_thread *t = &threads[i]; + + pthread_mutex_unlock(&t->lock); + } + + for (failed = i = 0; i < nr_cpus; i++) { + struct clock_thread *t = &threads[i]; + void *ret; + + pthread_join(t->thread, &ret); + if (ret) + failed++; + } + free(threads); + + if (failed) { + log_err("Clocksource test: %lu threads failed\n", failed); + goto err; + } + + qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp); + + for (failed = i = 0; i < tentries; i++) { + this = &entries[i]; + + if (!i) { + prev = this; + continue; + } + + if (prev->tsc > this->tsc) { + uint64_t diff = prev->tsc - this->tsc; + + log_info("cs: CPU clock mismatch (diff=%llu):\n", + (unsigned long long) diff); + log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq); + log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq); + failed++; + } + + prev = this; + } + + if (failed) + log_info("cs: Failed: %lu\n", failed); + else + log_info("cs: Pass!\n"); + +err: + free(entries); + return !!failed; } -void fio_gtod_update(void) +#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */ + +int fio_monotonic_clocktest(void) { - gettimeofday(fio_tv, NULL); + log_info("cs: current platform does not support CPU clocks\n"); + return 0; } + +#endif