X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=gettime.c;h=549c35e9c0dc34826a69108e202f169ddffead05;hp=4f6ec955c561cf49b9bfae3740f1c4968d51a0ff;hb=486332e567826eb284bfd9064440867f31fccd7f;hpb=130fe6161f5e6d2bb98537affdc1573f676e2764 diff --git a/gettime.c b/gettime.c index 4f6ec955..549c35e9 100644 --- a/gettime.c +++ b/gettime.c @@ -3,25 +3,36 @@ */ #include +#include #include +#include #include "fio.h" -#include "os.h" +#include "smalloc.h" #include "hash.h" -static int clock_gettime_works = 0; +#ifdef ARCH_HAVE_CPU_CLOCK +static unsigned long cycles_per_usec; +static unsigned long last_cycles; +int tsc_reliable = 0; +#endif +static struct timeval last_tv; +static int last_tv_valid; + +enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE; +int fio_clock_source_set = 0; #ifdef FIO_DEBUG_TIME #define HASH_BITS 8 #define HASH_SIZE (1 << HASH_BITS) -static struct list_head hash[HASH_SIZE]; +static struct flist_head hash[HASH_SIZE]; static int gtod_inited; struct gtod_log { - struct list_head list; + struct flist_head list; void *caller; unsigned long calls; }; @@ -29,10 +40,11 @@ struct gtod_log { static struct gtod_log *find_hash(void *caller) { unsigned long h = hash_ptr(caller, HASH_BITS); - struct list_head *entry; + struct flist_head *entry; - list_for_each(entry, &hash[h]) { - struct gtod_log *log = list_entry(entry, struct gtod_log, list); + flist_for_each(entry, &hash[h]) { + struct gtod_log *log = flist_entry(entry, struct gtod_log, + list); if (log->caller == caller) return log; @@ -49,12 +61,12 @@ static struct gtod_log *find_log(void *caller) unsigned long h; log = malloc(sizeof(*log)); - INIT_LIST_HEAD(&log->list); + INIT_FLIST_HEAD(&log->list); log->caller = caller; log->calls = 0; h = hash_ptr(caller, HASH_BITS); - list_add_tail(&log->list, &hash[h]); + flist_add_tail(&log->list, &hash[h]); } return log; @@ -75,13 +87,14 @@ static void fio_exit fio_dump_gtod(void) int i; for (i = 0; i < HASH_SIZE; i++) { - struct list_head *entry; + struct flist_head *entry; struct gtod_log *log; - list_for_each(entry, &hash[i]) { - log = list_entry(entry, struct gtod_log, list); + flist_for_each(entry, &hash[i]) { + log = flist_entry(entry, struct gtod_log, list); - printf("function %p, calls %lu\n", log->caller, log->calls); + printf("function %p, calls %lu\n", log->caller, + log->calls); total_calls += log->calls; } } @@ -94,7 +107,7 @@ static void fio_init gtod_init(void) int i; for (i = 0; i < HASH_SIZE; i++) - INIT_LIST_HEAD(&hash[i]); + INIT_FLIST_HEAD(&hash[i]); gtod_inited = 1; } @@ -113,18 +126,218 @@ void fio_gettime(struct timeval *tp, void fio_unused *caller) gtod_log_caller(caller); #endif - if (!clock_gettime_works) { -gtod: + if (fio_tv) { + memcpy(tp, fio_tv, sizeof(*tp)); + return; + } + + switch (fio_clock_source) { + case CS_GTOD: gettimeofday(tp, NULL); - } else { + break; + case CS_CGETTIME: { struct timespec ts; +#ifdef FIO_HAVE_CLOCK_MONOTONIC if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0) { - clock_gettime_works = 0; - goto gtod; +#else + if (clock_gettime(CLOCK_REALTIME, &ts) < 0) { +#endif + log_err("fio: clock_gettime fails\n"); + assert(0); } tp->tv_sec = ts.tv_sec; tp->tv_usec = ts.tv_nsec / 1000; + break; + } +#ifdef ARCH_HAVE_CPU_CLOCK + case CS_CPUCLOCK: { + unsigned long long usecs, t; + + t = get_cpu_clock(); + if (t < last_cycles) { + dprint(FD_TIME, "CPU clock going back in time\n"); + t = last_cycles; + } + + usecs = t / cycles_per_usec; + tp->tv_sec = usecs / 1000000; + tp->tv_usec = usecs % 1000000; + last_cycles = t; + break; + } +#endif + default: + log_err("fio: invalid clock source %d\n", fio_clock_source); + break; + } + + /* + * If Linux is using the tsc clock on non-synced processors, + * sometimes time can appear to drift backwards. Fix that up. + */ + if (last_tv_valid) { + if (tp->tv_sec < last_tv.tv_sec) + tp->tv_sec = last_tv.tv_sec; + else if (last_tv.tv_sec == tp->tv_sec && + tp->tv_usec < last_tv.tv_usec) + tp->tv_usec = last_tv.tv_usec; } + last_tv_valid = 1; + memcpy(&last_tv, tp, sizeof(*tp)); +} + +#ifdef ARCH_HAVE_CPU_CLOCK +static unsigned long get_cycles_per_usec(void) +{ + struct timeval s, e; + unsigned long long c_s, c_e; + + gettimeofday(&s, NULL); + c_s = get_cpu_clock(); + do { + unsigned long long elapsed; + + gettimeofday(&e, NULL); + elapsed = utime_since(&s, &e); + if (elapsed >= 1280) { + c_e = get_cpu_clock(); + break; + } + } while (1); + + return (c_e - c_s) >> 7; +} + +#define NR_TIME_ITERS 50 + +static void calibrate_cpu_clock(void) +{ + double delta, mean, S; + unsigned long avg, cycles[NR_TIME_ITERS]; + int i, samples; + + cycles[0] = get_cycles_per_usec(); + S = delta = mean = 0.0; + for (i = 0; i < NR_TIME_ITERS; i++) { + cycles[i] = get_cycles_per_usec(); + delta = cycles[i] - mean; + if (delta) { + mean += delta / (i + 1.0); + S += delta * (cycles[i] - mean); + } + } + + S = sqrt(S / (NR_TIME_ITERS - 1.0)); + + samples = avg = 0; + for (i = 0; i < NR_TIME_ITERS; i++) { + double this = cycles[i]; + + if ((fmax(this, mean) - fmin(this, mean)) > S) + continue; + samples++; + avg += this; + } + + S /= (double) NR_TIME_ITERS; + mean /= (double) NR_TIME_ITERS; + + for (i = 0; i < NR_TIME_ITERS; i++) + dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10); + + avg /= (samples * 10); + dprint(FD_TIME, "avg: %lu\n", avg); + dprint(FD_TIME, "mean=%f, S=%f\n", mean, S); + + cycles_per_usec = avg; +} +#else +static void calibrate_cpu_clock(void) +{ +} +#endif + +void fio_clock_init(void) +{ + last_tv_valid = 0; + calibrate_cpu_clock(); + + /* + * If the arch sets tsc_reliable != 0, then it must be good enough + * to use as THE clock source. For x86 CPUs, this means the TSC + * runs at a constant rate and is synced across CPU cores. + */ + if (tsc_reliable) { + if (!fio_clock_source_set) + fio_clock_source = CS_CPUCLOCK; + } else if (fio_clock_source == CS_CPUCLOCK) + log_info("fio: clocksource=cpu may not be reliable\n"); +} + +unsigned long long utime_since(struct timeval *s, struct timeval *e) +{ + long sec, usec; + unsigned long long ret; + + sec = e->tv_sec - s->tv_sec; + usec = e->tv_usec - s->tv_usec; + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + /* + * time warp bug on some kernels? + */ + if (sec < 0 || (sec == 0 && usec < 0)) + return 0; + + ret = sec * 1000000ULL + usec; + + return ret; +} + +unsigned long long utime_since_now(struct timeval *s) +{ + struct timeval t; + + fio_gettime(&t, NULL); + return utime_since(s, &t); +} + +unsigned long mtime_since(struct timeval *s, struct timeval *e) +{ + long sec, usec, ret; + + sec = e->tv_sec - s->tv_sec; + usec = e->tv_usec - s->tv_usec; + if (sec > 0 && usec < 0) { + sec--; + usec += 1000000; + } + + if (sec < 0 || (sec == 0 && usec < 0)) + return 0; + + sec *= 1000UL; + usec /= 1000UL; + ret = sec + usec; + + return ret; +} + +unsigned long mtime_since_now(struct timeval *s) +{ + struct timeval t; + void *p = __builtin_return_address(0); + + fio_gettime(&t, p); + return mtime_since(s, &t); +} + +unsigned long time_since_now(struct timeval *s) +{ + return mtime_since_now(s) / 1000; }