clock: hardwire tsc as unreliable on Solaris for now
[fio.git] / gettime.c
CommitLineData
02bcaa8c 1/*
f5cc024a 2 * Clock functions
02bcaa8c 3 */
f5cc024a 4
02bcaa8c 5#include <unistd.h>
c223da83 6#include <math.h>
02bcaa8c 7#include <sys/time.h>
03e20d68 8#include <time.h>
02bcaa8c
JA
9
10#include "fio.h"
be4ecfdf 11#include "smalloc.h"
02bcaa8c
JA
12
13#include "hash.h"
7d11f871 14#include "os/os.h"
02bcaa8c 15
09a32402 16#ifdef ARCH_HAVE_CPU_CLOCK
c223da83 17static unsigned long cycles_per_usec;
71339117 18static unsigned long inv_cycles_per_usec;
09a32402 19#endif
4de98eb0 20int tsc_reliable = 0;
5d879392
JA
21
22struct tv_valid {
23 struct timeval last_tv;
ba458c2f 24 uint64_t last_cycles;
5d879392
JA
25 int last_tv_valid;
26};
67bf9823
JA
27#ifdef CONFIG_TLS_THREAD
28static struct tv_valid __thread static_tv_valid;
29#else
5d879392 30static pthread_key_t tv_tls_key;
67bf9823 31#endif
02bcaa8c 32
16de1bf9 33enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
fa80feae 34int fio_clock_source_set = 0;
01423eae 35enum fio_cs fio_clock_source_inited = CS_INVAL;
c223da83 36
02bcaa8c
JA
37#ifdef FIO_DEBUG_TIME
38
39#define HASH_BITS 8
40#define HASH_SIZE (1 << HASH_BITS)
41
01743ee1 42static struct flist_head hash[HASH_SIZE];
02bcaa8c
JA
43static int gtod_inited;
44
45struct gtod_log {
01743ee1 46 struct flist_head list;
02bcaa8c
JA
47 void *caller;
48 unsigned long calls;
49};
50
51static struct gtod_log *find_hash(void *caller)
52{
53 unsigned long h = hash_ptr(caller, HASH_BITS);
01743ee1 54 struct flist_head *entry;
02bcaa8c 55
01743ee1
JA
56 flist_for_each(entry, &hash[h]) {
57 struct gtod_log *log = flist_entry(entry, struct gtod_log,
58 list);
02bcaa8c
JA
59
60 if (log->caller == caller)
61 return log;
62 }
63
64 return NULL;
65}
66
67static struct gtod_log *find_log(void *caller)
68{
69 struct gtod_log *log = find_hash(caller);
70
71 if (!log) {
72 unsigned long h;
73
74 log = malloc(sizeof(*log));
01743ee1 75 INIT_FLIST_HEAD(&log->list);
02bcaa8c
JA
76 log->caller = caller;
77 log->calls = 0;
78
79 h = hash_ptr(caller, HASH_BITS);
01743ee1 80 flist_add_tail(&log->list, &hash[h]);
02bcaa8c
JA
81 }
82
83 return log;
84}
85
86static void gtod_log_caller(void *caller)
87{
88 if (gtod_inited) {
89 struct gtod_log *log = find_log(caller);
90
91 log->calls++;
92 }
93}
94
95static void fio_exit fio_dump_gtod(void)
96{
97 unsigned long total_calls = 0;
98 int i;
99
100 for (i = 0; i < HASH_SIZE; i++) {
01743ee1 101 struct flist_head *entry;
02bcaa8c
JA
102 struct gtod_log *log;
103
01743ee1
JA
104 flist_for_each(entry, &hash[i]) {
105 log = flist_entry(entry, struct gtod_log, list);
02bcaa8c 106
5ec10eaa
JA
107 printf("function %p, calls %lu\n", log->caller,
108 log->calls);
02bcaa8c
JA
109 total_calls += log->calls;
110 }
111 }
112
113 printf("Total %lu gettimeofday\n", total_calls);
114}
115
116static void fio_init gtod_init(void)
117{
118 int i;
119
120 for (i = 0; i < HASH_SIZE; i++)
01743ee1 121 INIT_FLIST_HEAD(&hash[i]);
02bcaa8c
JA
122
123 gtod_inited = 1;
124}
125
126#endif /* FIO_DEBUG_TIME */
127
67bf9823 128#ifdef CONFIG_CLOCK_GETTIME
9ff1c070
JA
129static int fill_clock_gettime(struct timespec *ts)
130{
67bf9823 131#ifdef CONFIG_CLOCK_MONOTONIC
9ff1c070
JA
132 return clock_gettime(CLOCK_MONOTONIC, ts);
133#else
134 return clock_gettime(CLOCK_REALTIME, ts);
135#endif
136}
1e97cce9 137#endif
67bf9823
JA
138
139static void *__fio_gettime(struct timeval *tp)
02bcaa8c 140{
93f0b09a 141 struct tv_valid *tv;
5d879392 142
67bf9823
JA
143#ifdef CONFIG_TLS_THREAD
144 tv = &static_tv_valid;
145#else
93f0b09a 146 tv = pthread_getspecific(tv_tls_key);
67bf9823 147#endif
93f0b09a 148
c223da83 149 switch (fio_clock_source) {
67bf9823 150#ifdef CONFIG_GETTIMEOFDAY
c223da83 151 case CS_GTOD:
02bcaa8c 152 gettimeofday(tp, NULL);
c223da83 153 break;
67bf9823
JA
154#endif
155#ifdef CONFIG_CLOCK_GETTIME
c223da83 156 case CS_CGETTIME: {
02bcaa8c
JA
157 struct timespec ts;
158
9ff1c070 159 if (fill_clock_gettime(&ts) < 0) {
c223da83
JA
160 log_err("fio: clock_gettime fails\n");
161 assert(0);
02bcaa8c
JA
162 }
163
164 tp->tv_sec = ts.tv_sec;
165 tp->tv_usec = ts.tv_nsec / 1000;
c223da83
JA
166 break;
167 }
67bf9823 168#endif
c223da83
JA
169#ifdef ARCH_HAVE_CPU_CLOCK
170 case CS_CPUCLOCK: {
ba458c2f 171 uint64_t usecs, t;
c223da83
JA
172
173 t = get_cpu_clock();
93f0b09a 174 if (tv && t < tv->last_cycles) {
c223da83 175 dprint(FD_TIME, "CPU clock going back in time\n");
93f0b09a
JA
176 t = tv->last_cycles;
177 } else if (tv)
178 tv->last_cycles = t;
c223da83 179
71339117 180 usecs = (t * inv_cycles_per_usec) / 16777216UL;
c223da83
JA
181 tp->tv_sec = usecs / 1000000;
182 tp->tv_usec = usecs % 1000000;
c223da83
JA
183 break;
184 }
185#endif
186 default:
187 log_err("fio: invalid clock source %d\n", fio_clock_source);
188 break;
02bcaa8c 189 }
3e488920 190
67bf9823
JA
191 return tv;
192}
193
194#ifdef FIO_DEBUG_TIME
195void fio_gettime(struct timeval *tp, void *caller)
196#else
197void fio_gettime(struct timeval *tp, void fio_unused *caller)
198#endif
199{
200 struct tv_valid *tv;
201
202#ifdef FIO_DEBUG_TIME
203 if (!caller)
204 caller = __builtin_return_address(0);
205
206 gtod_log_caller(caller);
207#endif
208 if (fio_tv) {
209 memcpy(tp, fio_tv, sizeof(*tp));
210 return;
211 }
212
213 tv = __fio_gettime(tp);
214
3e488920
JA
215 /*
216 * If Linux is using the tsc clock on non-synced processors,
217 * sometimes time can appear to drift backwards. Fix that up.
218 */
93f0b09a
JA
219 if (tv) {
220 if (tv->last_tv_valid) {
221 if (tp->tv_sec < tv->last_tv.tv_sec)
222 tp->tv_sec = tv->last_tv.tv_sec;
223 else if (tv->last_tv.tv_sec == tp->tv_sec &&
224 tp->tv_usec < tv->last_tv.tv_usec)
225 tp->tv_usec = tv->last_tv.tv_usec;
5d879392 226 }
93f0b09a
JA
227 tv->last_tv_valid = 1;
228 memcpy(&tv->last_tv, tp, sizeof(*tp));
3e488920 229 }
02bcaa8c 230}
be4ecfdf 231
09a32402 232#ifdef ARCH_HAVE_CPU_CLOCK
c223da83
JA
233static unsigned long get_cycles_per_usec(void)
234{
235 struct timeval s, e;
ba458c2f 236 uint64_t c_s, c_e;
67bf9823 237 enum fio_cs old_cs = fio_clock_source;
c223da83 238
67bf9823
JA
239#ifdef CONFIG_CLOCK_GETTIME
240 fio_clock_source = CS_CGETTIME;
241#else
242 fio_clock_source = CS_GTOD;
243#endif
244 __fio_gettime(&s);
9ff1c070 245
c223da83
JA
246 c_s = get_cpu_clock();
247 do {
ba458c2f 248 uint64_t elapsed;
c223da83 249
67bf9823 250 __fio_gettime(&e);
9ff1c070 251
c223da83 252 elapsed = utime_since(&s, &e);
486332e5 253 if (elapsed >= 1280) {
c223da83
JA
254 c_e = get_cpu_clock();
255 break;
256 }
257 } while (1);
258
67bf9823 259 fio_clock_source = old_cs;
c011000d 260 return (c_e - c_s + 127) >> 7;
c223da83
JA
261}
262
fa80feae
JA
263#define NR_TIME_ITERS 50
264
09a32402 265static void calibrate_cpu_clock(void)
c223da83
JA
266{
267 double delta, mean, S;
ba458c2f 268 uint64_t avg, cycles[NR_TIME_ITERS];
c223da83
JA
269 int i, samples;
270
c223da83
JA
271 cycles[0] = get_cycles_per_usec();
272 S = delta = mean = 0.0;
fa80feae 273 for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83
JA
274 cycles[i] = get_cycles_per_usec();
275 delta = cycles[i] - mean;
276 if (delta) {
277 mean += delta / (i + 1.0);
278 S += delta * (cycles[i] - mean);
279 }
280 }
281
fa80feae 282 S = sqrt(S / (NR_TIME_ITERS - 1.0));
c223da83
JA
283
284 samples = avg = 0;
fa80feae 285 for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83
JA
286 double this = cycles[i];
287
03e20d68 288 if ((fmax(this, mean) - fmin(this, mean)) > S)
c223da83
JA
289 continue;
290 samples++;
291 avg += this;
292 }
293
fa80feae 294 S /= (double) NR_TIME_ITERS;
89db727d 295 mean /= 10.0;
c223da83 296
fa80feae 297 for (i = 0; i < NR_TIME_ITERS; i++)
c223da83
JA
298 dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10);
299
d7abad3d 300 avg /= samples;
b0ff22d7 301 avg = (avg + 5) / 10;
c223da83
JA
302 dprint(FD_TIME, "avg: %lu\n", avg);
303 dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);
304
305 cycles_per_usec = avg;
71339117
JA
306 inv_cycles_per_usec = 16777216UL / cycles_per_usec;
307 dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
09a32402
JA
308}
309#else
310static void calibrate_cpu_clock(void)
311{
312}
313#endif
314
67bf9823 315#ifndef CONFIG_TLS_THREAD
5d879392
JA
316void fio_local_clock_init(int is_thread)
317{
318 struct tv_valid *t;
319
320 t = calloc(sizeof(*t), 1);
321 if (pthread_setspecific(tv_tls_key, t))
322 log_err("fio: can't set TLS key\n");
323}
324
325static void kill_tv_tls_key(void *data)
326{
327 free(data);
328}
67bf9823
JA
329#else
330void fio_local_clock_init(int is_thread)
331{
332}
333#endif
5d879392 334
09a32402
JA
335void fio_clock_init(void)
336{
01423eae
JA
337 if (fio_clock_source == fio_clock_source_inited)
338 return;
339
67bf9823 340#ifndef CONFIG_TLS_THREAD
5d879392
JA
341 if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
342 log_err("fio: can't create TLS key\n");
67bf9823 343#endif
5d879392 344
372fd659
JA
345 /*
346 * Probably an AMD issue, will need to investigate. Until that
347 * is done, disable the CPU clock.
348 */
349#if FIO_OS == os_solaris
350 tsc_reliable = 0;
351#endif
352
01423eae 353 fio_clock_source_inited = fio_clock_source;
09a32402 354 calibrate_cpu_clock();
fa80feae
JA
355
356 /*
357 * If the arch sets tsc_reliable != 0, then it must be good enough
358 * to use as THE clock source. For x86 CPUs, this means the TSC
359 * runs at a constant rate and is synced across CPU cores.
360 */
361 if (tsc_reliable) {
362 if (!fio_clock_source_set)
363 fio_clock_source = CS_CPUCLOCK;
364 } else if (fio_clock_source == CS_CPUCLOCK)
365 log_info("fio: clocksource=cpu may not be reliable\n");
c223da83
JA
366}
367
aa60bc58 368uint64_t utime_since(struct timeval *s, struct timeval *e)
be4ecfdf 369{
39ab7da2 370 long sec, usec;
aa60bc58 371 uint64_t ret;
39ab7da2
JA
372
373 sec = e->tv_sec - s->tv_sec;
374 usec = e->tv_usec - s->tv_usec;
375 if (sec > 0 && usec < 0) {
376 sec--;
377 usec += 1000000;
378 }
379
380 /*
381 * time warp bug on some kernels?
382 */
383 if (sec < 0 || (sec == 0 && usec < 0))
384 return 0;
385
386 ret = sec * 1000000ULL + usec;
387
388 return ret;
be4ecfdf
JA
389}
390
aa60bc58 391uint64_t utime_since_now(struct timeval *s)
be4ecfdf 392{
39ab7da2
JA
393 struct timeval t;
394
395 fio_gettime(&t, NULL);
396 return utime_since(s, &t);
be4ecfdf 397}
783a3eb1 398
aa60bc58 399uint64_t mtime_since(struct timeval *s, struct timeval *e)
783a3eb1 400{
39ab7da2 401 long sec, usec, ret;
783a3eb1 402
39ab7da2
JA
403 sec = e->tv_sec - s->tv_sec;
404 usec = e->tv_usec - s->tv_usec;
405 if (sec > 0 && usec < 0) {
406 sec--;
407 usec += 1000000;
783a3eb1
JA
408 }
409
39ab7da2
JA
410 if (sec < 0 || (sec == 0 && usec < 0))
411 return 0;
412
413 sec *= 1000UL;
414 usec /= 1000UL;
415 ret = sec + usec;
416
417 return ret;
783a3eb1
JA
418}
419
aa60bc58 420uint64_t mtime_since_now(struct timeval *s)
783a3eb1 421{
39ab7da2
JA
422 struct timeval t;
423 void *p = __builtin_return_address(0);
783a3eb1 424
39ab7da2
JA
425 fio_gettime(&t, p);
426 return mtime_since(s, &t);
427}
783a3eb1 428
aa60bc58 429uint64_t time_since_now(struct timeval *s)
39ab7da2
JA
430{
431 return mtime_since_now(s) / 1000;
783a3eb1 432}
7d11f871 433
67bf9823
JA
434#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
435 defined(CONFIG_SFAA)
7d11f871
JA
436
437#define CLOCK_ENTRIES 100000
438
439struct clock_entry {
ba458c2f
JA
440 uint64_t seq;
441 uint64_t tsc;
442 uint64_t cpu;
7d11f871
JA
443};
444
445struct clock_thread {
446 pthread_t thread;
447 int cpu;
448 pthread_mutex_t lock;
449 pthread_mutex_t started;
450 uint64_t *seq;
451 struct clock_entry *entries;
452};
453
454static inline uint64_t atomic64_inc_return(uint64_t *seq)
455{
456 return 1 + __sync_fetch_and_add(seq, 1);
457}
458
459static void *clock_thread_fn(void *data)
460{
461 struct clock_thread *t = data;
462 struct clock_entry *c;
463 os_cpu_mask_t cpu_mask;
464 int i;
465
466 memset(&cpu_mask, 0, sizeof(cpu_mask));
467 fio_cpu_set(&cpu_mask, t->cpu);
468
469 if (fio_setaffinity(gettid(), cpu_mask) == -1) {
470 log_err("clock setaffinity failed\n");
471 return (void *) 1;
472 }
473
7d11f871 474 pthread_mutex_lock(&t->lock);
b9b3498e 475 pthread_mutex_unlock(&t->started);
7d11f871
JA
476
477 c = &t->entries[0];
478 for (i = 0; i < CLOCK_ENTRIES; i++, c++) {
479 uint64_t seq, tsc;
480
481 c->cpu = t->cpu;
482 do {
483 seq = atomic64_inc_return(t->seq);
484 tsc = get_cpu_clock();
485 } while (seq != *t->seq);
486
487 c->seq = seq;
488 c->tsc = tsc;
489 }
490
491 log_info("cs: cpu%3d: %lu clocks seen\n", t->cpu, t->entries[CLOCK_ENTRIES - 1].tsc - t->entries[0].tsc);
492 return NULL;
493}
494
495static int clock_cmp(const void *p1, const void *p2)
496{
497 const struct clock_entry *c1 = p1;
498 const struct clock_entry *c2 = p2;
499
b9b3498e
JA
500 if (c1->seq == c2->seq)
501 log_err("cs: bug in atomic sequence!\n");
502
7d11f871
JA
503 return c1->seq - c2->seq;
504}
505
506int fio_monotonic_clocktest(void)
507{
508 struct clock_thread *threads;
509 unsigned int nr_cpus = cpus_online();
510 struct clock_entry *entries;
511 unsigned long tentries, failed;
512 uint64_t seq = 0;
513 int i;
514
d5e3f5d8
JA
515 log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
516
4f1d43c2
JA
517 fio_debug |= 1U << FD_TIME;
518 calibrate_cpu_clock();
519 fio_debug &= ~(1U << FD_TIME);
520
7d11f871
JA
521 threads = malloc(nr_cpus * sizeof(struct clock_thread));
522 tentries = CLOCK_ENTRIES * nr_cpus;
523 entries = malloc(tentries * sizeof(struct clock_entry));
524
525 log_info("cs: Testing %u CPUs\n", nr_cpus);
526
527 for (i = 0; i < nr_cpus; i++) {
528 struct clock_thread *t = &threads[i];
529
530 t->cpu = i;
531 t->seq = &seq;
532 t->entries = &entries[i * CLOCK_ENTRIES];
533 pthread_mutex_init(&t->lock, NULL);
534 pthread_mutex_init(&t->started, NULL);
535 pthread_mutex_lock(&t->lock);
536 pthread_create(&t->thread, NULL, clock_thread_fn, t);
537 }
538
539 for (i = 0; i < nr_cpus; i++) {
540 struct clock_thread *t = &threads[i];
541
542 pthread_mutex_lock(&t->started);
543 }
544
545 for (i = 0; i < nr_cpus; i++) {
546 struct clock_thread *t = &threads[i];
547
548 pthread_mutex_unlock(&t->lock);
549 }
550
551 for (failed = i = 0; i < nr_cpus; i++) {
552 struct clock_thread *t = &threads[i];
553 void *ret;
554
555 pthread_join(t->thread, &ret);
556 if (ret)
557 failed++;
558 }
559 free(threads);
560
561 if (failed) {
562 log_err("Clocksource test: %u threads failed\n", failed);
563 goto err;
564 }
565
566 qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
567
568 for (failed = i = 0; i < tentries; i++) {
569 struct clock_entry *prev, *this = &entries[i];
570
571 if (!i) {
572 prev = this;
573 continue;
574 }
575
576 if (prev->tsc > this->tsc) {
577 uint64_t diff = prev->tsc - this->tsc;
578
579 log_info("cs: CPU clock mismatch (diff=%lu):\n", diff);
580 log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", prev->cpu, prev->tsc, prev->seq);
581 log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", this->cpu, this->tsc, this->seq);
582 failed++;
583 }
584
585 prev = this;
586 }
587
588 if (failed)
589 log_info("cs: Failed: %lu\n", failed);
590 else
591 log_info("cs: Pass!\n");
592
593err:
594 free(entries);
595 return !!failed;
596}
597
598#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
599
600int fio_monotonic_clocktest(void)
601{
602 log_info("cs: current platform does not support CPU clocks\n");
603 return 0;
604}
605
606#endif