Fix a few 4.8 extra anal warnings
[fio.git] / gettime.c
CommitLineData
02bcaa8c 1/*
f5cc024a 2 * Clock functions
02bcaa8c 3 */
f5cc024a 4
02bcaa8c 5#include <unistd.h>
c223da83 6#include <math.h>
02bcaa8c 7#include <sys/time.h>
03e20d68 8#include <time.h>
02bcaa8c
JA
9
10#include "fio.h"
be4ecfdf 11#include "smalloc.h"
02bcaa8c
JA
12
13#include "hash.h"
7d11f871 14#include "os/os.h"
02bcaa8c 15
09a32402 16#ifdef ARCH_HAVE_CPU_CLOCK
c223da83 17static unsigned long cycles_per_usec;
71339117 18static unsigned long inv_cycles_per_usec;
09a32402 19#endif
4de98eb0 20int tsc_reliable = 0;
5d879392
JA
21
22struct tv_valid {
23 struct timeval last_tv;
ba458c2f 24 uint64_t last_cycles;
5d879392
JA
25 int last_tv_valid;
26};
67bf9823 27#ifdef CONFIG_TLS_THREAD
b4ea84da 28static __thread struct tv_valid static_tv_valid;
67bf9823 29#else
5d879392 30static pthread_key_t tv_tls_key;
67bf9823 31#endif
02bcaa8c 32
16de1bf9 33enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
fa80feae 34int fio_clock_source_set = 0;
01423eae 35enum fio_cs fio_clock_source_inited = CS_INVAL;
c223da83 36
02bcaa8c
JA
37#ifdef FIO_DEBUG_TIME
38
39#define HASH_BITS 8
40#define HASH_SIZE (1 << HASH_BITS)
41
01743ee1 42static struct flist_head hash[HASH_SIZE];
02bcaa8c
JA
43static int gtod_inited;
44
45struct gtod_log {
01743ee1 46 struct flist_head list;
02bcaa8c
JA
47 void *caller;
48 unsigned long calls;
49};
50
51static struct gtod_log *find_hash(void *caller)
52{
53 unsigned long h = hash_ptr(caller, HASH_BITS);
01743ee1 54 struct flist_head *entry;
02bcaa8c 55
01743ee1
JA
56 flist_for_each(entry, &hash[h]) {
57 struct gtod_log *log = flist_entry(entry, struct gtod_log,
58 list);
02bcaa8c
JA
59
60 if (log->caller == caller)
61 return log;
62 }
63
64 return NULL;
65}
66
67static struct gtod_log *find_log(void *caller)
68{
69 struct gtod_log *log = find_hash(caller);
70
71 if (!log) {
72 unsigned long h;
73
74 log = malloc(sizeof(*log));
01743ee1 75 INIT_FLIST_HEAD(&log->list);
02bcaa8c
JA
76 log->caller = caller;
77 log->calls = 0;
78
79 h = hash_ptr(caller, HASH_BITS);
01743ee1 80 flist_add_tail(&log->list, &hash[h]);
02bcaa8c
JA
81 }
82
83 return log;
84}
85
86static void gtod_log_caller(void *caller)
87{
88 if (gtod_inited) {
89 struct gtod_log *log = find_log(caller);
90
91 log->calls++;
92 }
93}
94
95static void fio_exit fio_dump_gtod(void)
96{
97 unsigned long total_calls = 0;
98 int i;
99
100 for (i = 0; i < HASH_SIZE; i++) {
01743ee1 101 struct flist_head *entry;
02bcaa8c
JA
102 struct gtod_log *log;
103
01743ee1
JA
104 flist_for_each(entry, &hash[i]) {
105 log = flist_entry(entry, struct gtod_log, list);
02bcaa8c 106
5ec10eaa
JA
107 printf("function %p, calls %lu\n", log->caller,
108 log->calls);
02bcaa8c
JA
109 total_calls += log->calls;
110 }
111 }
112
113 printf("Total %lu gettimeofday\n", total_calls);
114}
115
116static void fio_init gtod_init(void)
117{
118 int i;
119
120 for (i = 0; i < HASH_SIZE; i++)
01743ee1 121 INIT_FLIST_HEAD(&hash[i]);
02bcaa8c
JA
122
123 gtod_inited = 1;
124}
125
126#endif /* FIO_DEBUG_TIME */
127
67bf9823 128#ifdef CONFIG_CLOCK_GETTIME
9ff1c070
JA
129static int fill_clock_gettime(struct timespec *ts)
130{
67bf9823 131#ifdef CONFIG_CLOCK_MONOTONIC
9ff1c070
JA
132 return clock_gettime(CLOCK_MONOTONIC, ts);
133#else
134 return clock_gettime(CLOCK_REALTIME, ts);
135#endif
136}
1e97cce9 137#endif
67bf9823
JA
138
139static void *__fio_gettime(struct timeval *tp)
02bcaa8c 140{
93f0b09a 141 struct tv_valid *tv;
5d879392 142
67bf9823
JA
143#ifdef CONFIG_TLS_THREAD
144 tv = &static_tv_valid;
145#else
93f0b09a 146 tv = pthread_getspecific(tv_tls_key);
67bf9823 147#endif
93f0b09a 148
c223da83 149 switch (fio_clock_source) {
67bf9823 150#ifdef CONFIG_GETTIMEOFDAY
c223da83 151 case CS_GTOD:
02bcaa8c 152 gettimeofday(tp, NULL);
c223da83 153 break;
67bf9823
JA
154#endif
155#ifdef CONFIG_CLOCK_GETTIME
c223da83 156 case CS_CGETTIME: {
02bcaa8c
JA
157 struct timespec ts;
158
9ff1c070 159 if (fill_clock_gettime(&ts) < 0) {
c223da83
JA
160 log_err("fio: clock_gettime fails\n");
161 assert(0);
02bcaa8c
JA
162 }
163
164 tp->tv_sec = ts.tv_sec;
165 tp->tv_usec = ts.tv_nsec / 1000;
c223da83
JA
166 break;
167 }
67bf9823 168#endif
c223da83
JA
169#ifdef ARCH_HAVE_CPU_CLOCK
170 case CS_CPUCLOCK: {
ba458c2f 171 uint64_t usecs, t;
c223da83
JA
172
173 t = get_cpu_clock();
93f0b09a 174 if (tv && t < tv->last_cycles) {
c223da83 175 dprint(FD_TIME, "CPU clock going back in time\n");
93f0b09a
JA
176 t = tv->last_cycles;
177 } else if (tv)
178 tv->last_cycles = t;
c223da83 179
71339117 180 usecs = (t * inv_cycles_per_usec) / 16777216UL;
c223da83
JA
181 tp->tv_sec = usecs / 1000000;
182 tp->tv_usec = usecs % 1000000;
c223da83
JA
183 break;
184 }
185#endif
186 default:
187 log_err("fio: invalid clock source %d\n", fio_clock_source);
188 break;
02bcaa8c 189 }
3e488920 190
67bf9823
JA
191 return tv;
192}
193
194#ifdef FIO_DEBUG_TIME
195void fio_gettime(struct timeval *tp, void *caller)
196#else
197void fio_gettime(struct timeval *tp, void fio_unused *caller)
198#endif
199{
200 struct tv_valid *tv;
201
202#ifdef FIO_DEBUG_TIME
203 if (!caller)
204 caller = __builtin_return_address(0);
205
206 gtod_log_caller(caller);
207#endif
208 if (fio_tv) {
209 memcpy(tp, fio_tv, sizeof(*tp));
210 return;
211 }
212
213 tv = __fio_gettime(tp);
214
3e488920
JA
215 /*
216 * If Linux is using the tsc clock on non-synced processors,
217 * sometimes time can appear to drift backwards. Fix that up.
218 */
93f0b09a
JA
219 if (tv) {
220 if (tv->last_tv_valid) {
221 if (tp->tv_sec < tv->last_tv.tv_sec)
222 tp->tv_sec = tv->last_tv.tv_sec;
223 else if (tv->last_tv.tv_sec == tp->tv_sec &&
224 tp->tv_usec < tv->last_tv.tv_usec)
225 tp->tv_usec = tv->last_tv.tv_usec;
5d879392 226 }
93f0b09a
JA
227 tv->last_tv_valid = 1;
228 memcpy(&tv->last_tv, tp, sizeof(*tp));
3e488920 229 }
02bcaa8c 230}
be4ecfdf 231
09a32402 232#ifdef ARCH_HAVE_CPU_CLOCK
c223da83
JA
233static unsigned long get_cycles_per_usec(void)
234{
235 struct timeval s, e;
ba458c2f 236 uint64_t c_s, c_e;
67bf9823 237 enum fio_cs old_cs = fio_clock_source;
c223da83 238
67bf9823
JA
239#ifdef CONFIG_CLOCK_GETTIME
240 fio_clock_source = CS_CGETTIME;
241#else
242 fio_clock_source = CS_GTOD;
243#endif
244 __fio_gettime(&s);
9ff1c070 245
c223da83
JA
246 c_s = get_cpu_clock();
247 do {
ba458c2f 248 uint64_t elapsed;
c223da83 249
67bf9823 250 __fio_gettime(&e);
9ff1c070 251
c223da83 252 elapsed = utime_since(&s, &e);
486332e5 253 if (elapsed >= 1280) {
c223da83
JA
254 c_e = get_cpu_clock();
255 break;
256 }
257 } while (1);
258
67bf9823 259 fio_clock_source = old_cs;
c011000d 260 return (c_e - c_s + 127) >> 7;
c223da83
JA
261}
262
fa80feae
JA
263#define NR_TIME_ITERS 50
264
e259879e 265static int calibrate_cpu_clock(void)
c223da83
JA
266{
267 double delta, mean, S;
ba458c2f 268 uint64_t avg, cycles[NR_TIME_ITERS];
c223da83
JA
269 int i, samples;
270
c223da83
JA
271 cycles[0] = get_cycles_per_usec();
272 S = delta = mean = 0.0;
fa80feae 273 for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83
JA
274 cycles[i] = get_cycles_per_usec();
275 delta = cycles[i] - mean;
276 if (delta) {
277 mean += delta / (i + 1.0);
278 S += delta * (cycles[i] - mean);
279 }
280 }
281
e259879e
JA
282 /*
283 * The most common platform clock breakage is returning zero
284 * indefinitely. Check for that and return failure.
285 */
286 if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
287 return 1;
288
fa80feae 289 S = sqrt(S / (NR_TIME_ITERS - 1.0));
c223da83
JA
290
291 samples = avg = 0;
fa80feae 292 for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83
JA
293 double this = cycles[i];
294
03e20d68 295 if ((fmax(this, mean) - fmin(this, mean)) > S)
c223da83
JA
296 continue;
297 samples++;
298 avg += this;
299 }
300
fa80feae 301 S /= (double) NR_TIME_ITERS;
89db727d 302 mean /= 10.0;
c223da83 303
fa80feae 304 for (i = 0; i < NR_TIME_ITERS; i++)
4b91ee8f
JA
305 dprint(FD_TIME, "cycles[%d]=%llu\n", i,
306 (unsigned long long) cycles[i] / 10);
c223da83 307
d7abad3d 308 avg /= samples;
b0ff22d7 309 avg = (avg + 5) / 10;
4b91ee8f 310 dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
c223da83
JA
311 dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);
312
313 cycles_per_usec = avg;
71339117
JA
314 inv_cycles_per_usec = 16777216UL / cycles_per_usec;
315 dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
e259879e 316 return 0;
09a32402
JA
317}
318#else
e259879e 319static int calibrate_cpu_clock(void)
09a32402 320{
e259879e 321 return 1;
09a32402
JA
322}
323#endif
324
67bf9823 325#ifndef CONFIG_TLS_THREAD
5d879392
JA
326void fio_local_clock_init(int is_thread)
327{
328 struct tv_valid *t;
329
330 t = calloc(sizeof(*t), 1);
331 if (pthread_setspecific(tv_tls_key, t))
332 log_err("fio: can't set TLS key\n");
333}
334
335static void kill_tv_tls_key(void *data)
336{
337 free(data);
338}
67bf9823
JA
339#else
340void fio_local_clock_init(int is_thread)
341{
342}
343#endif
5d879392 344
09a32402
JA
345void fio_clock_init(void)
346{
01423eae
JA
347 if (fio_clock_source == fio_clock_source_inited)
348 return;
349
67bf9823 350#ifndef CONFIG_TLS_THREAD
5d879392
JA
351 if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
352 log_err("fio: can't create TLS key\n");
67bf9823 353#endif
5d879392 354
01423eae 355 fio_clock_source_inited = fio_clock_source;
e259879e
JA
356
357 if (calibrate_cpu_clock())
358 tsc_reliable = 0;
fa80feae
JA
359
360 /*
361 * If the arch sets tsc_reliable != 0, then it must be good enough
362 * to use as THE clock source. For x86 CPUs, this means the TSC
363 * runs at a constant rate and is synced across CPU cores.
364 */
365 if (tsc_reliable) {
366 if (!fio_clock_source_set)
367 fio_clock_source = CS_CPUCLOCK;
368 } else if (fio_clock_source == CS_CPUCLOCK)
369 log_info("fio: clocksource=cpu may not be reliable\n");
c223da83
JA
370}
371
aa60bc58 372uint64_t utime_since(struct timeval *s, struct timeval *e)
be4ecfdf 373{
39ab7da2 374 long sec, usec;
aa60bc58 375 uint64_t ret;
39ab7da2
JA
376
377 sec = e->tv_sec - s->tv_sec;
378 usec = e->tv_usec - s->tv_usec;
379 if (sec > 0 && usec < 0) {
380 sec--;
381 usec += 1000000;
382 }
383
384 /*
385 * time warp bug on some kernels?
386 */
387 if (sec < 0 || (sec == 0 && usec < 0))
388 return 0;
389
390 ret = sec * 1000000ULL + usec;
391
392 return ret;
be4ecfdf
JA
393}
394
aa60bc58 395uint64_t utime_since_now(struct timeval *s)
be4ecfdf 396{
39ab7da2
JA
397 struct timeval t;
398
399 fio_gettime(&t, NULL);
400 return utime_since(s, &t);
be4ecfdf 401}
783a3eb1 402
aa60bc58 403uint64_t mtime_since(struct timeval *s, struct timeval *e)
783a3eb1 404{
39ab7da2 405 long sec, usec, ret;
783a3eb1 406
39ab7da2
JA
407 sec = e->tv_sec - s->tv_sec;
408 usec = e->tv_usec - s->tv_usec;
409 if (sec > 0 && usec < 0) {
410 sec--;
411 usec += 1000000;
783a3eb1
JA
412 }
413
39ab7da2
JA
414 if (sec < 0 || (sec == 0 && usec < 0))
415 return 0;
416
417 sec *= 1000UL;
418 usec /= 1000UL;
419 ret = sec + usec;
420
421 return ret;
783a3eb1
JA
422}
423
aa60bc58 424uint64_t mtime_since_now(struct timeval *s)
783a3eb1 425{
39ab7da2
JA
426 struct timeval t;
427 void *p = __builtin_return_address(0);
783a3eb1 428
39ab7da2
JA
429 fio_gettime(&t, p);
430 return mtime_since(s, &t);
431}
783a3eb1 432
aa60bc58 433uint64_t time_since_now(struct timeval *s)
39ab7da2
JA
434{
435 return mtime_since_now(s) / 1000;
783a3eb1 436}
7d11f871 437
67bf9823
JA
438#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
439 defined(CONFIG_SFAA)
7d11f871
JA
440
441#define CLOCK_ENTRIES 100000
442
443struct clock_entry {
58002f9a
JA
444 uint32_t seq;
445 uint32_t cpu;
ba458c2f 446 uint64_t tsc;
7d11f871
JA
447};
448
449struct clock_thread {
450 pthread_t thread;
451 int cpu;
452 pthread_mutex_t lock;
453 pthread_mutex_t started;
58002f9a 454 uint32_t *seq;
7d11f871
JA
455 struct clock_entry *entries;
456};
457
58002f9a 458static inline uint32_t atomic32_inc_return(uint32_t *seq)
7d11f871
JA
459{
460 return 1 + __sync_fetch_and_add(seq, 1);
461}
462
463static void *clock_thread_fn(void *data)
464{
465 struct clock_thread *t = data;
466 struct clock_entry *c;
467 os_cpu_mask_t cpu_mask;
58002f9a 468 uint32_t last_seq;
7d11f871
JA
469 int i;
470
471 memset(&cpu_mask, 0, sizeof(cpu_mask));
472 fio_cpu_set(&cpu_mask, t->cpu);
473
474 if (fio_setaffinity(gettid(), cpu_mask) == -1) {
475 log_err("clock setaffinity failed\n");
476 return (void *) 1;
477 }
478
7d11f871 479 pthread_mutex_lock(&t->lock);
b9b3498e 480 pthread_mutex_unlock(&t->started);
7d11f871 481
58002f9a 482 last_seq = 0;
7d11f871
JA
483 c = &t->entries[0];
484 for (i = 0; i < CLOCK_ENTRIES; i++, c++) {
58002f9a
JA
485 uint32_t seq;
486 uint64_t tsc;
7d11f871
JA
487
488 c->cpu = t->cpu;
489 do {
58002f9a
JA
490 seq = atomic32_inc_return(t->seq);
491 if (seq < last_seq)
492 break;
7d11f871
JA
493 tsc = get_cpu_clock();
494 } while (seq != *t->seq);
495
496 c->seq = seq;
497 c->tsc = tsc;
498 }
499
0f78b22c
JA
500 log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu,
501 (unsigned long long) t->entries[i - 1].tsc - t->entries[0].tsc);
58002f9a 502
e259879e
JA
503 /*
504 * The most common platform clock breakage is returning zero
505 * indefinitely. Check for that and return failure.
506 */
58002f9a 507 if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
e259879e
JA
508 return (void *) 1;
509
7d11f871
JA
510 return NULL;
511}
512
513static int clock_cmp(const void *p1, const void *p2)
514{
515 const struct clock_entry *c1 = p1;
516 const struct clock_entry *c2 = p2;
517
b9b3498e
JA
518 if (c1->seq == c2->seq)
519 log_err("cs: bug in atomic sequence!\n");
520
7d11f871
JA
521 return c1->seq - c2->seq;
522}
523
524int fio_monotonic_clocktest(void)
525{
526 struct clock_thread *threads;
527 unsigned int nr_cpus = cpus_online();
528 struct clock_entry *entries;
529 unsigned long tentries, failed;
80da8a8f 530 struct clock_entry *prev, *this;
58002f9a 531 uint32_t seq = 0;
7d11f871
JA
532 int i;
533
d5e3f5d8
JA
534 log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
535
4f1d43c2
JA
536 fio_debug |= 1U << FD_TIME;
537 calibrate_cpu_clock();
538 fio_debug &= ~(1U << FD_TIME);
539
7d11f871
JA
540 threads = malloc(nr_cpus * sizeof(struct clock_thread));
541 tentries = CLOCK_ENTRIES * nr_cpus;
542 entries = malloc(tentries * sizeof(struct clock_entry));
543
544 log_info("cs: Testing %u CPUs\n", nr_cpus);
545
546 for (i = 0; i < nr_cpus; i++) {
547 struct clock_thread *t = &threads[i];
548
549 t->cpu = i;
550 t->seq = &seq;
551 t->entries = &entries[i * CLOCK_ENTRIES];
552 pthread_mutex_init(&t->lock, NULL);
553 pthread_mutex_init(&t->started, NULL);
554 pthread_mutex_lock(&t->lock);
555 pthread_create(&t->thread, NULL, clock_thread_fn, t);
556 }
557
558 for (i = 0; i < nr_cpus; i++) {
559 struct clock_thread *t = &threads[i];
560
561 pthread_mutex_lock(&t->started);
562 }
563
564 for (i = 0; i < nr_cpus; i++) {
565 struct clock_thread *t = &threads[i];
566
567 pthread_mutex_unlock(&t->lock);
568 }
569
570 for (failed = i = 0; i < nr_cpus; i++) {
571 struct clock_thread *t = &threads[i];
572 void *ret;
573
574 pthread_join(t->thread, &ret);
575 if (ret)
576 failed++;
577 }
578 free(threads);
579
580 if (failed) {
581 log_err("Clocksource test: %u threads failed\n", failed);
582 goto err;
583 }
584
585 qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
586
587 for (failed = i = 0; i < tentries; i++) {
80da8a8f 588 this = &entries[i];
7d11f871
JA
589
590 if (!i) {
591 prev = this;
592 continue;
593 }
594
595 if (prev->tsc > this->tsc) {
596 uint64_t diff = prev->tsc - this->tsc;
597
598 log_info("cs: CPU clock mismatch (diff=%lu):\n", diff);
599 log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", prev->cpu, prev->tsc, prev->seq);
600 log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", this->cpu, this->tsc, this->seq);
601 failed++;
602 }
603
604 prev = this;
605 }
606
607 if (failed)
608 log_info("cs: Failed: %lu\n", failed);
609 else
610 log_info("cs: Pass!\n");
611
612err:
613 free(entries);
614 return !!failed;
615}
616
617#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
618
619int fio_monotonic_clocktest(void)
620{
621 log_info("cs: current platform does not support CPU clocks\n");
622 return 0;
623}
624
625#endif