Merge branch 'master' of https://github.com/ben-ihelputech/fio
[fio.git] / gettime.c
CommitLineData
02bcaa8c 1/*
f5cc024a 2 * Clock functions
02bcaa8c 3 */
f5cc024a 4
c223da83 5#include <math.h>
02bcaa8c
JA
6
7#include "fio.h"
7d11f871 8#include "os/os.h"
02bcaa8c 9
dac499a0
AV
10#if defined(ARCH_HAVE_CPU_CLOCK)
11#ifndef ARCH_CPU_CLOCK_CYCLES_PER_USEC
cae9edd9 12static unsigned long long cycles_per_msec;
6d02b37b
VF
13static unsigned long long cycles_start;
14static unsigned long long clock_mult;
15static unsigned long long max_cycles_mask;
16static unsigned long long nsecs_for_max_cycles;
17static unsigned int clock_shift;
18static unsigned int max_cycles_shift;
19#define MAX_CLOCK_SEC 60*60
96170421
CE
20#endif
21#ifdef ARCH_CPU_CLOCK_WRAPS
6d02b37b 22static unsigned int cycles_wrap;
09a32402 23#endif
dac499a0 24#endif
24575392 25bool tsc_reliable = false;
5d879392
JA
26
27struct tv_valid {
02dcf81c 28 int warned;
5d879392 29};
03be65c9 30#ifdef ARCH_HAVE_CPU_CLOCK
67bf9823 31#ifdef CONFIG_TLS_THREAD
b4ea84da 32static __thread struct tv_valid static_tv_valid;
67bf9823 33#else
5d879392 34static pthread_key_t tv_tls_key;
67bf9823 35#endif
03be65c9 36#endif
02bcaa8c 37
16de1bf9 38enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
fa80feae 39int fio_clock_source_set = 0;
10aa136b 40static enum fio_cs fio_clock_source_inited = CS_INVAL;
c223da83 41
02bcaa8c
JA
42#ifdef FIO_DEBUG_TIME
43
44#define HASH_BITS 8
45#define HASH_SIZE (1 << HASH_BITS)
46
01743ee1 47static struct flist_head hash[HASH_SIZE];
02bcaa8c
JA
48static int gtod_inited;
49
50struct gtod_log {
01743ee1 51 struct flist_head list;
02bcaa8c
JA
52 void *caller;
53 unsigned long calls;
54};
55
56static struct gtod_log *find_hash(void *caller)
57{
58 unsigned long h = hash_ptr(caller, HASH_BITS);
01743ee1 59 struct flist_head *entry;
02bcaa8c 60
01743ee1
JA
61 flist_for_each(entry, &hash[h]) {
62 struct gtod_log *log = flist_entry(entry, struct gtod_log,
63 list);
02bcaa8c
JA
64
65 if (log->caller == caller)
66 return log;
67 }
68
69 return NULL;
70}
71
d5e16441 72static void inc_caller(void *caller)
02bcaa8c
JA
73{
74 struct gtod_log *log = find_hash(caller);
75
76 if (!log) {
77 unsigned long h;
78
79 log = malloc(sizeof(*log));
01743ee1 80 INIT_FLIST_HEAD(&log->list);
02bcaa8c
JA
81 log->caller = caller;
82 log->calls = 0;
83
84 h = hash_ptr(caller, HASH_BITS);
01743ee1 85 flist_add_tail(&log->list, &hash[h]);
02bcaa8c
JA
86 }
87
d5e16441 88 log->calls++;
02bcaa8c
JA
89}
90
91static void gtod_log_caller(void *caller)
92{
d5e16441
JA
93 if (gtod_inited)
94 inc_caller(caller);
02bcaa8c
JA
95}
96
97static void fio_exit fio_dump_gtod(void)
98{
99 unsigned long total_calls = 0;
100 int i;
101
102 for (i = 0; i < HASH_SIZE; i++) {
01743ee1 103 struct flist_head *entry;
02bcaa8c
JA
104 struct gtod_log *log;
105
01743ee1
JA
106 flist_for_each(entry, &hash[i]) {
107 log = flist_entry(entry, struct gtod_log, list);
02bcaa8c 108
5ec10eaa
JA
109 printf("function %p, calls %lu\n", log->caller,
110 log->calls);
02bcaa8c
JA
111 total_calls += log->calls;
112 }
113 }
114
115 printf("Total %lu gettimeofday\n", total_calls);
116}
117
118static void fio_init gtod_init(void)
119{
120 int i;
121
122 for (i = 0; i < HASH_SIZE; i++)
01743ee1 123 INIT_FLIST_HEAD(&hash[i]);
02bcaa8c
JA
124
125 gtod_inited = 1;
126}
127
128#endif /* FIO_DEBUG_TIME */
129
69212fc4
BVA
130/*
131 * Queries the value of the monotonic clock if a monotonic clock is available
132 * or the wall clock time if no monotonic clock is available. Returns 0 if
133 * querying the clock succeeded or -1 if querying the clock failed.
134 */
135int fio_get_mono_time(struct timespec *ts)
9ff1c070 136{
69212fc4
BVA
137 int ret;
138
139#ifdef CONFIG_CLOCK_GETTIME
140#if defined(CONFIG_CLOCK_MONOTONIC)
141 ret = clock_gettime(CLOCK_MONOTONIC, ts);
9ff1c070 142#else
69212fc4 143 ret = clock_gettime(CLOCK_REALTIME, ts);
9ff1c070 144#endif
69212fc4
BVA
145#else
146 struct timeval tv;
147
148 ret = gettimeofday(&tv, NULL);
149 if (ret == 0) {
150 ts->tv_sec = tv.tv_sec;
151 ts->tv_nsec = tv.tv_usec * 1000;
152 }
1e97cce9 153#endif
69212fc4
BVA
154 assert(ret <= 0);
155 return ret;
156}
67bf9823 157
8b6a404c 158static void __fio_gettime(struct timespec *tp)
02bcaa8c 159{
c223da83 160 switch (fio_clock_source) {
67bf9823 161#ifdef CONFIG_GETTIMEOFDAY
8b6a404c
VF
162 case CS_GTOD: {
163 struct timeval tv;
164 gettimeofday(&tv, NULL);
165
166 tp->tv_sec = tv.tv_sec;
167 tp->tv_nsec = tv.tv_usec * 1000;
c223da83 168 break;
8b6a404c 169 }
67bf9823
JA
170#endif
171#ifdef CONFIG_CLOCK_GETTIME
c223da83 172 case CS_CGETTIME: {
69212fc4
BVA
173 if (fio_get_mono_time(tp) < 0) {
174 log_err("fio: fio_get_mono_time() fails\n");
c223da83 175 assert(0);
02bcaa8c 176 }
c223da83
JA
177 break;
178 }
67bf9823 179#endif
c223da83
JA
180#ifdef ARCH_HAVE_CPU_CLOCK
181 case CS_CPUCLOCK: {
6d02b37b 182 uint64_t nsecs, t, multiples;
03be65c9
JA
183 struct tv_valid *tv;
184
185#ifdef CONFIG_TLS_THREAD
186 tv = &static_tv_valid;
187#else
188 tv = pthread_getspecific(tv_tls_key);
189#endif
c223da83
JA
190
191 t = get_cpu_clock();
96170421 192#ifdef ARCH_CPU_CLOCK_WRAPS
73df3e07
JA
193 if (t < cycles_start && !cycles_wrap)
194 cycles_wrap = 1;
03be65c9
JA
195 else if (cycles_wrap && t >= cycles_start && !tv->warned) {
196 log_err("fio: double CPU clock wrap\n");
197 tv->warned = 1;
02dcf81c 198 }
96170421 199#endif
919e789d 200#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
6d02b37b 201 nsecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC * 1000;
919e789d 202#else
6d02b37b
VF
203 t -= cycles_start;
204 multiples = t >> max_cycles_shift;
205 nsecs = multiples * nsecs_for_max_cycles;
206 nsecs += ((t & max_cycles_mask) * clock_mult) >> clock_shift;
919e789d 207#endif
8b6a404c
VF
208 tp->tv_sec = nsecs / 1000000000ULL;
209 tp->tv_nsec = nsecs % 1000000000ULL;
c223da83
JA
210 break;
211 }
212#endif
213 default:
214 log_err("fio: invalid clock source %d\n", fio_clock_source);
215 break;
02bcaa8c 216 }
67bf9823
JA
217}
218
219#ifdef FIO_DEBUG_TIME
8b6a404c 220void fio_gettime(struct timespec *tp, void *caller)
67bf9823 221#else
8b6a404c 222void fio_gettime(struct timespec *tp, void fio_unused *caller)
67bf9823
JA
223#endif
224{
67bf9823
JA
225#ifdef FIO_DEBUG_TIME
226 if (!caller)
227 caller = __builtin_return_address(0);
228
229 gtod_log_caller(caller);
230#endif
27325ed5 231 if (fio_unlikely(fio_gettime_offload(tp)))
67bf9823 232 return;
67bf9823 233
9eb271b9 234 __fio_gettime(tp);
02bcaa8c 235}
be4ecfdf 236
919e789d 237#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
2aebc5a1 238static unsigned long get_cycles_per_msec(void)
c223da83 239{
8b6a404c 240 struct timespec s, e;
ba458c2f 241 uint64_t c_s, c_e;
99afcdb5 242 uint64_t elapsed;
c223da83 243
440f88da 244 fio_get_mono_time(&s);
9ff1c070 245
c223da83
JA
246 c_s = get_cpu_clock();
247 do {
440f88da 248 fio_get_mono_time(&e);
152529ae 249 c_e = get_cpu_clock();
9ff1c070 250
a0318e17
VF
251 elapsed = ntime_since(&s, &e);
252 if (elapsed >= 1280000)
c223da83 253 break;
c223da83
JA
254 } while (1);
255
a0318e17 256 return (c_e - c_s) * 1000000 / elapsed;
c223da83
JA
257}
258
fa80feae
JA
259#define NR_TIME_ITERS 50
260
e259879e 261static int calibrate_cpu_clock(void)
c223da83
JA
262{
263 double delta, mean, S;
e51a6629 264 uint64_t minc, maxc, avg, cycles[NR_TIME_ITERS];
6d02b37b
VF
265 int i, samples, sft = 0;
266 unsigned long long tmp, max_ticks, max_mult;
c223da83 267
2aebc5a1 268 cycles[0] = get_cycles_per_msec();
c223da83 269 S = delta = mean = 0.0;
fa80feae 270 for (i = 0; i < NR_TIME_ITERS; i++) {
2aebc5a1 271 cycles[i] = get_cycles_per_msec();
c223da83
JA
272 delta = cycles[i] - mean;
273 if (delta) {
274 mean += delta / (i + 1.0);
275 S += delta * (cycles[i] - mean);
276 }
277 }
278
e259879e
JA
279 /*
280 * The most common platform clock breakage is returning zero
281 * indefinitely. Check for that and return failure.
282 */
283 if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
284 return 1;
285
fa80feae 286 S = sqrt(S / (NR_TIME_ITERS - 1.0));
c223da83 287
e51a6629
JA
288 minc = -1ULL;
289 maxc = samples = avg = 0;
fa80feae 290 for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83
JA
291 double this = cycles[i];
292
e51a6629
JA
293 minc = min(cycles[i], minc);
294 maxc = max(cycles[i], maxc);
295
03e20d68 296 if ((fmax(this, mean) - fmin(this, mean)) > S)
c223da83
JA
297 continue;
298 samples++;
299 avg += this;
300 }
301
fa80feae 302 S /= (double) NR_TIME_ITERS;
c223da83 303
fa80feae 304 for (i = 0; i < NR_TIME_ITERS; i++)
13aa415a 305 dprint(FD_TIME, "cycles[%d]=%llu\n", i, (unsigned long long) cycles[i]);
c223da83 306
d7abad3d 307 avg /= samples;
2aebc5a1 308 cycles_per_msec = avg;
a0318e17 309 dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f, N=%d\n",
e51a6629 310 (unsigned long long) minc,
a0318e17
VF
311 (unsigned long long) maxc, mean, S, NR_TIME_ITERS);
312 dprint(FD_TIME, "trimmed mean=%llu, N=%d\n", (unsigned long long) avg, samples);
c223da83 313
2aebc5a1 314 max_ticks = MAX_CLOCK_SEC * cycles_per_msec * 1000ULL;
74558486
JA
315 max_mult = ULLONG_MAX / max_ticks;
316 dprint(FD_TIME, "\n\nmax_ticks=%llu, __builtin_clzll=%d, "
317 "max_mult=%llu\n", max_ticks,
318 __builtin_clzll(max_ticks), max_mult);
6d02b37b
VF
319
320 /*
321 * Find the largest shift count that will produce
322 * a multiplier that does not exceed max_mult
323 */
2aebc5a1 324 tmp = max_mult * cycles_per_msec / 1000000;
6d02b37b
VF
325 while (tmp > 1) {
326 tmp >>= 1;
327 sft++;
328 dprint(FD_TIME, "tmp=%llu, sft=%u\n", tmp, sft);
329 }
330
74558486
JA
331 clock_shift = sft;
332 clock_mult = (1ULL << sft) * 1000000 / cycles_per_msec;
333 dprint(FD_TIME, "clock_shift=%u, clock_mult=%llu\n", clock_shift,
334 clock_mult);
6d02b37b 335
74558486
JA
336 /*
337 * Find the greatest power of 2 clock ticks that is less than the
338 * ticks in MAX_CLOCK_SEC_2STAGE
339 */
6d02b37b 340 max_cycles_shift = max_cycles_mask = 0;
2aebc5a1 341 tmp = MAX_CLOCK_SEC * 1000ULL * cycles_per_msec;
74558486
JA
342 dprint(FD_TIME, "tmp=%llu, max_cycles_shift=%u\n", tmp,
343 max_cycles_shift);
6d02b37b
VF
344 while (tmp > 1) {
345 tmp >>= 1;
346 max_cycles_shift++;
347 dprint(FD_TIME, "tmp=%llu, max_cycles_shift=%u\n", tmp, max_cycles_shift);
348 }
74558486
JA
349 /*
350 * if use use (1ULL << max_cycles_shift) * 1000 / cycles_per_msec
351 * here we will have a discontinuity every
352 * (1ULL << max_cycles_shift) cycles
353 */
354 nsecs_for_max_cycles = ((1ULL << max_cycles_shift) * clock_mult)
355 >> clock_shift;
6d02b37b 356
74558486 357 /* Use a bitmask to calculate ticks % (1ULL << max_cycles_shift) */
6d02b37b
VF
358 for (tmp = 0; tmp < max_cycles_shift; tmp++)
359 max_cycles_mask |= 1ULL << tmp;
360
74558486
JA
361 dprint(FD_TIME, "max_cycles_shift=%u, 2^max_cycles_shift=%llu, "
362 "nsecs_for_max_cycles=%llu, "
363 "max_cycles_mask=%016llx\n",
364 max_cycles_shift, (1ULL << max_cycles_shift),
365 nsecs_for_max_cycles, max_cycles_mask);
6d02b37b 366
73df3e07
JA
367 cycles_start = get_cpu_clock();
368 dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
e259879e 369 return 0;
09a32402
JA
370}
371#else
e259879e 372static int calibrate_cpu_clock(void)
09a32402 373{
919e789d
CE
374#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
375 return 0;
376#else
e259879e 377 return 1;
09a32402 378#endif
919e789d
CE
379}
380#endif // ARCH_HAVE_CPU_CLOCK
09a32402 381
f1e3a7da 382#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(CONFIG_TLS_THREAD)
a0eba820 383void fio_local_clock_init(void)
5d879392
JA
384{
385 struct tv_valid *t;
386
572cfb3f 387 t = calloc(1, sizeof(*t));
9eb271b9 388 if (pthread_setspecific(tv_tls_key, t)) {
5d879392 389 log_err("fio: can't set TLS key\n");
9eb271b9
JA
390 assert(0);
391 }
5d879392
JA
392}
393
394static void kill_tv_tls_key(void *data)
395{
396 free(data);
397}
67bf9823 398#else
a0eba820 399void fio_local_clock_init(void)
67bf9823
JA
400{
401}
402#endif
5d879392 403
09a32402
JA
404void fio_clock_init(void)
405{
01423eae
JA
406 if (fio_clock_source == fio_clock_source_inited)
407 return;
408
f1e3a7da 409#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(CONFIG_TLS_THREAD)
5d879392
JA
410 if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
411 log_err("fio: can't create TLS key\n");
67bf9823 412#endif
5d879392 413
01423eae 414 fio_clock_source_inited = fio_clock_source;
e259879e
JA
415
416 if (calibrate_cpu_clock())
24575392 417 tsc_reliable = false;
fa80feae
JA
418
419 /*
420 * If the arch sets tsc_reliable != 0, then it must be good enough
421 * to use as THE clock source. For x86 CPUs, this means the TSC
422 * runs at a constant rate and is synced across CPU cores.
423 */
424 if (tsc_reliable) {
aad918e4 425 if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
fa80feae
JA
426 fio_clock_source = CS_CPUCLOCK;
427 } else if (fio_clock_source == CS_CPUCLOCK)
428 log_info("fio: clocksource=cpu may not be reliable\n");
dae89ac9 429 dprint(FD_TIME, "gettime: clocksource=%d\n", (int) fio_clock_source);
c223da83
JA
430}
431
d6bb626e
VF
432uint64_t ntime_since(const struct timespec *s, const struct timespec *e)
433{
434 int64_t sec, nsec;
435
436 sec = e->tv_sec - s->tv_sec;
437 nsec = e->tv_nsec - s->tv_nsec;
438 if (sec > 0 && nsec < 0) {
6d02b37b
VF
439 sec--;
440 nsec += 1000000000LL;
d6bb626e
VF
441 }
442
443 /*
6d02b37b
VF
444 * time warp bug on some kernels?
445 */
d6bb626e 446 if (sec < 0 || (sec == 0 && nsec < 0))
6d02b37b 447 return 0;
d6bb626e
VF
448
449 return nsec + (sec * 1000000000LL);
450}
451
0410e783
JA
452uint64_t ntime_since_now(const struct timespec *s)
453{
454 struct timespec now;
455
456 fio_gettime(&now, NULL);
457 return ntime_since(s, &now);
458}
459
8b6a404c 460uint64_t utime_since(const struct timespec *s, const struct timespec *e)
be4ecfdf 461{
90eff1c9 462 int64_t sec, usec;
39ab7da2
JA
463
464 sec = e->tv_sec - s->tv_sec;
8b6a404c 465 usec = (e->tv_nsec - s->tv_nsec) / 1000;
39ab7da2
JA
466 if (sec > 0 && usec < 0) {
467 sec--;
468 usec += 1000000;
469 }
470
471 /*
472 * time warp bug on some kernels?
473 */
474 if (sec < 0 || (sec == 0 && usec < 0))
475 return 0;
476
20ac4e77 477 return usec + (sec * 1000000);
be4ecfdf
JA
478}
479
8b6a404c 480uint64_t utime_since_now(const struct timespec *s)
be4ecfdf 481{
8b6a404c 482 struct timespec t;
f52e9198
VF
483#ifdef FIO_DEBUG_TIME
484 void *p = __builtin_return_address(0);
39ab7da2 485
f52e9198
VF
486 fio_gettime(&t, p);
487#else
39ab7da2 488 fio_gettime(&t, NULL);
f52e9198
VF
489#endif
490
39ab7da2 491 return utime_since(s, &t);
be4ecfdf 492}
783a3eb1 493
8b6a404c 494uint64_t mtime_since_tv(const struct timeval *s, const struct timeval *e)
783a3eb1 495{
8b6a404c 496 int64_t sec, usec;
783a3eb1 497
39ab7da2 498 sec = e->tv_sec - s->tv_sec;
8b6a404c 499 usec = (e->tv_usec - s->tv_usec);
39ab7da2
JA
500 if (sec > 0 && usec < 0) {
501 sec--;
502 usec += 1000000;
783a3eb1
JA
503 }
504
39ab7da2
JA
505 if (sec < 0 || (sec == 0 && usec < 0))
506 return 0;
507
be6bb2b7
Y
508 sec *= 1000;
509 usec /= 1000;
20ac4e77 510 return sec + usec;
783a3eb1
JA
511}
512
8b6a404c 513uint64_t mtime_since_now(const struct timespec *s)
783a3eb1 514{
8b6a404c 515 struct timespec t;
f52e9198 516#ifdef FIO_DEBUG_TIME
39ab7da2 517 void *p = __builtin_return_address(0);
783a3eb1 518
39ab7da2 519 fio_gettime(&t, p);
f52e9198
VF
520#else
521 fio_gettime(&t, NULL);
522#endif
523
39ab7da2
JA
524 return mtime_since(s, &t);
525}
783a3eb1 526
383c7960
BVA
527/*
528 * Returns *e - *s in milliseconds as a signed integer. Note: rounding is
529 * asymmetric. If the difference yields +1 ns then 0 is returned. If the
530 * difference yields -1 ns then -1 is returned.
531 */
532int64_t rel_time_since(const struct timespec *s, const struct timespec *e)
8b6a404c 533{
383c7960 534 int64_t sec, nsec;
8b6a404c
VF
535
536 sec = e->tv_sec - s->tv_sec;
383c7960
BVA
537 nsec = e->tv_nsec - s->tv_nsec;
538 if (nsec < 0) {
8b6a404c 539 sec--;
383c7960 540 nsec += 1000ULL * 1000 * 1000;
8b6a404c 541 }
383c7960 542 assert(0 <= nsec && nsec < 1000ULL * 1000 * 1000);
8b6a404c 543
383c7960
BVA
544 return sec * 1000 + nsec / (1000 * 1000);
545}
8b6a404c 546
383c7960
BVA
547/*
548 * Returns *e - *s in milliseconds as an unsigned integer. Returns 0 if
549 * *e < *s.
550 */
551uint64_t mtime_since(const struct timespec *s, const struct timespec *e)
552{
553 return max(rel_time_since(s, e), (int64_t)0);
8b6a404c
VF
554}
555
556uint64_t time_since_now(const struct timespec *s)
39ab7da2
JA
557{
558 return mtime_since_now(s) / 1000;
783a3eb1 559}
7d11f871 560
67bf9823 561#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
a250edd0 562 defined(CONFIG_SYNC_SYNC) && defined(CONFIG_CMP_SWAP)
7d11f871 563
aad918e4 564#define CLOCK_ENTRIES_DEBUG 100000
5283741f 565#define CLOCK_ENTRIES_TEST 1000
7d11f871
JA
566
567struct clock_entry {
58002f9a
JA
568 uint32_t seq;
569 uint32_t cpu;
ba458c2f 570 uint64_t tsc;
7d11f871
JA
571};
572
573struct clock_thread {
574 pthread_t thread;
575 int cpu;
aad918e4 576 int debug;
17031c6d 577 struct fio_sem lock;
aad918e4 578 unsigned long nr_entries;
58002f9a 579 uint32_t *seq;
7d11f871
JA
580 struct clock_entry *entries;
581};
582
a250edd0
JA
583static inline uint32_t atomic32_compare_and_swap(uint32_t *ptr, uint32_t old,
584 uint32_t new)
7d11f871 585{
a250edd0 586 return __sync_val_compare_and_swap(ptr, old, new);
7d11f871
JA
587}
588
589static void *clock_thread_fn(void *data)
590{
591 struct clock_thread *t = data;
592 struct clock_entry *c;
593 os_cpu_mask_t cpu_mask;
5896d827 594 unsigned long long first;
7d11f871
JA
595 int i;
596
c763aea6 597 if (fio_cpuset_init(&cpu_mask)) {
37e20021 598 int __err = errno;
c763aea6
JA
599
600 log_err("clock cpuset init failed: %s\n", strerror(__err));
601 goto err_out;
602 }
603
7d11f871
JA
604 fio_cpu_set(&cpu_mask, t->cpu);
605
606 if (fio_setaffinity(gettid(), cpu_mask) == -1) {
e66d7f90
JA
607 int __err = errno;
608
609 log_err("clock setaffinity failed: %s\n", strerror(__err));
c763aea6 610 goto err;
7d11f871
JA
611 }
612
17031c6d 613 fio_sem_down(&t->lock);
7d11f871 614
5896d827 615 first = get_cpu_clock();
7d11f871 616 c = &t->entries[0];
aad918e4 617 for (i = 0; i < t->nr_entries; i++, c++) {
58002f9a
JA
618 uint32_t seq;
619 uint64_t tsc;
7d11f871
JA
620
621 c->cpu = t->cpu;
622 do {
a250edd0
JA
623 seq = *t->seq;
624 if (seq == UINT_MAX)
58002f9a 625 break;
a250edd0 626 __sync_synchronize();
7d11f871 627 tsc = get_cpu_clock();
a250edd0
JA
628 } while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1));
629
630 if (seq == UINT_MAX)
631 break;
7d11f871
JA
632
633 c->seq = seq;
634 c->tsc = tsc;
635 }
636
aad918e4
JA
637 if (t->debug) {
638 unsigned long long clocks;
639
640 clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
5896d827
JA
641 log_info("cs: cpu%3d: %llu clocks seen, first %llu\n", t->cpu,
642 clocks, first);
aad918e4 643 }
58002f9a 644
e259879e
JA
645 /*
646 * The most common platform clock breakage is returning zero
647 * indefinitely. Check for that and return failure.
648 */
a250edd0 649 if (i > 1 && !t->entries[i - 1].tsc && !t->entries[0].tsc)
c763aea6 650 goto err;
e259879e 651
d47d7cb3 652 fio_cpuset_exit(&cpu_mask);
7d11f871 653 return NULL;
c763aea6
JA
654err:
655 fio_cpuset_exit(&cpu_mask);
656err_out:
657 return (void *) 1;
7d11f871
JA
658}
659
660static int clock_cmp(const void *p1, const void *p2)
661{
662 const struct clock_entry *c1 = p1;
663 const struct clock_entry *c2 = p2;
664
b9b3498e
JA
665 if (c1->seq == c2->seq)
666 log_err("cs: bug in atomic sequence!\n");
667
7d11f871
JA
668 return c1->seq - c2->seq;
669}
670
aad918e4 671int fio_monotonic_clocktest(int debug)
7d11f871 672{
8a1db9a1 673 struct clock_thread *cthreads;
db760d88 674 unsigned int seen_cpus, nr_cpus = cpus_online();
7d11f871 675 struct clock_entry *entries;
aad918e4 676 unsigned long nr_entries, tentries, failed = 0;
80da8a8f 677 struct clock_entry *prev, *this;
58002f9a 678 uint32_t seq = 0;
caa3eb1c 679 unsigned int i;
db760d88
JA
680 os_cpu_mask_t mask;
681
e9d2a04d 682#ifdef FIO_HAVE_GET_THREAD_AFFINITY
db760d88 683 fio_get_thread_affinity(mask);
6202c70d
JA
684#else
685 memset(&mask, 0, sizeof(mask));
686 for (i = 0; i < nr_cpus; i++)
687 fio_cpu_set(&mask, i);
db760d88 688#endif
7d11f871 689
aad918e4
JA
690 if (debug) {
691 log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
d5e3f5d8 692
b5b571a3 693#ifdef FIO_INC_DEBUG
aad918e4 694 fio_debug |= 1U << FD_TIME;
b5b571a3 695#endif
aad918e4
JA
696 nr_entries = CLOCK_ENTRIES_DEBUG;
697 } else
698 nr_entries = CLOCK_ENTRIES_TEST;
699
4f1d43c2 700 calibrate_cpu_clock();
aad918e4
JA
701
702 if (debug) {
b5b571a3 703#ifdef FIO_INC_DEBUG
aad918e4 704 fio_debug &= ~(1U << FD_TIME);
b5b571a3 705#endif
aad918e4 706 }
4f1d43c2 707
8a1db9a1 708 cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
aad918e4 709 tentries = nr_entries * nr_cpus;
7d11f871
JA
710 entries = malloc(tentries * sizeof(struct clock_entry));
711
aad918e4
JA
712 if (debug)
713 log_info("cs: Testing %u CPUs\n", nr_cpus);
7d11f871 714
db760d88 715 seen_cpus = 0;
7d11f871 716 for (i = 0; i < nr_cpus; i++) {
8a1db9a1 717 struct clock_thread *t = &cthreads[i];
7d11f871 718
db760d88
JA
719 if (!fio_cpu_isset(&mask, i))
720 continue;
7d11f871 721 t->cpu = i;
aad918e4 722 t->debug = debug;
7d11f871 723 t->seq = &seq;
aad918e4 724 t->nr_entries = nr_entries;
db760d88 725 t->entries = &entries[seen_cpus * nr_entries];
17031c6d 726 __fio_sem_init(&t->lock, FIO_SEM_LOCKED);
6b0110cd
JA
727 if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
728 failed++;
729 nr_cpus = i;
730 break;
731 }
db760d88 732 seen_cpus++;
7d11f871
JA
733 }
734
735 for (i = 0; i < nr_cpus; i++) {
8a1db9a1 736 struct clock_thread *t = &cthreads[i];
6202c70d 737
db760d88
JA
738 if (!fio_cpu_isset(&mask, i))
739 continue;
17031c6d 740 fio_sem_up(&t->lock);
7d11f871
JA
741 }
742
814917be 743 for (i = 0; i < nr_cpus; i++) {
8a1db9a1 744 struct clock_thread *t = &cthreads[i];
7d11f871 745 void *ret;
6202c70d 746
db760d88
JA
747 if (!fio_cpu_isset(&mask, i))
748 continue;
7d11f871
JA
749 pthread_join(t->thread, &ret);
750 if (ret)
751 failed++;
17031c6d 752 __fio_sem_remove(&t->lock);
7d11f871 753 }
8a1db9a1 754 free(cthreads);
7d11f871
JA
755
756 if (failed) {
aad918e4
JA
757 if (debug)
758 log_err("Clocksource test: %lu threads failed\n", failed);
7d11f871
JA
759 goto err;
760 }
761
db760d88 762 tentries = nr_entries * seen_cpus;
7d11f871
JA
763 qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
764
a4aa3dc0
JA
765 /* silence silly gcc */
766 prev = NULL;
7d11f871 767 for (failed = i = 0; i < tentries; i++) {
80da8a8f 768 this = &entries[i];
7d11f871
JA
769
770 if (!i) {
771 prev = this;
772 continue;
773 }
774
775 if (prev->tsc > this->tsc) {
776 uint64_t diff = prev->tsc - this->tsc;
777
aad918e4
JA
778 if (!debug) {
779 failed++;
780 break;
781 }
782
4e0a8fa2
JA
783 log_info("cs: CPU clock mismatch (diff=%llu):\n",
784 (unsigned long long) diff);
785 log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
786 log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
7d11f871
JA
787 failed++;
788 }
789
790 prev = this;
791 }
792
aad918e4
JA
793 if (debug) {
794 if (failed)
795 log_info("cs: Failed: %lu\n", failed);
796 else
797 log_info("cs: Pass!\n");
798 }
7d11f871
JA
799err:
800 free(entries);
801 return !!failed;
802}
803
804#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
805
aad918e4 806int fio_monotonic_clocktest(int debug)
7d11f871 807{
aad918e4
JA
808 if (debug)
809 log_info("cs: current platform does not support CPU clocks\n");
810 return 1;
7d11f871
JA
811}
812
813#endif