gettime.c

   1 /*
   2  * Clock functions
   3  */
   4
   5 #include <math.h>
   6
   7 #include "fio.h"
   8 #include "os/os.h"
   9
  10 #if defined(ARCH_HAVE_CPU_CLOCK)
  11 #ifndef ARCH_CPU_CLOCK_CYCLES_PER_USEC
  12 static unsigned long long cycles_per_msec;
  13 static unsigned long long cycles_start;
  14 static unsigned long long clock_mult;
  15 static unsigned long long max_cycles_mask;
  16 static unsigned long long nsecs_for_max_cycles;
  17 static unsigned int clock_shift;
  18 static unsigned int max_cycles_shift;
  19 #define MAX_CLOCK_SEC 60*60
  20 #endif
  21 #ifdef ARCH_CPU_CLOCK_WRAPS
  22 static unsigned int cycles_wrap;
  23 #endif
  24 #endif
  25 bool tsc_reliable = false;
  26
  27 struct tv_valid {
  28         int warned;
  29 };
  30 #ifdef ARCH_HAVE_CPU_CLOCK
  31 #ifdef CONFIG_TLS_THREAD
  32 static __thread struct tv_valid static_tv_valid;
  33 #else
  34 static pthread_key_t tv_tls_key;
  35 #endif
  36 #endif
  37
  38 enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
  39 int fio_clock_source_set = 0;
  40 static enum fio_cs fio_clock_source_inited = CS_INVAL;
  41
  42 #ifdef FIO_DEBUG_TIME
  43
  44 #define HASH_BITS       8
  45 #define HASH_SIZE       (1 << HASH_BITS)
  46
  47 static struct flist_head hash[HASH_SIZE];
  48 static int gtod_inited;
  49
  50 struct gtod_log {
  51         struct flist_head list;
  52         void *caller;
  53         unsigned long calls;
  54 };
  55
  56 static struct gtod_log *find_hash(void *caller)
  57 {
  58         unsigned long h = hash_ptr(caller, HASH_BITS);
  59         struct flist_head *entry;
  60
  61         flist_for_each(entry, &hash[h]) {
  62                 struct gtod_log *log = flist_entry(entry, struct gtod_log,
  63                                                                         list);
  64
  65                 if (log->caller == caller)
  66                         return log;
  67         }
  68
  69         return NULL;
  70 }
  71
  72 static void inc_caller(void *caller)
  73 {
  74         struct gtod_log *log = find_hash(caller);
  75
  76         if (!log) {
  77                 unsigned long h;
  78
  79                 log = malloc(sizeof(*log));
  80                 INIT_FLIST_HEAD(&log->list);
  81                 log->caller = caller;
  82                 log->calls = 0;
  83
  84                 h = hash_ptr(caller, HASH_BITS);
  85                 flist_add_tail(&log->list, &hash[h]);
  86         }
  87
  88         log->calls++;
  89 }
  90
  91 static void gtod_log_caller(void *caller)
  92 {
  93         if (gtod_inited)
  94                 inc_caller(caller);
  95 }
  96
  97 static void fio_exit fio_dump_gtod(void)
  98 {
  99         unsigned long total_calls = 0;
 100         int i;
 101
 102         for (i = 0; i < HASH_SIZE; i++) {
 103                 struct flist_head *entry;
 104                 struct gtod_log *log;
 105
 106                 flist_for_each(entry, &hash[i]) {
 107                         log = flist_entry(entry, struct gtod_log, list);
 108
 109                         printf("function %p, calls %lu\n", log->caller,
 110                                                                 log->calls);
 111                         total_calls += log->calls;
 112                 }
 113         }
 114
 115         printf("Total %lu gettimeofday\n", total_calls);
 116 }
 117
 118 static void fio_init gtod_init(void)
 119 {
 120         int i;
 121
 122         for (i = 0; i < HASH_SIZE; i++)
 123                 INIT_FLIST_HEAD(&hash[i]);
 124
 125         gtod_inited = 1;
 126 }
 127
 128 #endif /* FIO_DEBUG_TIME */
 129
 130 #ifdef CONFIG_CLOCK_GETTIME
 131 static int fill_clock_gettime(struct timespec *ts)
 132 {
 133 #if defined(CONFIG_CLOCK_MONOTONIC_RAW)
 134         return clock_gettime(CLOCK_MONOTONIC_RAW, ts);
 135 #elif defined(CONFIG_CLOCK_MONOTONIC)
 136         return clock_gettime(CLOCK_MONOTONIC, ts);
 137 #else
 138         return clock_gettime(CLOCK_REALTIME, ts);
 139 #endif
 140 }
 141 #endif
 142
 143 static void __fio_gettime(struct timespec *tp)
 144 {
 145         switch (fio_clock_source) {
 146 #ifdef CONFIG_GETTIMEOFDAY
 147         case CS_GTOD: {
 148                 struct timeval tv;
 149                 gettimeofday(&tv, NULL);
 150
 151                 tp->tv_sec = tv.tv_sec;
 152                 tp->tv_nsec = tv.tv_usec * 1000;
 153                 break;
 154                 }
 155 #endif
 156 #ifdef CONFIG_CLOCK_GETTIME
 157         case CS_CGETTIME: {
 158                 if (fill_clock_gettime(tp) < 0) {
 159                         log_err("fio: clock_gettime fails\n");
 160                         assert(0);
 161                 }
 162                 break;
 163                 }
 164 #endif
 165 #ifdef ARCH_HAVE_CPU_CLOCK
 166         case CS_CPUCLOCK: {
 167                 uint64_t nsecs, t, multiples;
 168                 struct tv_valid *tv;
 169
 170 #ifdef CONFIG_TLS_THREAD
 171                 tv = &static_tv_valid;
 172 #else
 173                 tv = pthread_getspecific(tv_tls_key);
 174 #endif
 175
 176                 t = get_cpu_clock();
 177 #ifdef ARCH_CPU_CLOCK_WRAPS
 178                 if (t < cycles_start && !cycles_wrap)
 179                         cycles_wrap = 1;
 180                 else if (cycles_wrap && t >= cycles_start && !tv->warned) {
 181                         log_err("fio: double CPU clock wrap\n");
 182                         tv->warned = 1;
 183                 }
 184 #endif
 185 #ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
 186                 nsecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC * 1000;
 187 #else
 188                 t -= cycles_start;
 189                 multiples = t >> max_cycles_shift;
 190                 nsecs = multiples * nsecs_for_max_cycles;
 191                 nsecs += ((t & max_cycles_mask) * clock_mult) >> clock_shift;
 192 #endif
 193                 tp->tv_sec = nsecs / 1000000000ULL;
 194                 tp->tv_nsec = nsecs % 1000000000ULL;
 195                 break;
 196                 }
 197 #endif
 198         default:
 199                 log_err("fio: invalid clock source %d\n", fio_clock_source);
 200                 break;
 201         }
 202 }
 203
 204 #ifdef FIO_DEBUG_TIME
 205 void fio_gettime(struct timespec *tp, void *caller)
 206 #else
 207 void fio_gettime(struct timespec *tp, void fio_unused *caller)
 208 #endif
 209 {
 210 #ifdef FIO_DEBUG_TIME
 211         if (!caller)
 212                 caller = __builtin_return_address(0);
 213
 214         gtod_log_caller(caller);
 215 #endif
 216         if (fio_unlikely(fio_gettime_offload(tp)))
 217                 return;
 218
 219         __fio_gettime(tp);
 220 }
 221
 222 #if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
 223 static unsigned long get_cycles_per_msec(void)
 224 {
 225         struct timespec s, e;
 226         uint64_t c_s, c_e;
 227         enum fio_cs old_cs = fio_clock_source;
 228         uint64_t elapsed;
 229
 230 #ifdef CONFIG_CLOCK_GETTIME
 231         fio_clock_source = CS_CGETTIME;
 232 #else
 233         fio_clock_source = CS_GTOD;
 234 #endif
 235         __fio_gettime(&s);
 236
 237         c_s = get_cpu_clock();
 238         do {
 239                 __fio_gettime(&e);
 240
 241                 elapsed = utime_since(&s, &e);
 242                 if (elapsed >= 1280) {
 243                         c_e = get_cpu_clock();
 244                         break;
 245                 }
 246         } while (1);
 247
 248         fio_clock_source = old_cs;
 249         return (c_e - c_s) * 1000 / elapsed;
 250 }
 251
 252 #define NR_TIME_ITERS   50
 253
 254 static int calibrate_cpu_clock(void)
 255 {
 256         double delta, mean, S;
 257         uint64_t minc, maxc, avg, cycles[NR_TIME_ITERS];
 258         int i, samples, sft = 0;
 259         unsigned long long tmp, max_ticks, max_mult;
 260
 261         cycles[0] = get_cycles_per_msec();
 262         S = delta = mean = 0.0;
 263         for (i = 0; i < NR_TIME_ITERS; i++) {
 264                 cycles[i] = get_cycles_per_msec();
 265                 delta = cycles[i] - mean;
 266                 if (delta) {
 267                         mean += delta / (i + 1.0);
 268                         S += delta * (cycles[i] - mean);
 269                 }
 270         }
 271
 272         /*
 273          * The most common platform clock breakage is returning zero
 274          * indefinitely. Check for that and return failure.
 275          */
 276         if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
 277                 return 1;
 278
 279         S = sqrt(S / (NR_TIME_ITERS - 1.0));
 280
 281         minc = -1ULL;
 282         maxc = samples = avg = 0;
 283         for (i = 0; i < NR_TIME_ITERS; i++) {
 284                 double this = cycles[i];
 285
 286                 minc = min(cycles[i], minc);
 287                 maxc = max(cycles[i], maxc);
 288
 289                 if ((fmax(this, mean) - fmin(this, mean)) > S)
 290                         continue;
 291                 samples++;
 292                 avg += this;
 293         }
 294
 295         S /= (double) NR_TIME_ITERS;
 296
 297         for (i = 0; i < NR_TIME_ITERS; i++)
 298                 dprint(FD_TIME, "cycles[%d]=%llu\n", i, (unsigned long long) cycles[i]);
 299
 300         avg /= samples;
 301         cycles_per_msec = avg;
 302         dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
 303         dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f\n",
 304                         (unsigned long long) minc,
 305                         (unsigned long long) maxc, mean, S);
 306
 307         max_ticks = MAX_CLOCK_SEC * cycles_per_msec * 1000ULL;
 308         max_mult = ULLONG_MAX / max_ticks;
 309         dprint(FD_TIME, "\n\nmax_ticks=%llu, __builtin_clzll=%d, "
 310                         "max_mult=%llu\n", max_ticks,
 311                         __builtin_clzll(max_ticks), max_mult);
 312
 313         /*
 314          * Find the largest shift count that will produce
 315          * a multiplier that does not exceed max_mult
 316          */
 317         tmp = max_mult * cycles_per_msec / 1000000;
 318         while (tmp > 1) {
 319                 tmp >>= 1;
 320                 sft++;
 321                 dprint(FD_TIME, "tmp=%llu, sft=%u\n", tmp, sft);
 322         }
 323
 324         clock_shift = sft;
 325         clock_mult = (1ULL << sft) * 1000000 / cycles_per_msec;
 326         dprint(FD_TIME, "clock_shift=%u, clock_mult=%llu\n", clock_shift,
 327                                                         clock_mult);
 328
 329         /*
 330          * Find the greatest power of 2 clock ticks that is less than the
 331          * ticks in MAX_CLOCK_SEC_2STAGE
 332          */
 333         max_cycles_shift = max_cycles_mask = 0;
 334         tmp = MAX_CLOCK_SEC * 1000ULL * cycles_per_msec;
 335         dprint(FD_TIME, "tmp=%llu, max_cycles_shift=%u\n", tmp,
 336                                                         max_cycles_shift);
 337         while (tmp > 1) {
 338                 tmp >>= 1;
 339                 max_cycles_shift++;
 340                 dprint(FD_TIME, "tmp=%llu, max_cycles_shift=%u\n", tmp, max_cycles_shift);
 341         }
 342         /*
 343          * if use use (1ULL << max_cycles_shift) * 1000 / cycles_per_msec
 344          * here we will have a discontinuity every
 345          * (1ULL << max_cycles_shift) cycles
 346          */
 347         nsecs_for_max_cycles = ((1ULL << max_cycles_shift) * clock_mult)
 348                                         >> clock_shift;
 349
 350         /* Use a bitmask to calculate ticks % (1ULL << max_cycles_shift) */
 351         for (tmp = 0; tmp < max_cycles_shift; tmp++)
 352                 max_cycles_mask |= 1ULL << tmp;
 353
 354         dprint(FD_TIME, "max_cycles_shift=%u, 2^max_cycles_shift=%llu, "
 355                         "nsecs_for_max_cycles=%llu, "
 356                         "max_cycles_mask=%016llx\n",
 357                         max_cycles_shift, (1ULL << max_cycles_shift),
 358                         nsecs_for_max_cycles, max_cycles_mask);
 359
 360         cycles_start = get_cpu_clock();
 361         dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
 362         return 0;
 363 }
 364 #else
 365 static int calibrate_cpu_clock(void)
 366 {
 367 #ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
 368         return 0;
 369 #else
 370         return 1;
 371 #endif
 372 }
 373 #endif // ARCH_HAVE_CPU_CLOCK
 374
 375 #ifndef CONFIG_TLS_THREAD
 376 void fio_local_clock_init(void)
 377 {
 378         struct tv_valid *t;
 379
 380         t = calloc(1, sizeof(*t));
 381         if (pthread_setspecific(tv_tls_key, t)) {
 382                 log_err("fio: can't set TLS key\n");
 383                 assert(0);
 384         }
 385 }
 386
 387 static void kill_tv_tls_key(void *data)
 388 {
 389         free(data);
 390 }
 391 #else
 392 void fio_local_clock_init(void)
 393 {
 394 }
 395 #endif
 396
 397 void fio_clock_init(void)
 398 {
 399         if (fio_clock_source == fio_clock_source_inited)
 400                 return;
 401
 402 #ifndef CONFIG_TLS_THREAD
 403         if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
 404                 log_err("fio: can't create TLS key\n");
 405 #endif
 406
 407         fio_clock_source_inited = fio_clock_source;
 408
 409         if (calibrate_cpu_clock())
 410                 tsc_reliable = false;
 411
 412         /*
 413          * If the arch sets tsc_reliable != 0, then it must be good enough
 414          * to use as THE clock source. For x86 CPUs, this means the TSC
 415          * runs at a constant rate and is synced across CPU cores.
 416          */
 417         if (tsc_reliable) {
 418                 if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
 419                         fio_clock_source = CS_CPUCLOCK;
 420         } else if (fio_clock_source == CS_CPUCLOCK)
 421                 log_info("fio: clocksource=cpu may not be reliable\n");
 422         dprint(FD_TIME, "gettime: clocksource=%d\n", (int) fio_clock_source);
 423 }
 424
 425 uint64_t ntime_since(const struct timespec *s, const struct timespec *e)
 426 {
 427        int64_t sec, nsec;
 428
 429        sec = e->tv_sec - s->tv_sec;
 430        nsec = e->tv_nsec - s->tv_nsec;
 431        if (sec > 0 && nsec < 0) {
 432                sec--;
 433                nsec += 1000000000LL;
 434        }
 435
 436        /*
 437         * time warp bug on some kernels?
 438         */
 439        if (sec < 0 || (sec == 0 && nsec < 0))
 440                return 0;
 441
 442        return nsec + (sec * 1000000000LL);
 443 }
 444
 445 uint64_t ntime_since_now(const struct timespec *s)
 446 {
 447         struct timespec now;
 448
 449         fio_gettime(&now, NULL);
 450         return ntime_since(s, &now);
 451 }
 452
 453 uint64_t utime_since(const struct timespec *s, const struct timespec *e)
 454 {
 455         int64_t sec, usec;
 456
 457         sec = e->tv_sec - s->tv_sec;
 458         usec = (e->tv_nsec - s->tv_nsec) / 1000;
 459         if (sec > 0 && usec < 0) {
 460                 sec--;
 461                 usec += 1000000;
 462         }
 463
 464         /*
 465          * time warp bug on some kernels?
 466          */
 467         if (sec < 0 || (sec == 0 && usec < 0))
 468                 return 0;
 469
 470         return usec + (sec * 1000000);
 471 }
 472
 473 uint64_t utime_since_now(const struct timespec *s)
 474 {
 475         struct timespec t;
 476 #ifdef FIO_DEBUG_TIME
 477         void *p = __builtin_return_address(0);
 478
 479         fio_gettime(&t, p);
 480 #else
 481         fio_gettime(&t, NULL);
 482 #endif
 483
 484         return utime_since(s, &t);
 485 }
 486
 487 uint64_t mtime_since_tv(const struct timeval *s, const struct timeval *e)
 488 {
 489         int64_t sec, usec;
 490
 491         sec = e->tv_sec - s->tv_sec;
 492         usec = (e->tv_usec - s->tv_usec);
 493         if (sec > 0 && usec < 0) {
 494                 sec--;
 495                 usec += 1000000;
 496         }
 497
 498         if (sec < 0 || (sec == 0 && usec < 0))
 499                 return 0;
 500
 501         sec *= 1000;
 502         usec /= 1000;
 503         return sec + usec;
 504 }
 505
 506 uint64_t mtime_since_now(const struct timespec *s)
 507 {
 508         struct timespec t;
 509 #ifdef FIO_DEBUG_TIME
 510         void *p = __builtin_return_address(0);
 511
 512         fio_gettime(&t, p);
 513 #else
 514         fio_gettime(&t, NULL);
 515 #endif
 516
 517         return mtime_since(s, &t);
 518 }
 519
 520 uint64_t mtime_since(const struct timespec *s, const struct timespec *e)
 521 {
 522         int64_t sec, usec;
 523
 524         sec = e->tv_sec - s->tv_sec;
 525         usec = (e->tv_nsec - s->tv_nsec) / 1000;
 526         if (sec > 0 && usec < 0) {
 527                 sec--;
 528                 usec += 1000000;
 529         }
 530
 531         if (sec < 0 || (sec == 0 && usec < 0))
 532                 return 0;
 533
 534         sec *= 1000;
 535         usec /= 1000;
 536         return sec + usec;
 537 }
 538
 539 uint64_t time_since_now(const struct timespec *s)
 540 {
 541         return mtime_since_now(s) / 1000;
 542 }
 543
 544 #if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \
 545     defined(CONFIG_SYNC_SYNC) && defined(CONFIG_CMP_SWAP)
 546
 547 #define CLOCK_ENTRIES_DEBUG     100000
 548 #define CLOCK_ENTRIES_TEST      1000
 549
 550 struct clock_entry {
 551         uint32_t seq;
 552         uint32_t cpu;
 553         uint64_t tsc;
 554 };
 555
 556 struct clock_thread {
 557         pthread_t thread;
 558         int cpu;
 559         int debug;
 560         struct fio_sem lock;
 561         unsigned long nr_entries;
 562         uint32_t *seq;
 563         struct clock_entry *entries;
 564 };
 565
 566 static inline uint32_t atomic32_compare_and_swap(uint32_t *ptr, uint32_t old,
 567                                                  uint32_t new)
 568 {
 569         return __sync_val_compare_and_swap(ptr, old, new);
 570 }
 571
 572 static void *clock_thread_fn(void *data)
 573 {
 574         struct clock_thread *t = data;
 575         struct clock_entry *c;
 576         os_cpu_mask_t cpu_mask;
 577         unsigned long long first;
 578         int i;
 579
 580         if (fio_cpuset_init(&cpu_mask)) {
 581                 int __err = errno;
 582
 583                 log_err("clock cpuset init failed: %s\n", strerror(__err));
 584                 goto err_out;
 585         }
 586
 587         fio_cpu_set(&cpu_mask, t->cpu);
 588
 589         if (fio_setaffinity(gettid(), cpu_mask) == -1) {
 590                 int __err = errno;
 591
 592                 log_err("clock setaffinity failed: %s\n", strerror(__err));
 593                 goto err;
 594         }
 595
 596         fio_sem_down(&t->lock);
 597
 598         first = get_cpu_clock();
 599         c = &t->entries[0];
 600         for (i = 0; i < t->nr_entries; i++, c++) {
 601                 uint32_t seq;
 602                 uint64_t tsc;
 603
 604                 c->cpu = t->cpu;
 605                 do {
 606                         seq = *t->seq;
 607                         if (seq == UINT_MAX)
 608                                 break;
 609                         __sync_synchronize();
 610                         tsc = get_cpu_clock();
 611                 } while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1));
 612
 613                 if (seq == UINT_MAX)
 614                         break;
 615
 616                 c->seq = seq;
 617                 c->tsc = tsc;
 618         }
 619
 620         if (t->debug) {
 621                 unsigned long long clocks;
 622
 623                 clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
 624                 log_info("cs: cpu%3d: %llu clocks seen, first %llu\n", t->cpu,
 625                                                         clocks, first);
 626         }
 627
 628         /*
 629          * The most common platform clock breakage is returning zero
 630          * indefinitely. Check for that and return failure.
 631          */
 632         if (i > 1 && !t->entries[i - 1].tsc && !t->entries[0].tsc)
 633                 goto err;
 634
 635         fio_cpuset_exit(&cpu_mask);
 636         return NULL;
 637 err:
 638         fio_cpuset_exit(&cpu_mask);
 639 err_out:
 640         return (void *) 1;
 641 }
 642
 643 static int clock_cmp(const void *p1, const void *p2)
 644 {
 645         const struct clock_entry *c1 = p1;
 646         const struct clock_entry *c2 = p2;
 647
 648         if (c1->seq == c2->seq)
 649                 log_err("cs: bug in atomic sequence!\n");
 650
 651         return c1->seq - c2->seq;
 652 }
 653
 654 int fio_monotonic_clocktest(int debug)
 655 {
 656         struct clock_thread *cthreads;
 657         unsigned int nr_cpus = cpus_online();
 658         struct clock_entry *entries;
 659         unsigned long nr_entries, tentries, failed = 0;
 660         struct clock_entry *prev, *this;
 661         uint32_t seq = 0;
 662         unsigned int i;
 663
 664         if (debug) {
 665                 log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
 666
 667 #ifdef FIO_INC_DEBUG
 668                 fio_debug |= 1U << FD_TIME;
 669 #endif
 670                 nr_entries = CLOCK_ENTRIES_DEBUG;
 671         } else
 672                 nr_entries = CLOCK_ENTRIES_TEST;
 673
 674         calibrate_cpu_clock();
 675
 676         if (debug) {
 677 #ifdef FIO_INC_DEBUG
 678                 fio_debug &= ~(1U << FD_TIME);
 679 #endif
 680         }
 681
 682         cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
 683         tentries = nr_entries * nr_cpus;
 684         entries = malloc(tentries * sizeof(struct clock_entry));
 685
 686         if (debug)
 687                 log_info("cs: Testing %u CPUs\n", nr_cpus);
 688
 689         for (i = 0; i < nr_cpus; i++) {
 690                 struct clock_thread *t = &cthreads[i];
 691
 692                 t->cpu = i;
 693                 t->debug = debug;
 694                 t->seq = &seq;
 695                 t->nr_entries = nr_entries;
 696                 t->entries = &entries[i * nr_entries];
 697                 __fio_sem_init(&t->lock, FIO_SEM_LOCKED);
 698                 if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
 699                         failed++;
 700                         nr_cpus = i;
 701                         break;
 702                 }
 703         }
 704
 705         for (i = 0; i < nr_cpus; i++) {
 706                 struct clock_thread *t = &cthreads[i];
 707
 708                 fio_sem_up(&t->lock);
 709         }
 710
 711         for (i = 0; i < nr_cpus; i++) {
 712                 struct clock_thread *t = &cthreads[i];
 713                 void *ret;
 714
 715                 pthread_join(t->thread, &ret);
 716                 if (ret)
 717                         failed++;
 718                 __fio_sem_remove(&t->lock);
 719         }
 720         free(cthreads);
 721
 722         if (failed) {
 723                 if (debug)
 724                         log_err("Clocksource test: %lu threads failed\n", failed);
 725                 goto err;
 726         }
 727
 728         qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
 729
 730         /* silence silly gcc */
 731         prev = NULL;
 732         for (failed = i = 0; i < tentries; i++) {
 733                 this = &entries[i];
 734
 735                 if (!i) {
 736                         prev = this;
 737                         continue;
 738                 }
 739
 740                 if (prev->tsc > this->tsc) {
 741                         uint64_t diff = prev->tsc - this->tsc;
 742
 743                         if (!debug) {
 744                                 failed++;
 745                                 break;
 746                         }
 747
 748                         log_info("cs: CPU clock mismatch (diff=%llu):\n",
 749                                                 (unsigned long long) diff);
 750                         log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
 751                         log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
 752                         failed++;
 753                 }
 754
 755                 prev = this;
 756         }
 757
 758         if (debug) {
 759                 if (failed)
 760                         log_info("cs: Failed: %lu\n", failed);
 761                 else
 762                         log_info("cs: Pass!\n");
 763         }
 764 err:
 765         free(entries);
 766         return !!failed;
 767 }
 768
 769 #else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
 770
 771 int fio_monotonic_clocktest(int debug)
 772 {
 773         if (debug)
 774                 log_info("cs: current platform does not support CPU clocks\n");
 775         return 1;
 776 }
 777
 778 #endif