+ if (fio_clock_source == fio_clock_source_inited)
+ return;
+
+#ifndef CONFIG_TLS_THREAD
+ if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
+ log_err("fio: can't create TLS key\n");
+#endif
+
+ fio_clock_source_inited = fio_clock_source;
+
+ if (calibrate_cpu_clock())
+ tsc_reliable = false;
+
+ /*
+ * If the arch sets tsc_reliable != 0, then it must be good enough
+ * to use as THE clock source. For x86 CPUs, this means the TSC
+ * runs at a constant rate and is synced across CPU cores.
+ */
+ if (tsc_reliable) {
+ if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
+ fio_clock_source = CS_CPUCLOCK;
+ } else if (fio_clock_source == CS_CPUCLOCK)
+ log_info("fio: clocksource=cpu may not be reliable\n");
+ dprint(FD_TIME, "gettime: clocksource=%d\n", (int) fio_clock_source);
+}
+
+uint64_t ntime_since(const struct timespec *s, const struct timespec *e)
+{
+ int64_t sec, nsec;
+
+ sec = e->tv_sec - s->tv_sec;
+ nsec = e->tv_nsec - s->tv_nsec;
+ if (sec > 0 && nsec < 0) {
+ sec--;
+ nsec += 1000000000LL;
+ }
+
+ /*
+ * time warp bug on some kernels?
+ */
+ if (sec < 0 || (sec == 0 && nsec < 0))
+ return 0;
+
+ return nsec + (sec * 1000000000LL);
+}
+
+uint64_t ntime_since_now(const struct timespec *s)
+{
+ struct timespec now;
+
+ fio_gettime(&now, NULL);
+ return ntime_since(s, &now);
+}
+
+uint64_t utime_since(const struct timespec *s, const struct timespec *e)
+{
+ int64_t sec, usec;
+
+ sec = e->tv_sec - s->tv_sec;
+ usec = (e->tv_nsec - s->tv_nsec) / 1000;
+ if (sec > 0 && usec < 0) {
+ sec--;
+ usec += 1000000;
+ }
+
+ /*
+ * time warp bug on some kernels?
+ */
+ if (sec < 0 || (sec == 0 && usec < 0))
+ return 0;
+
+ return usec + (sec * 1000000);
+}
+
+uint64_t utime_since_now(const struct timespec *s)
+{
+ struct timespec t;
+#ifdef FIO_DEBUG_TIME
+ void *p = __builtin_return_address(0);
+
+ fio_gettime(&t, p);
+#else
+ fio_gettime(&t, NULL);
+#endif
+
+ return utime_since(s, &t);
+}
+
+uint64_t mtime_since_tv(const struct timeval *s, const struct timeval *e)
+{
+ int64_t sec, usec;
+
+ sec = e->tv_sec - s->tv_sec;
+ usec = (e->tv_usec - s->tv_usec);
+ if (sec > 0 && usec < 0) {
+ sec--;
+ usec += 1000000;
+ }
+
+ if (sec < 0 || (sec == 0 && usec < 0))
+ return 0;
+
+ sec *= 1000;
+ usec /= 1000;
+ return sec + usec;
+}
+
+uint64_t mtime_since_now(const struct timespec *s)
+{
+ struct timespec t;
+#ifdef FIO_DEBUG_TIME
+ void *p = __builtin_return_address(0);
+
+ fio_gettime(&t, p);
+#else
+ fio_gettime(&t, NULL);
+#endif
+
+ return mtime_since(s, &t);
+}
+
+uint64_t mtime_since(const struct timespec *s, const struct timespec *e)
+{
+ int64_t sec, usec;
+
+ sec = e->tv_sec - s->tv_sec;
+ usec = (e->tv_nsec - s->tv_nsec) / 1000;
+ if (sec > 0 && usec < 0) {
+ sec--;
+ usec += 1000000;
+ }
+
+ if (sec < 0 || (sec == 0 && usec < 0))
+ return 0;
+
+ sec *= 1000;
+ usec /= 1000;
+ return sec + usec;
+}
+
+uint64_t time_since_now(const struct timespec *s)
+{
+ return mtime_since_now(s) / 1000;
+}
+
+#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
+ defined(CONFIG_SYNC_SYNC) && defined(CONFIG_CMP_SWAP)
+
+#define CLOCK_ENTRIES_DEBUG 100000
+#define CLOCK_ENTRIES_TEST 1000
+
+struct clock_entry {
+ uint32_t seq;
+ uint32_t cpu;
+ uint64_t tsc;
+};
+
+struct clock_thread {
+ pthread_t thread;
+ int cpu;
+ int debug;
+ struct fio_sem lock;
+ unsigned long nr_entries;
+ uint32_t *seq;
+ struct clock_entry *entries;
+};
+
+static inline uint32_t atomic32_compare_and_swap(uint32_t *ptr, uint32_t old,
+ uint32_t new)
+{
+ return __sync_val_compare_and_swap(ptr, old, new);
+}
+
+static void *clock_thread_fn(void *data)
+{
+ struct clock_thread *t = data;
+ struct clock_entry *c;
+ os_cpu_mask_t cpu_mask;
+ unsigned long long first;
+ int i;
+
+ if (fio_cpuset_init(&cpu_mask)) {
+ int __err = errno;
+
+ log_err("clock cpuset init failed: %s\n", strerror(__err));
+ goto err_out;
+ }
+
+ fio_cpu_set(&cpu_mask, t->cpu);
+
+ if (fio_setaffinity(gettid(), cpu_mask) == -1) {
+ int __err = errno;
+
+ log_err("clock setaffinity failed: %s\n", strerror(__err));
+ goto err;
+ }
+
+ fio_sem_down(&t->lock);
+
+ first = get_cpu_clock();
+ c = &t->entries[0];
+ for (i = 0; i < t->nr_entries; i++, c++) {
+ uint32_t seq;
+ uint64_t tsc;
+
+ c->cpu = t->cpu;
+ do {
+ seq = *t->seq;
+ if (seq == UINT_MAX)
+ break;
+ __sync_synchronize();
+ tsc = get_cpu_clock();
+ } while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1));
+
+ if (seq == UINT_MAX)
+ break;
+
+ c->seq = seq;
+ c->tsc = tsc;
+ }
+
+ if (t->debug) {
+ unsigned long long clocks;
+
+ clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
+ log_info("cs: cpu%3d: %llu clocks seen, first %llu\n", t->cpu,
+ clocks, first);
+ }
+
+ /*
+ * The most common platform clock breakage is returning zero
+ * indefinitely. Check for that and return failure.
+ */
+ if (i > 1 && !t->entries[i - 1].tsc && !t->entries[0].tsc)
+ goto err;
+
+ fio_cpuset_exit(&cpu_mask);
+ return NULL;
+err:
+ fio_cpuset_exit(&cpu_mask);
+err_out:
+ return (void *) 1;