[fio.git] / gettime.c

/*
 * Clock functions
 */

#include <unistd.h>
#include <math.h>
#include <sys/time.h>
#include <time.h>

#include "fio.h"
#include "smalloc.h"

#include "hash.h"
#include "os/os.h"

#if defined(ARCH_HAVE_CPU_CLOCK)
#ifndef ARCH_CPU_CLOCK_CYCLES_PER_USEC
static unsigned long cycles_per_usec;
static unsigned long inv_cycles_per_usec;
static uint64_t max_cycles_for_mult;
#endif
#ifdef ARCH_CPU_CLOCK_WRAPS
static unsigned long long cycles_start, cycles_wrap;
#endif
#endif
int tsc_reliable = 0;

struct tv_valid {
	uint64_t last_cycles;
	int last_tv_valid;
	int warned;
};
#ifdef ARCH_HAVE_CPU_CLOCK
#ifdef CONFIG_TLS_THREAD
static __thread struct tv_valid static_tv_valid;
#else
static pthread_key_t tv_tls_key;
#endif
#endif

enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
int fio_clock_source_set = 0;
static enum fio_cs fio_clock_source_inited = CS_INVAL;

#ifdef FIO_DEBUG_TIME

#define HASH_BITS	8
#define HASH_SIZE	(1 << HASH_BITS)

static struct flist_head hash[HASH_SIZE];
static int gtod_inited;

struct gtod_log {
	struct flist_head list;
	void *caller;
	unsigned long calls;
};

static struct gtod_log *find_hash(void *caller)
{
	unsigned long h = hash_ptr(caller, HASH_BITS);
	struct flist_head *entry;

	flist_for_each(entry, &hash[h]) {
		struct gtod_log *log = flist_entry(entry, struct gtod_log,
									list);

		if (log->caller == caller)
			return log;
	}

	return NULL;
}

static void inc_caller(void *caller)
{
	struct gtod_log *log = find_hash(caller);

	if (!log) {
		unsigned long h;

		log = malloc(sizeof(*log));
		INIT_FLIST_HEAD(&log->list);
		log->caller = caller;
		log->calls = 0;

		h = hash_ptr(caller, HASH_BITS);
		flist_add_tail(&log->list, &hash[h]);
	}

	log->calls++;
}

static void gtod_log_caller(void *caller)
{
	if (gtod_inited)
		inc_caller(caller);
}

static void fio_exit fio_dump_gtod(void)
{
	unsigned long total_calls = 0;
	int i;

	for (i = 0; i < HASH_SIZE; i++) {
		struct flist_head *entry;
		struct gtod_log *log;

		flist_for_each(entry, &hash[i]) {
			log = flist_entry(entry, struct gtod_log, list);

			printf("function %p, calls %lu\n", log->caller,
								log->calls);
			total_calls += log->calls;
		}
	}

	printf("Total %lu gettimeofday\n", total_calls);
}

static void fio_init gtod_init(void)
{
	int i;

	for (i = 0; i < HASH_SIZE; i++)
		INIT_FLIST_HEAD(&hash[i]);

	gtod_inited = 1;
}

#endif /* FIO_DEBUG_TIME */

#ifdef CONFIG_CLOCK_GETTIME
static int fill_clock_gettime(struct timespec *ts)
{
#ifdef CONFIG_CLOCK_MONOTONIC
	return clock_gettime(CLOCK_MONOTONIC, ts);
#else
	return clock_gettime(CLOCK_REALTIME, ts);
#endif
}
#endif

static void __fio_gettime(struct timeval *tp)
{
	switch (fio_clock_source) {
#ifdef CONFIG_GETTIMEOFDAY
	case CS_GTOD:
		gettimeofday(tp, NULL);
		break;
#endif
#ifdef CONFIG_CLOCK_GETTIME
	case CS_CGETTIME: {
		struct timespec ts;

		if (fill_clock_gettime(&ts) < 0) {
			log_err("fio: clock_gettime fails\n");
			assert(0);
		}

		tp->tv_sec = ts.tv_sec;
		tp->tv_usec = ts.tv_nsec / 1000;
		break;
		}
#endif
#ifdef ARCH_HAVE_CPU_CLOCK
	case CS_CPUCLOCK: {
		uint64_t usecs, t;
		struct tv_valid *tv;

#ifdef CONFIG_TLS_THREAD
		tv = &static_tv_valid;
#else
		tv = pthread_getspecific(tv_tls_key);
#endif

		t = get_cpu_clock();
#ifdef ARCH_CPU_CLOCK_WRAPS
		if (t < cycles_start && !cycles_wrap)
			cycles_wrap = 1;
		else if (cycles_wrap && t >= cycles_start && !tv->warned) {
			log_err("fio: double CPU clock wrap\n");
			tv->warned = 1;
		}

		t -= cycles_start;
#endif
		tv->last_cycles = t;
		tv->last_tv_valid = 1;
#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
		usecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC;
#else
		if (t < max_cycles_for_mult)
			usecs = (t * inv_cycles_per_usec) / 16777216UL;
		else
			usecs = t / cycles_per_usec;
#endif
		tp->tv_sec = usecs / 1000000;
		tp->tv_usec = usecs % 1000000;
		break;
		}
#endif
	default:
		log_err("fio: invalid clock source %d\n", fio_clock_source);
		break;
	}
}

#ifdef FIO_DEBUG_TIME
void fio_gettime(struct timeval *tp, void *caller)
#else
void fio_gettime(struct timeval *tp, void fio_unused *caller)
#endif
{
#ifdef FIO_DEBUG_TIME
	if (!caller)
		caller = __builtin_return_address(0);

	gtod_log_caller(caller);
#endif
	if (fio_unlikely(fio_gettime_offload(tp)))
		return;

	__fio_gettime(tp);
}

#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
static unsigned long get_cycles_per_usec(void)
{
	struct timeval s, e;
	uint64_t c_s, c_e;
	enum fio_cs old_cs = fio_clock_source;

#ifdef CONFIG_CLOCK_GETTIME
	fio_clock_source = CS_CGETTIME;
#else
	fio_clock_source = CS_GTOD;
#endif
	__fio_gettime(&s);

	c_s = get_cpu_clock();
	do {
		uint64_t elapsed;

		__fio_gettime(&e);

		elapsed = utime_since(&s, &e);
		if (elapsed >= 1280) {
			c_e = get_cpu_clock();
			break;
		}
	} while (1);

	fio_clock_source = old_cs;
	return (c_e - c_s + 127) >> 7;
}

#define NR_TIME_ITERS	50

static int calibrate_cpu_clock(void)
{
	double delta, mean, S;
	uint64_t minc, maxc, avg, cycles[NR_TIME_ITERS];
	int i, samples;

	cycles[0] = get_cycles_per_usec();
	S = delta = mean = 0.0;
	for (i = 0; i < NR_TIME_ITERS; i++) {
		cycles[i] = get_cycles_per_usec();
		delta = cycles[i] - mean;
		if (delta) {
			mean += delta / (i + 1.0);
			S += delta * (cycles[i] - mean);
		}
	}

	/*
	 * The most common platform clock breakage is returning zero
	 * indefinitely. Check for that and return failure.
	 */
	if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
		return 1;

	S = sqrt(S / (NR_TIME_ITERS - 1.0));

	minc = -1ULL;
	maxc = samples = avg = 0;
	for (i = 0; i < NR_TIME_ITERS; i++) {
		double this = cycles[i];

		minc = min(cycles[i], minc);
		maxc = max(cycles[i], maxc);

		if ((fmax(this, mean) - fmin(this, mean)) > S)
			continue;
		samples++;
		avg += this;
	}

	S /= (double) NR_TIME_ITERS;
	mean /= 10.0;

	for (i = 0; i < NR_TIME_ITERS; i++)
		dprint(FD_TIME, "cycles[%d]=%llu\n", i,
					(unsigned long long) cycles[i] / 10);

	avg /= samples;
	avg = (avg + 5) / 10;
	minc /= 10;
	maxc /= 10;
	dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
	dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f\n",
			(unsigned long long) minc,
			(unsigned long long) maxc, mean, S);

	cycles_per_usec = avg;
	inv_cycles_per_usec = 16777216UL / cycles_per_usec;
	max_cycles_for_mult = ~0ULL / inv_cycles_per_usec;
	dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
#ifdef ARCH_CPU_CLOCK_WRAPS
	cycles_start = get_cpu_clock();
	dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
#endif
	return 0;
}
#else
static int calibrate_cpu_clock(void)
{
#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
	return 0;
#else
	return 1;
#endif
}
#endif // ARCH_HAVE_CPU_CLOCK

#ifndef CONFIG_TLS_THREAD
void fio_local_clock_init(int is_thread)
{
	struct tv_valid *t;

	t = calloc(1, sizeof(*t));
	if (pthread_setspecific(tv_tls_key, t)) {
		log_err("fio: can't set TLS key\n");
		assert(0);
	}
}

static void kill_tv_tls_key(void *data)
{
	free(data);
}
#else
void fio_local_clock_init(int is_thread)
{
}
#endif

void fio_clock_init(void)
{
	if (fio_clock_source == fio_clock_source_inited)
		return;

#ifndef CONFIG_TLS_THREAD
	if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
		log_err("fio: can't create TLS key\n");
#endif

	fio_clock_source_inited = fio_clock_source;

	if (calibrate_cpu_clock())
		tsc_reliable = 0;

	/*
	 * If the arch sets tsc_reliable != 0, then it must be good enough
	 * to use as THE clock source. For x86 CPUs, this means the TSC
	 * runs at a constant rate and is synced across CPU cores.
	 */
	if (tsc_reliable) {
		if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
			fio_clock_source = CS_CPUCLOCK;
	} else if (fio_clock_source == CS_CPUCLOCK)
		log_info("fio: clocksource=cpu may not be reliable\n");
}

uint64_t utime_since(const struct timeval *s, const struct timeval *e)
{
	long sec, usec;
	uint64_t ret;

	sec = e->tv_sec - s->tv_sec;
	usec = e->tv_usec - s->tv_usec;
	if (sec > 0 && usec < 0) {
		sec--;
		usec += 1000000;
	}

	/*
	 * time warp bug on some kernels?
	 */
	if (sec < 0 || (sec == 0 && usec < 0))
		return 0;

	ret = sec * 1000000ULL + usec;

	return ret;
}

uint64_t utime_since_now(const struct timeval *s)
{
	struct timeval t;

	fio_gettime(&t, NULL);
	return utime_since(s, &t);
}

uint64_t mtime_since(const struct timeval *s, const struct timeval *e)
{
	long sec, usec, ret;

	sec = e->tv_sec - s->tv_sec;
	usec = e->tv_usec - s->tv_usec;
	if (sec > 0 && usec < 0) {
		sec--;
		usec += 1000000;
	}

	if (sec < 0 || (sec == 0 && usec < 0))
		return 0;

	sec *= 1000UL;
	usec /= 1000UL;
	ret = sec + usec;

	return ret;
}

uint64_t mtime_since_now(const struct timeval *s)
{
	struct timeval t;
	void *p = __builtin_return_address(0);

	fio_gettime(&t, p);
	return mtime_since(s, &t);
}

uint64_t time_since_now(const struct timeval *s)
{
	return mtime_since_now(s) / 1000;
}

#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \
    defined(CONFIG_SFAA)

#define CLOCK_ENTRIES_DEBUG	100000
#define CLOCK_ENTRIES_TEST	10000

struct clock_entry {
	uint32_t seq;
	uint32_t cpu;
	uint64_t tsc;
};

struct clock_thread {
	pthread_t thread;
	int cpu;
	int debug;
	pthread_mutex_t lock;
	pthread_mutex_t started;
	unsigned long nr_entries;
	uint32_t *seq;
	struct clock_entry *entries;
};

static inline uint32_t atomic32_inc_return(uint32_t *seq)
{
	return 1 + __sync_fetch_and_add(seq, 1);
}

static void *clock_thread_fn(void *data)
{
	struct clock_thread *t = data;
	struct clock_entry *c;
	os_cpu_mask_t cpu_mask;
	uint32_t last_seq;
	int i;

	if (fio_cpuset_init(&cpu_mask)) {
		int __err = errno;

		log_err("clock cpuset init failed: %s\n", strerror(__err));
		goto err_out;
	}

	fio_cpu_set(&cpu_mask, t->cpu);

	if (fio_setaffinity(gettid(), cpu_mask) == -1) {
		int __err = errno;

		log_err("clock setaffinity failed: %s\n", strerror(__err));
		goto err;
	}

	pthread_mutex_lock(&t->lock);
	pthread_mutex_unlock(&t->started);

	last_seq = 0;
	c = &t->entries[0];
	for (i = 0; i < t->nr_entries; i++, c++) {
		uint32_t seq;
		uint64_t tsc;

		c->cpu = t->cpu;
		do {
			seq = atomic32_inc_return(t->seq);
			if (seq < last_seq)
				break;
			tsc = get_cpu_clock();
		} while (seq != *t->seq);

		c->seq = seq;
		c->tsc = tsc;
	}

	if (t->debug) {
		unsigned long long clocks;

		clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
		log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu, clocks);
	}

	/*
	 * The most common platform clock breakage is returning zero
	 * indefinitely. Check for that and return failure.
	 */
	if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
		goto err;

	fio_cpuset_exit(&cpu_mask);
	return NULL;
err:
	fio_cpuset_exit(&cpu_mask);
err_out:
	return (void *) 1;
}

static int clock_cmp(const void *p1, const void *p2)
{
	const struct clock_entry *c1 = p1;
	const struct clock_entry *c2 = p2;

	if (c1->seq == c2->seq)
		log_err("cs: bug in atomic sequence!\n");

	return c1->seq - c2->seq;
}

int fio_monotonic_clocktest(int debug)
{
	struct clock_thread *cthreads;
	unsigned int nr_cpus = cpus_online();
	struct clock_entry *entries;
	unsigned long nr_entries, tentries, failed = 0;
	struct clock_entry *prev, *this;
	uint32_t seq = 0;
	unsigned int i;

	if (debug) {
		log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");

#ifdef FIO_INC_DEBUG
		fio_debug |= 1U << FD_TIME;
#endif
		nr_entries = CLOCK_ENTRIES_DEBUG;
	} else
		nr_entries = CLOCK_ENTRIES_TEST;

	calibrate_cpu_clock();

	if (debug) {
#ifdef FIO_INC_DEBUG
		fio_debug &= ~(1U << FD_TIME);
#endif
	}

	cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
	tentries = nr_entries * nr_cpus;
	entries = malloc(tentries * sizeof(struct clock_entry));

	if (debug)
		log_info("cs: Testing %u CPUs\n", nr_cpus);

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];

		t->cpu = i;
		t->debug = debug;
		t->seq = &seq;
		t->nr_entries = nr_entries;
		t->entries = &entries[i * nr_entries];
		pthread_mutex_init(&t->lock, NULL);
		pthread_mutex_init(&t->started, NULL);
		pthread_mutex_lock(&t->lock);
		if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
			failed++;
			nr_cpus = i;
			break;
		}
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];

		pthread_mutex_lock(&t->started);
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];

		pthread_mutex_unlock(&t->lock);
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];
		void *ret;

		pthread_join(t->thread, &ret);
		if (ret)
			failed++;
	}
	free(cthreads);

	if (failed) {
		if (debug)
			log_err("Clocksource test: %lu threads failed\n", failed);
		goto err;
	}

	qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);

	for (failed = i = 0; i < tentries; i++) {
		this = &entries[i];

		if (!i) {
			prev = this;
			continue;
		}

		if (prev->tsc > this->tsc) {
			uint64_t diff = prev->tsc - this->tsc;

			if (!debug) {
				failed++;
				break;
			}

			log_info("cs: CPU clock mismatch (diff=%llu):\n",
						(unsigned long long) diff);
			log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
			log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
			failed++;
		}

		prev = this;
	}

	if (debug) {
		if (failed)
			log_info("cs: Failed: %lu\n", failed);
		else
			log_info("cs: Pass!\n");
	}
err:
	free(entries);
	return !!failed;
}

#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */

int fio_monotonic_clocktest(int debug)
{
	if (debug)
		log_info("cs: current platform does not support CPU clocks\n");
	return 1;
}

#endif
Commit	Line	Data
02bcaa8c	1	/*
f5cc024a	2	* Clock functions
02bcaa8c	3	*/
f5cc024a	4
02bcaa8c	5	#include <unistd.h>
c223da83	6	#include <math.h>
02bcaa8c	7	#include <sys/time.h>
03e20d68	8	#include <time.h>
02bcaa8c JA	9
02bcaa8c JA	10	#include "fio.h"
be4ecfdf	11	#include "smalloc.h"
02bcaa8c JA	12
02bcaa8c JA	13	#include "hash.h"
7d11f871	14	#include "os/os.h"
02bcaa8c	15
dac499a0 AV	16	#if defined(ARCH_HAVE_CPU_CLOCK)
dac499a0 AV	17	#ifndef ARCH_CPU_CLOCK_CYCLES_PER_USEC
c223da83	18	static unsigned long cycles_per_usec;
71339117	19	static unsigned long inv_cycles_per_usec;
b3fa625b	20	static uint64_t max_cycles_for_mult;
96170421 CE	21	#endif
96170421 CE	22	#ifdef ARCH_CPU_CLOCK_WRAPS
73df3e07	23	static unsigned long long cycles_start, cycles_wrap;
09a32402	24	#endif
dac499a0	25	#endif
4de98eb0	26	int tsc_reliable = 0;
5d879392 JA	27
5d879392 JA	28	struct tv_valid {
ba458c2f	29	uint64_t last_cycles;
02dcf81c JA	30	int last_tv_valid;
02dcf81c JA	31	int warned;
5d879392	32	};
03be65c9	33	#ifdef ARCH_HAVE_CPU_CLOCK
67bf9823	34	#ifdef CONFIG_TLS_THREAD
b4ea84da	35	static __thread struct tv_valid static_tv_valid;
67bf9823	36	#else
5d879392	37	static pthread_key_t tv_tls_key;
67bf9823	38	#endif
03be65c9	39	#endif
02bcaa8c	40
16de1bf9	41	enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
fa80feae	42	int fio_clock_source_set = 0;
10aa136b	43	static enum fio_cs fio_clock_source_inited = CS_INVAL;
c223da83	44
02bcaa8c JA	45	#ifdef FIO_DEBUG_TIME
	46
	47	#define HASH_BITS 8
	48	#define HASH_SIZE (1 << HASH_BITS)
	49
01743ee1	50	static struct flist_head hash[HASH_SIZE];
02bcaa8c JA	51	static int gtod_inited;
	52
	53	struct gtod_log {
01743ee1	54	struct flist_head list;
02bcaa8c JA	55	void *caller;
	56	unsigned long calls;
	57	};
	58
	59	static struct gtod_log find_hash(void caller)
	60	{
	61	unsigned long h = hash_ptr(caller, HASH_BITS);
01743ee1	62	struct flist_head *entry;
02bcaa8c	63
01743ee1 JA	64	flist_for_each(entry, &hash[h]) {
	65	struct gtod_log *log = flist_entry(entry, struct gtod_log,
	66	list);
02bcaa8c JA	67
	68	if (log->caller == caller)
	69	return log;
	70	}
	71
	72	return NULL;
	73	}
	74
d5e16441	75	static void inc_caller(void *caller)
02bcaa8c JA	76	{
	77	struct gtod_log *log = find_hash(caller);
	78
	79	if (!log) {
	80	unsigned long h;
	81
	82	log = malloc(sizeof(*log));
01743ee1	83	INIT_FLIST_HEAD(&log->list);
02bcaa8c JA	84	log->caller = caller;
	85	log->calls = 0;
	86
	87	h = hash_ptr(caller, HASH_BITS);
01743ee1	88	flist_add_tail(&log->list, &hash[h]);
02bcaa8c JA	89	}
02bcaa8c JA	90
d5e16441	91	log->calls++;
02bcaa8c JA	92	}
	93
	94	static void gtod_log_caller(void *caller)
	95	{
d5e16441 JA	96	if (gtod_inited)
d5e16441 JA	97	inc_caller(caller);
02bcaa8c JA	98	}
	99
	100	static void fio_exit fio_dump_gtod(void)
	101	{
	102	unsigned long total_calls = 0;
	103	int i;
	104
	105	for (i = 0; i < HASH_SIZE; i++) {
01743ee1	106	struct flist_head *entry;
02bcaa8c JA	107	struct gtod_log *log;
02bcaa8c JA	108
01743ee1 JA	109	flist_for_each(entry, &hash[i]) {
01743ee1 JA	110	log = flist_entry(entry, struct gtod_log, list);
02bcaa8c	111
5ec10eaa JA	112	printf("function %p, calls %lu\n", log->caller,
5ec10eaa JA	113	log->calls);
02bcaa8c JA	114	total_calls += log->calls;
	115	}
	116	}
	117
	118	printf("Total %lu gettimeofday\n", total_calls);
	119	}
	120
	121	static void fio_init gtod_init(void)
	122	{
	123	int i;
	124
	125	for (i = 0; i < HASH_SIZE; i++)
01743ee1	126	INIT_FLIST_HEAD(&hash[i]);
02bcaa8c JA	127
	128	gtod_inited = 1;
	129	}
	130
	131	#endif /* FIO_DEBUG_TIME */
	132
67bf9823	133	#ifdef CONFIG_CLOCK_GETTIME
9ff1c070 JA	134	static int fill_clock_gettime(struct timespec *ts)
9ff1c070 JA	135	{
67bf9823	136	#ifdef CONFIG_CLOCK_MONOTONIC
9ff1c070 JA	137	return clock_gettime(CLOCK_MONOTONIC, ts);
	138	#else
	139	return clock_gettime(CLOCK_REALTIME, ts);
	140	#endif
	141	}
1e97cce9	142	#endif
67bf9823	143
9eb271b9	144	static void __fio_gettime(struct timeval *tp)
02bcaa8c	145	{
c223da83	146	switch (fio_clock_source) {
67bf9823	147	#ifdef CONFIG_GETTIMEOFDAY
c223da83	148	case CS_GTOD:
02bcaa8c	149	gettimeofday(tp, NULL);
c223da83	150	break;
67bf9823 JA	151	#endif
67bf9823 JA	152	#ifdef CONFIG_CLOCK_GETTIME
c223da83	153	case CS_CGETTIME: {
02bcaa8c JA	154	struct timespec ts;
02bcaa8c JA	155
9ff1c070	156	if (fill_clock_gettime(&ts) < 0) {
c223da83 JA	157	log_err("fio: clock_gettime fails\n");
c223da83 JA	158	assert(0);
02bcaa8c JA	159	}
	160
	161	tp->tv_sec = ts.tv_sec;
	162	tp->tv_usec = ts.tv_nsec / 1000;
c223da83 JA	163	break;
c223da83 JA	164	}
67bf9823	165	#endif
c223da83 JA	166	#ifdef ARCH_HAVE_CPU_CLOCK
c223da83 JA	167	case CS_CPUCLOCK: {
ba458c2f	168	uint64_t usecs, t;
03be65c9 JA	169	struct tv_valid *tv;
	170
	171	#ifdef CONFIG_TLS_THREAD
	172	tv = &static_tv_valid;
	173	#else
	174	tv = pthread_getspecific(tv_tls_key);
	175	#endif
c223da83 JA	176
c223da83 JA	177	t = get_cpu_clock();
96170421	178	#ifdef ARCH_CPU_CLOCK_WRAPS
73df3e07 JA	179	if (t < cycles_start && !cycles_wrap)
73df3e07 JA	180	cycles_wrap = 1;
03be65c9 JA	181	else if (cycles_wrap && t >= cycles_start && !tv->warned) {
	182	log_err("fio: double CPU clock wrap\n");
	183	tv->warned = 1;
02dcf81c	184	}
c223da83	185
73df3e07	186	t -= cycles_start;
96170421	187	#endif
9eb271b9 JA	188	tv->last_cycles = t;
9eb271b9 JA	189	tv->last_tv_valid = 1;
919e789d CE	190	#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
	191	usecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC;
	192	#else
b3fa625b JA	193	if (t < max_cycles_for_mult)
	194	usecs = (t * inv_cycles_per_usec) / 16777216UL;
	195	else
	196	usecs = t / cycles_per_usec;
919e789d	197	#endif
c223da83 JA	198	tp->tv_sec = usecs / 1000000;
c223da83 JA	199	tp->tv_usec = usecs % 1000000;
c223da83 JA	200	break;
	201	}
	202	#endif
	203	default:
	204	log_err("fio: invalid clock source %d\n", fio_clock_source);
	205	break;
02bcaa8c	206	}
67bf9823 JA	207	}
	208
	209	#ifdef FIO_DEBUG_TIME
	210	void fio_gettime(struct timeval tp, void caller)
	211	#else
	212	void fio_gettime(struct timeval tp, void fio_unused caller)
	213	#endif
	214	{
67bf9823 JA	215	#ifdef FIO_DEBUG_TIME
	216	if (!caller)
	217	caller = __builtin_return_address(0);
	218
	219	gtod_log_caller(caller);
	220	#endif
27325ed5	221	if (fio_unlikely(fio_gettime_offload(tp)))
67bf9823	222	return;
67bf9823	223
9eb271b9	224	__fio_gettime(tp);
02bcaa8c	225	}
be4ecfdf	226
919e789d	227	#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
c223da83 JA	228	static unsigned long get_cycles_per_usec(void)
	229	{
	230	struct timeval s, e;
ba458c2f	231	uint64_t c_s, c_e;
67bf9823	232	enum fio_cs old_cs = fio_clock_source;
c223da83	233
67bf9823 JA	234	#ifdef CONFIG_CLOCK_GETTIME
	235	fio_clock_source = CS_CGETTIME;
	236	#else
	237	fio_clock_source = CS_GTOD;
	238	#endif
	239	__fio_gettime(&s);
9ff1c070	240
c223da83 JA	241	c_s = get_cpu_clock();
c223da83 JA	242	do {
ba458c2f	243	uint64_t elapsed;
c223da83	244
67bf9823	245	__fio_gettime(&e);
9ff1c070	246
c223da83	247	elapsed = utime_since(&s, &e);
486332e5	248	if (elapsed >= 1280) {
c223da83 JA	249	c_e = get_cpu_clock();
	250	break;
	251	}
	252	} while (1);
	253
67bf9823	254	fio_clock_source = old_cs;
c011000d	255	return (c_e - c_s + 127) >> 7;
c223da83 JA	256	}
c223da83 JA	257
fa80feae JA	258	#define NR_TIME_ITERS 50
fa80feae JA	259
e259879e	260	static int calibrate_cpu_clock(void)
c223da83 JA	261	{
c223da83 JA	262	double delta, mean, S;
e51a6629	263	uint64_t minc, maxc, avg, cycles[NR_TIME_ITERS];
c223da83 JA	264	int i, samples;
c223da83 JA	265
c223da83 JA	266	cycles[0] = get_cycles_per_usec();
c223da83 JA	267	S = delta = mean = 0.0;
fa80feae	268	for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83 JA	269	cycles[i] = get_cycles_per_usec();
	270	delta = cycles[i] - mean;
	271	if (delta) {
	272	mean += delta / (i + 1.0);
	273	S += delta * (cycles[i] - mean);
	274	}
	275	}
	276
e259879e JA	277	/*
	278	* The most common platform clock breakage is returning zero
	279	* indefinitely. Check for that and return failure.
	280	*/
	281	if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
	282	return 1;
	283
fa80feae	284	S = sqrt(S / (NR_TIME_ITERS - 1.0));
c223da83	285
e51a6629 JA	286	minc = -1ULL;
e51a6629 JA	287	maxc = samples = avg = 0;
fa80feae	288	for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83 JA	289	double this = cycles[i];
c223da83 JA	290
e51a6629 JA	291	minc = min(cycles[i], minc);
	292	maxc = max(cycles[i], maxc);
	293
03e20d68	294	if ((fmax(this, mean) - fmin(this, mean)) > S)
c223da83 JA	295	continue;
	296	samples++;
	297	avg += this;
	298	}
	299
fa80feae	300	S /= (double) NR_TIME_ITERS;
89db727d	301	mean /= 10.0;
c223da83	302
fa80feae	303	for (i = 0; i < NR_TIME_ITERS; i++)
4b91ee8f JA	304	dprint(FD_TIME, "cycles[%d]=%llu\n", i,
4b91ee8f JA	305	(unsigned long long) cycles[i] / 10);
c223da83	306
d7abad3d	307	avg /= samples;
b0ff22d7	308	avg = (avg + 5) / 10;
e51a6629 JA	309	minc /= 10;
e51a6629 JA	310	maxc /= 10;
4b91ee8f	311	dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
e51a6629 JA	312	dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f\n",
	313	(unsigned long long) minc,
	314	(unsigned long long) maxc, mean, S);
c223da83 JA	315
c223da83 JA	316	cycles_per_usec = avg;
71339117	317	inv_cycles_per_usec = 16777216UL / cycles_per_usec;
b3fa625b	318	max_cycles_for_mult = ~0ULL / inv_cycles_per_usec;
71339117	319	dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
96170421	320	#ifdef ARCH_CPU_CLOCK_WRAPS
73df3e07 JA	321	cycles_start = get_cpu_clock();
73df3e07 JA	322	dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
96170421	323	#endif
e259879e	324	return 0;
09a32402 JA	325	}
09a32402 JA	326	#else
e259879e	327	static int calibrate_cpu_clock(void)
09a32402	328	{
919e789d CE	329	#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
	330	return 0;
	331	#else
e259879e	332	return 1;
09a32402	333	#endif
919e789d CE	334	}
919e789d CE	335	#endif // ARCH_HAVE_CPU_CLOCK
09a32402	336
67bf9823	337	#ifndef CONFIG_TLS_THREAD
5d879392 JA	338	void fio_local_clock_init(int is_thread)
	339	{
	340	struct tv_valid *t;
	341
572cfb3f	342	t = calloc(1, sizeof(*t));
9eb271b9	343	if (pthread_setspecific(tv_tls_key, t)) {
5d879392	344	log_err("fio: can't set TLS key\n");
9eb271b9 JA	345	assert(0);
9eb271b9 JA	346	}
5d879392 JA	347	}
	348
	349	static void kill_tv_tls_key(void *data)
	350	{
	351	free(data);
	352	}
67bf9823 JA	353	#else
	354	void fio_local_clock_init(int is_thread)
	355	{
	356	}
	357	#endif
5d879392	358
09a32402 JA	359	void fio_clock_init(void)
09a32402 JA	360	{
01423eae JA	361	if (fio_clock_source == fio_clock_source_inited)
	362	return;
	363
67bf9823	364	#ifndef CONFIG_TLS_THREAD
5d879392 JA	365	if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
5d879392 JA	366	log_err("fio: can't create TLS key\n");
67bf9823	367	#endif
5d879392	368
01423eae	369	fio_clock_source_inited = fio_clock_source;
e259879e JA	370
	371	if (calibrate_cpu_clock())
	372	tsc_reliable = 0;
fa80feae JA	373
	374	/*
	375	* If the arch sets tsc_reliable != 0, then it must be good enough
	376	* to use as THE clock source. For x86 CPUs, this means the TSC
	377	* runs at a constant rate and is synced across CPU cores.
	378	*/
	379	if (tsc_reliable) {
aad918e4	380	if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
fa80feae JA	381	fio_clock_source = CS_CPUCLOCK;
	382	} else if (fio_clock_source == CS_CPUCLOCK)
	383	log_info("fio: clocksource=cpu may not be reliable\n");
c223da83 JA	384	}
c223da83 JA	385
1f440ece	386	uint64_t utime_since(const struct timeval s, const struct timeval e)
be4ecfdf	387	{
39ab7da2	388	long sec, usec;
aa60bc58	389	uint64_t ret;
39ab7da2 JA	390
	391	sec = e->tv_sec - s->tv_sec;
	392	usec = e->tv_usec - s->tv_usec;
	393	if (sec > 0 && usec < 0) {
	394	sec--;
	395	usec += 1000000;
	396	}
	397
	398	/*
	399	* time warp bug on some kernels?
	400	*/
	401	if (sec < 0 \|\| (sec == 0 && usec < 0))
	402	return 0;
	403
	404	ret = sec * 1000000ULL + usec;
	405
	406	return ret;
be4ecfdf JA	407	}
be4ecfdf JA	408
1f440ece	409	uint64_t utime_since_now(const struct timeval *s)
be4ecfdf	410	{
39ab7da2 JA	411	struct timeval t;
	412
	413	fio_gettime(&t, NULL);
	414	return utime_since(s, &t);
be4ecfdf	415	}
783a3eb1	416
1f440ece	417	uint64_t mtime_since(const struct timeval s, const struct timeval e)
783a3eb1	418	{
39ab7da2	419	long sec, usec, ret;
783a3eb1	420
39ab7da2 JA	421	sec = e->tv_sec - s->tv_sec;
	422	usec = e->tv_usec - s->tv_usec;
	423	if (sec > 0 && usec < 0) {
	424	sec--;
	425	usec += 1000000;
783a3eb1 JA	426	}
783a3eb1 JA	427
39ab7da2 JA	428	if (sec < 0 \|\| (sec == 0 && usec < 0))
	429	return 0;
	430
	431	sec *= 1000UL;
	432	usec /= 1000UL;
	433	ret = sec + usec;
	434
	435	return ret;
783a3eb1 JA	436	}
783a3eb1 JA	437
1f440ece	438	uint64_t mtime_since_now(const struct timeval *s)
783a3eb1	439	{
39ab7da2 JA	440	struct timeval t;
39ab7da2 JA	441	void *p = __builtin_return_address(0);
783a3eb1	442
39ab7da2 JA	443	fio_gettime(&t, p);
	444	return mtime_since(s, &t);
	445	}
783a3eb1	446
1f440ece	447	uint64_t time_since_now(const struct timeval *s)
39ab7da2 JA	448	{
39ab7da2 JA	449	return mtime_since_now(s) / 1000;
783a3eb1	450	}
7d11f871	451
67bf9823 JA	452	#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
67bf9823 JA	453	defined(CONFIG_SFAA)
7d11f871	454
aad918e4 JA	455	#define CLOCK_ENTRIES_DEBUG 100000
aad918e4 JA	456	#define CLOCK_ENTRIES_TEST 10000
7d11f871 JA	457
7d11f871 JA	458	struct clock_entry {
58002f9a JA	459	uint32_t seq;
58002f9a JA	460	uint32_t cpu;
ba458c2f	461	uint64_t tsc;
7d11f871 JA	462	};
	463
	464	struct clock_thread {
	465	pthread_t thread;
	466	int cpu;
aad918e4	467	int debug;
7d11f871 JA	468	pthread_mutex_t lock;
7d11f871 JA	469	pthread_mutex_t started;
aad918e4	470	unsigned long nr_entries;
58002f9a	471	uint32_t *seq;
7d11f871 JA	472	struct clock_entry *entries;
	473	};
	474
58002f9a	475	static inline uint32_t atomic32_inc_return(uint32_t *seq)
7d11f871 JA	476	{
	477	return 1 + __sync_fetch_and_add(seq, 1);
	478	}
	479
	480	static void clock_thread_fn(void data)
	481	{
	482	struct clock_thread *t = data;
	483	struct clock_entry *c;
	484	os_cpu_mask_t cpu_mask;
58002f9a	485	uint32_t last_seq;
7d11f871 JA	486	int i;
7d11f871 JA	487
c763aea6	488	if (fio_cpuset_init(&cpu_mask)) {
37e20021	489	int __err = errno;
c763aea6 JA	490
	491	log_err("clock cpuset init failed: %s\n", strerror(__err));
	492	goto err_out;
	493	}
	494
7d11f871 JA	495	fio_cpu_set(&cpu_mask, t->cpu);
	496
	497	if (fio_setaffinity(gettid(), cpu_mask) == -1) {
e66d7f90 JA	498	int __err = errno;
	499
	500	log_err("clock setaffinity failed: %s\n", strerror(__err));
c763aea6	501	goto err;
7d11f871 JA	502	}
7d11f871 JA	503
7d11f871	504	pthread_mutex_lock(&t->lock);
b9b3498e	505	pthread_mutex_unlock(&t->started);
7d11f871	506
58002f9a	507	last_seq = 0;
7d11f871	508	c = &t->entries[0];
aad918e4	509	for (i = 0; i < t->nr_entries; i++, c++) {
58002f9a JA	510	uint32_t seq;
58002f9a JA	511	uint64_t tsc;
7d11f871 JA	512
	513	c->cpu = t->cpu;
	514	do {
58002f9a JA	515	seq = atomic32_inc_return(t->seq);
	516	if (seq < last_seq)
	517	break;
7d11f871 JA	518	tsc = get_cpu_clock();
	519	} while (seq != *t->seq);
	520
	521	c->seq = seq;
	522	c->tsc = tsc;
	523	}
	524
aad918e4 JA	525	if (t->debug) {
	526	unsigned long long clocks;
	527
	528	clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
	529	log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu, clocks);
	530	}
58002f9a	531
e259879e JA	532	/*
	533	* The most common platform clock breakage is returning zero
	534	* indefinitely. Check for that and return failure.
	535	*/
58002f9a	536	if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
c763aea6	537	goto err;
e259879e	538
d47d7cb3	539	fio_cpuset_exit(&cpu_mask);
7d11f871	540	return NULL;
c763aea6 JA	541	err:
	542	fio_cpuset_exit(&cpu_mask);
	543	err_out:
	544	return (void *) 1;
7d11f871 JA	545	}
	546
	547	static int clock_cmp(const void p1, const void p2)
	548	{
	549	const struct clock_entry *c1 = p1;
	550	const struct clock_entry *c2 = p2;
	551
b9b3498e JA	552	if (c1->seq == c2->seq)
	553	log_err("cs: bug in atomic sequence!\n");
	554
7d11f871 JA	555	return c1->seq - c2->seq;
	556	}
	557
aad918e4	558	int fio_monotonic_clocktest(int debug)
7d11f871	559	{
8a1db9a1	560	struct clock_thread *cthreads;
7d11f871 JA	561	unsigned int nr_cpus = cpus_online();
7d11f871 JA	562	struct clock_entry *entries;
aad918e4	563	unsigned long nr_entries, tentries, failed = 0;
80da8a8f	564	struct clock_entry prev, this;
58002f9a	565	uint32_t seq = 0;
caa3eb1c	566	unsigned int i;
7d11f871	567
aad918e4 JA	568	if (debug) {
aad918e4 JA	569	log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
d5e3f5d8	570
b5b571a3	571	#ifdef FIO_INC_DEBUG
aad918e4	572	fio_debug \|= 1U << FD_TIME;
b5b571a3	573	#endif
aad918e4 JA	574	nr_entries = CLOCK_ENTRIES_DEBUG;
	575	} else
	576	nr_entries = CLOCK_ENTRIES_TEST;
	577
4f1d43c2	578	calibrate_cpu_clock();
aad918e4 JA	579
aad918e4 JA	580	if (debug) {
b5b571a3	581	#ifdef FIO_INC_DEBUG
aad918e4	582	fio_debug &= ~(1U << FD_TIME);
b5b571a3	583	#endif
aad918e4	584	}
4f1d43c2	585
8a1db9a1	586	cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
aad918e4	587	tentries = nr_entries * nr_cpus;
7d11f871 JA	588	entries = malloc(tentries * sizeof(struct clock_entry));
7d11f871 JA	589
aad918e4 JA	590	if (debug)
aad918e4 JA	591	log_info("cs: Testing %u CPUs\n", nr_cpus);
7d11f871 JA	592
7d11f871 JA	593	for (i = 0; i < nr_cpus; i++) {
8a1db9a1	594	struct clock_thread *t = &cthreads[i];
7d11f871 JA	595
7d11f871 JA	596	t->cpu = i;
aad918e4	597	t->debug = debug;
7d11f871	598	t->seq = &seq;
aad918e4 JA	599	t->nr_entries = nr_entries;
aad918e4 JA	600	t->entries = &entries[i * nr_entries];
7d11f871 JA	601	pthread_mutex_init(&t->lock, NULL);
	602	pthread_mutex_init(&t->started, NULL);
	603	pthread_mutex_lock(&t->lock);
6b0110cd JA	604	if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
	605	failed++;
	606	nr_cpus = i;
	607	break;
	608	}
7d11f871 JA	609	}
	610
	611	for (i = 0; i < nr_cpus; i++) {
8a1db9a1	612	struct clock_thread *t = &cthreads[i];
7d11f871 JA	613
	614	pthread_mutex_lock(&t->started);
	615	}
	616
	617	for (i = 0; i < nr_cpus; i++) {
8a1db9a1	618	struct clock_thread *t = &cthreads[i];
7d11f871 JA	619
	620	pthread_mutex_unlock(&t->lock);
	621	}
	622
814917be	623	for (i = 0; i < nr_cpus; i++) {
8a1db9a1	624	struct clock_thread *t = &cthreads[i];
7d11f871 JA	625	void *ret;
	626
	627	pthread_join(t->thread, &ret);
	628	if (ret)
	629	failed++;
	630	}
8a1db9a1	631	free(cthreads);
7d11f871 JA	632
7d11f871 JA	633	if (failed) {
aad918e4 JA	634	if (debug)
aad918e4 JA	635	log_err("Clocksource test: %lu threads failed\n", failed);
7d11f871 JA	636	goto err;
	637	}
	638
	639	qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
	640
	641	for (failed = i = 0; i < tentries; i++) {
80da8a8f	642	this = &entries[i];
7d11f871 JA	643
	644	if (!i) {
	645	prev = this;
	646	continue;
	647	}
	648
	649	if (prev->tsc > this->tsc) {
	650	uint64_t diff = prev->tsc - this->tsc;
	651
aad918e4 JA	652	if (!debug) {
	653	failed++;
	654	break;
	655	}
	656
4e0a8fa2 JA	657	log_info("cs: CPU clock mismatch (diff=%llu):\n",
	658	(unsigned long long) diff);
	659	log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
	660	log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
7d11f871 JA	661	failed++;
	662	}
	663
	664	prev = this;
	665	}
	666
aad918e4 JA	667	if (debug) {
	668	if (failed)
	669	log_info("cs: Failed: %lu\n", failed);
	670	else
	671	log_info("cs: Pass!\n");
	672	}
7d11f871 JA	673	err:
	674	free(entries);
	675	return !!failed;
	676	}
	677
	678	#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
	679
aad918e4	680	int fio_monotonic_clocktest(int debug)
7d11f871	681	{
aad918e4 JA	682	if (debug)
	683	log_info("cs: current platform does not support CPU clocks\n");
	684	return 1;
7d11f871 JA	685	}
	686
	687	#endif