[fio.git] / gettime.c

/*
 * Clock functions
 */

#include <unistd.h>
#include <math.h>
#include <sys/time.h>
#include <time.h>

#include "fio.h"
#include "smalloc.h"

#include "hash.h"
#include "os/os.h"

#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
static unsigned long cycles_per_usec;
static unsigned long inv_cycles_per_usec;
static uint64_t max_cycles_for_mult;
static unsigned long long cycles_start, cycles_wrap;
#endif
int tsc_reliable = 0;

struct tv_valid {
	uint64_t last_cycles;
	int last_tv_valid;
	int warned;
};
#ifdef CONFIG_TLS_THREAD
static __thread struct tv_valid static_tv_valid;
#else
static pthread_key_t tv_tls_key;
#endif

enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
int fio_clock_source_set = 0;
static enum fio_cs fio_clock_source_inited = CS_INVAL;

#ifdef FIO_DEBUG_TIME

#define HASH_BITS	8
#define HASH_SIZE	(1 << HASH_BITS)

static struct flist_head hash[HASH_SIZE];
static int gtod_inited;

struct gtod_log {
	struct flist_head list;
	void *caller;
	unsigned long calls;
};

static struct gtod_log *find_hash(void *caller)
{
	unsigned long h = hash_ptr(caller, HASH_BITS);
	struct flist_head *entry;

	flist_for_each(entry, &hash[h]) {
		struct gtod_log *log = flist_entry(entry, struct gtod_log,
									list);

		if (log->caller == caller)
			return log;
	}

	return NULL;
}

static void inc_caller(void *caller)
{
	struct gtod_log *log = find_hash(caller);

	if (!log) {
		unsigned long h;

		log = malloc(sizeof(*log));
		INIT_FLIST_HEAD(&log->list);
		log->caller = caller;
		log->calls = 0;

		h = hash_ptr(caller, HASH_BITS);
		flist_add_tail(&log->list, &hash[h]);
	}

	log->calls++;
}

static void gtod_log_caller(void *caller)
{
	if (gtod_inited)
		inc_caller(caller);
}

static void fio_exit fio_dump_gtod(void)
{
	unsigned long total_calls = 0;
	int i;

	for (i = 0; i < HASH_SIZE; i++) {
		struct flist_head *entry;
		struct gtod_log *log;

		flist_for_each(entry, &hash[i]) {
			log = flist_entry(entry, struct gtod_log, list);

			printf("function %p, calls %lu\n", log->caller,
								log->calls);
			total_calls += log->calls;
		}
	}

	printf("Total %lu gettimeofday\n", total_calls);
}

static void fio_init gtod_init(void)
{
	int i;

	for (i = 0; i < HASH_SIZE; i++)
		INIT_FLIST_HEAD(&hash[i]);

	gtod_inited = 1;
}

#endif /* FIO_DEBUG_TIME */

#ifdef CONFIG_CLOCK_GETTIME
static int fill_clock_gettime(struct timespec *ts)
{
#ifdef CONFIG_CLOCK_MONOTONIC
	return clock_gettime(CLOCK_MONOTONIC, ts);
#else
	return clock_gettime(CLOCK_REALTIME, ts);
#endif
}
#endif

static void __fio_gettime(struct timeval *tp)
{
	struct tv_valid *tv;

#ifdef CONFIG_TLS_THREAD
	tv = &static_tv_valid;
#else
	tv = pthread_getspecific(tv_tls_key);
#endif

	switch (fio_clock_source) {
#ifdef CONFIG_GETTIMEOFDAY
	case CS_GTOD:
		gettimeofday(tp, NULL);
		break;
#endif
#ifdef CONFIG_CLOCK_GETTIME
	case CS_CGETTIME: {
		struct timespec ts;

		if (fill_clock_gettime(&ts) < 0) {
			log_err("fio: clock_gettime fails\n");
			assert(0);
		}

		tp->tv_sec = ts.tv_sec;
		tp->tv_usec = ts.tv_nsec / 1000;
		break;
		}
#endif
#ifdef ARCH_HAVE_CPU_CLOCK
	case CS_CPUCLOCK: {
		uint64_t usecs, t;

		t = get_cpu_clock();
		if (t < cycles_start && !cycles_wrap)
			cycles_wrap = 1;
		else if (cycles_wrap && t >= cycles_start) {
			if (!tv->warned) {
				log_err("fio: double CPU clock wrap\n");
				tv->warned = 1;
			}
		}

		t -= cycles_start;
		tv->last_cycles = t;
		tv->last_tv_valid = 1;
#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
		usecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC;
#else
		if (t < max_cycles_for_mult)
			usecs = (t * inv_cycles_per_usec) / 16777216UL;
		else
			usecs = t / cycles_per_usec;
#endif
		tp->tv_sec = usecs / 1000000;
		tp->tv_usec = usecs % 1000000;
		break;
		}
#endif
	default:
		log_err("fio: invalid clock source %d\n", fio_clock_source);
		break;
	}
}

#ifdef FIO_DEBUG_TIME
void fio_gettime(struct timeval *tp, void *caller)
#else
void fio_gettime(struct timeval *tp, void fio_unused *caller)
#endif
{
#ifdef FIO_DEBUG_TIME
	if (!caller)
		caller = __builtin_return_address(0);

	gtod_log_caller(caller);
#endif
	if (fio_unlikely(fio_gettime_offload(tp)))
		return;

	__fio_gettime(tp);
}

#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
static unsigned long get_cycles_per_usec(void)
{
	struct timeval s, e;
	uint64_t c_s, c_e;
	enum fio_cs old_cs = fio_clock_source;

#ifdef CONFIG_CLOCK_GETTIME
	fio_clock_source = CS_CGETTIME;
#else
	fio_clock_source = CS_GTOD;
#endif
	__fio_gettime(&s);

	c_s = get_cpu_clock();
	do {
		uint64_t elapsed;

		__fio_gettime(&e);

		elapsed = utime_since(&s, &e);
		if (elapsed >= 1280) {
			c_e = get_cpu_clock();
			break;
		}
	} while (1);

	fio_clock_source = old_cs;
	return (c_e - c_s + 127) >> 7;
}

#define NR_TIME_ITERS	50

static int calibrate_cpu_clock(void)
{
	double delta, mean, S;
	uint64_t avg, cycles[NR_TIME_ITERS];
	int i, samples;

	cycles[0] = get_cycles_per_usec();
	S = delta = mean = 0.0;
	for (i = 0; i < NR_TIME_ITERS; i++) {
		cycles[i] = get_cycles_per_usec();
		delta = cycles[i] - mean;
		if (delta) {
			mean += delta / (i + 1.0);
			S += delta * (cycles[i] - mean);
		}
	}

	/*
	 * The most common platform clock breakage is returning zero
	 * indefinitely. Check for that and return failure.
	 */
	if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
		return 1;

	S = sqrt(S / (NR_TIME_ITERS - 1.0));

	samples = avg = 0;
	for (i = 0; i < NR_TIME_ITERS; i++) {
		double this = cycles[i];

		if ((fmax(this, mean) - fmin(this, mean)) > S)
			continue;
		samples++;
		avg += this;
	}

	S /= (double) NR_TIME_ITERS;
	mean /= 10.0;

	for (i = 0; i < NR_TIME_ITERS; i++)
		dprint(FD_TIME, "cycles[%d]=%llu\n", i,
					(unsigned long long) cycles[i] / 10);

	avg /= samples;
	avg = (avg + 5) / 10;
	dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
	dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);

	cycles_per_usec = avg;
	inv_cycles_per_usec = 16777216UL / cycles_per_usec;
	max_cycles_for_mult = ~0ULL / inv_cycles_per_usec;
	dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
	cycles_start = get_cpu_clock();
	dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
	return 0;
}
#else
static int calibrate_cpu_clock(void)
{
#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
	return 0;
#else
	return 1;
#endif
}
#endif // ARCH_HAVE_CPU_CLOCK

#ifndef CONFIG_TLS_THREAD
void fio_local_clock_init(int is_thread)
{
	struct tv_valid *t;

	t = calloc(1, sizeof(*t));
	if (pthread_setspecific(tv_tls_key, t)) {
		log_err("fio: can't set TLS key\n");
		assert(0);
	}
}

static void kill_tv_tls_key(void *data)
{
	free(data);
}
#else
void fio_local_clock_init(int is_thread)
{
}
#endif

void fio_clock_init(void)
{
	if (fio_clock_source == fio_clock_source_inited)
		return;

#ifndef CONFIG_TLS_THREAD
	if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
		log_err("fio: can't create TLS key\n");
#endif

	fio_clock_source_inited = fio_clock_source;

	if (calibrate_cpu_clock())
		tsc_reliable = 0;

	/*
	 * If the arch sets tsc_reliable != 0, then it must be good enough
	 * to use as THE clock source. For x86 CPUs, this means the TSC
	 * runs at a constant rate and is synced across CPU cores.
	 */
	if (tsc_reliable) {
		if (!fio_clock_source_set)
			fio_clock_source = CS_CPUCLOCK;
	} else if (fio_clock_source == CS_CPUCLOCK)
		log_info("fio: clocksource=cpu may not be reliable\n");
}

uint64_t utime_since(const struct timeval *s, const struct timeval *e)
{
	long sec, usec;
	uint64_t ret;

	sec = e->tv_sec - s->tv_sec;
	usec = e->tv_usec - s->tv_usec;
	if (sec > 0 && usec < 0) {
		sec--;
		usec += 1000000;
	}

	/*
	 * time warp bug on some kernels?
	 */
	if (sec < 0 || (sec == 0 && usec < 0))
		return 0;

	ret = sec * 1000000ULL + usec;

	return ret;
}

uint64_t utime_since_now(const struct timeval *s)
{
	struct timeval t;

	fio_gettime(&t, NULL);
	return utime_since(s, &t);
}

uint64_t mtime_since(const struct timeval *s, const struct timeval *e)
{
	long sec, usec, ret;

	sec = e->tv_sec - s->tv_sec;
	usec = e->tv_usec - s->tv_usec;
	if (sec > 0 && usec < 0) {
		sec--;
		usec += 1000000;
	}

	if (sec < 0 || (sec == 0 && usec < 0))
		return 0;

	sec *= 1000UL;
	usec /= 1000UL;
	ret = sec + usec;

	return ret;
}

uint64_t mtime_since_now(const struct timeval *s)
{
	struct timeval t;
	void *p = __builtin_return_address(0);

	fio_gettime(&t, p);
	return mtime_since(s, &t);
}

uint64_t time_since_now(const struct timeval *s)
{
	return mtime_since_now(s) / 1000;
}

#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \
    defined(CONFIG_SFAA)

#define CLOCK_ENTRIES	100000

struct clock_entry {
	uint32_t seq;
	uint32_t cpu;
	uint64_t tsc;
};

struct clock_thread {
	pthread_t thread;
	int cpu;
	pthread_mutex_t lock;
	pthread_mutex_t started;
	uint32_t *seq;
	struct clock_entry *entries;
};

static inline uint32_t atomic32_inc_return(uint32_t *seq)
{
	return 1 + __sync_fetch_and_add(seq, 1);
}

static void *clock_thread_fn(void *data)
{
	struct clock_thread *t = data;
	struct clock_entry *c;
	os_cpu_mask_t cpu_mask;
	uint32_t last_seq;
	int i;

	memset(&cpu_mask, 0, sizeof(cpu_mask));
	fio_cpu_set(&cpu_mask, t->cpu);

	if (fio_setaffinity(gettid(), cpu_mask) == -1) {
		log_err("clock setaffinity failed\n");
		return (void *) 1;
	}

	pthread_mutex_lock(&t->lock);
	pthread_mutex_unlock(&t->started);

	last_seq = 0;
	c = &t->entries[0];
	for (i = 0; i < CLOCK_ENTRIES; i++, c++) {
		uint32_t seq;
		uint64_t tsc;

		c->cpu = t->cpu;
		do {
			seq = atomic32_inc_return(t->seq);
			if (seq < last_seq)
				break;
			tsc = get_cpu_clock();
		} while (seq != *t->seq);

		c->seq = seq;
		c->tsc = tsc;
	}

	log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu,
		(unsigned long long) t->entries[i - 1].tsc - t->entries[0].tsc);

	/*
	 * The most common platform clock breakage is returning zero
	 * indefinitely. Check for that and return failure.
	 */
	if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
		return (void *) 1;

	return NULL;
}

static int clock_cmp(const void *p1, const void *p2)
{
	const struct clock_entry *c1 = p1;
	const struct clock_entry *c2 = p2;

	if (c1->seq == c2->seq)
		log_err("cs: bug in atomic sequence!\n");

	return c1->seq - c2->seq;
}

int fio_monotonic_clocktest(void)
{
	struct clock_thread *cthreads;
	unsigned int nr_cpus = cpus_online();
	struct clock_entry *entries;
	unsigned long tentries, failed = 0;
	struct clock_entry *prev, *this;
	uint32_t seq = 0;
	unsigned int i;

	log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");

#ifdef FIO_INC_DEBUG
	fio_debug |= 1U << FD_TIME;
#endif
	calibrate_cpu_clock();
#ifdef FIO_INC_DEBUG
	fio_debug &= ~(1U << FD_TIME);
#endif

	cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
	tentries = CLOCK_ENTRIES * nr_cpus;
	entries = malloc(tentries * sizeof(struct clock_entry));

	log_info("cs: Testing %u CPUs\n", nr_cpus);

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];

		t->cpu = i;
		t->seq = &seq;
		t->entries = &entries[i * CLOCK_ENTRIES];
		pthread_mutex_init(&t->lock, NULL);
		pthread_mutex_init(&t->started, NULL);
		pthread_mutex_lock(&t->lock);
		if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
			failed++;
			nr_cpus = i;
			break;
		}
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];

		pthread_mutex_lock(&t->started);
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];

		pthread_mutex_unlock(&t->lock);
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &cthreads[i];
		void *ret;

		pthread_join(t->thread, &ret);
		if (ret)
			failed++;
	}
	free(cthreads);

	if (failed) {
		log_err("Clocksource test: %lu threads failed\n", failed);
		goto err;
	}

	qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);

	for (failed = i = 0; i < tentries; i++) {
		this = &entries[i];

		if (!i) {
			prev = this;
			continue;
		}

		if (prev->tsc > this->tsc) {
			uint64_t diff = prev->tsc - this->tsc;

			log_info("cs: CPU clock mismatch (diff=%llu):\n",
						(unsigned long long) diff);
			log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
			log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
			failed++;
		}

		prev = this;
	}

	if (failed)
		log_info("cs: Failed: %lu\n", failed);
	else
		log_info("cs: Pass!\n");

err:
	free(entries);
	return !!failed;
}

#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */

int fio_monotonic_clocktest(void)
{
	log_info("cs: current platform does not support CPU clocks\n");
	return 0;
}

#endif
Commit	Line	Data
	1	/*
	2	* Clock functions
	3	*/
	4
	5	#include <unistd.h>
	6	#include <math.h>
	7	#include <sys/time.h>
	8	#include <time.h>
	9
	10	#include "fio.h"
	11	#include "smalloc.h"
	12
	13	#include "hash.h"
	14	#include "os/os.h"
	15
	16	#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
	17	static unsigned long cycles_per_usec;
	18	static unsigned long inv_cycles_per_usec;
	19	static uint64_t max_cycles_for_mult;
	20	static unsigned long long cycles_start, cycles_wrap;
	21	#endif
	22	int tsc_reliable = 0;
	23
	24	struct tv_valid {
	25	uint64_t last_cycles;
	26	int last_tv_valid;
	27	int warned;
	28	};
	29	#ifdef CONFIG_TLS_THREAD
	30	static __thread struct tv_valid static_tv_valid;
	31	#else
	32	static pthread_key_t tv_tls_key;
	33	#endif
	34
	35	enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
	36	int fio_clock_source_set = 0;
	37	static enum fio_cs fio_clock_source_inited = CS_INVAL;
	38
	39	#ifdef FIO_DEBUG_TIME
	40
	41	#define HASH_BITS 8
	42	#define HASH_SIZE (1 << HASH_BITS)
	43
	44	static struct flist_head hash[HASH_SIZE];
	45	static int gtod_inited;
	46
	47	struct gtod_log {
	48	struct flist_head list;
	49	void *caller;
	50	unsigned long calls;
	51	};
	52
	53	static struct gtod_log find_hash(void caller)
	54	{
	55	unsigned long h = hash_ptr(caller, HASH_BITS);
	56	struct flist_head *entry;
	57
	58	flist_for_each(entry, &hash[h]) {
	59	struct gtod_log *log = flist_entry(entry, struct gtod_log,
	60	list);
	61
	62	if (log->caller == caller)
	63	return log;
	64	}
	65
	66	return NULL;
	67	}
	68
	69	static void inc_caller(void *caller)
	70	{
	71	struct gtod_log *log = find_hash(caller);
	72
	73	if (!log) {
	74	unsigned long h;
	75
	76	log = malloc(sizeof(*log));
	77	INIT_FLIST_HEAD(&log->list);
	78	log->caller = caller;
	79	log->calls = 0;
	80
	81	h = hash_ptr(caller, HASH_BITS);
	82	flist_add_tail(&log->list, &hash[h]);
	83	}
	84
	85	log->calls++;
	86	}
	87
	88	static void gtod_log_caller(void *caller)
	89	{
	90	if (gtod_inited)
	91	inc_caller(caller);
	92	}
	93
	94	static void fio_exit fio_dump_gtod(void)
	95	{
	96	unsigned long total_calls = 0;
	97	int i;
	98
	99	for (i = 0; i < HASH_SIZE; i++) {
	100	struct flist_head *entry;
	101	struct gtod_log *log;
	102
	103	flist_for_each(entry, &hash[i]) {
	104	log = flist_entry(entry, struct gtod_log, list);
	105
	106	printf("function %p, calls %lu\n", log->caller,
	107	log->calls);
	108	total_calls += log->calls;
	109	}
	110	}
	111
	112	printf("Total %lu gettimeofday\n", total_calls);
	113	}
	114
	115	static void fio_init gtod_init(void)
	116	{
	117	int i;
	118
	119	for (i = 0; i < HASH_SIZE; i++)
	120	INIT_FLIST_HEAD(&hash[i]);
	121
	122	gtod_inited = 1;
	123	}
	124
	125	#endif /* FIO_DEBUG_TIME */
	126
	127	#ifdef CONFIG_CLOCK_GETTIME
	128	static int fill_clock_gettime(struct timespec *ts)
	129	{
	130	#ifdef CONFIG_CLOCK_MONOTONIC
	131	return clock_gettime(CLOCK_MONOTONIC, ts);
	132	#else
	133	return clock_gettime(CLOCK_REALTIME, ts);
	134	#endif
	135	}
	136	#endif
	137
	138	static void __fio_gettime(struct timeval *tp)
	139	{
	140	struct tv_valid *tv;
	141
	142	#ifdef CONFIG_TLS_THREAD
	143	tv = &static_tv_valid;
	144	#else
	145	tv = pthread_getspecific(tv_tls_key);
	146	#endif
	147
	148	switch (fio_clock_source) {
	149	#ifdef CONFIG_GETTIMEOFDAY
	150	case CS_GTOD:
	151	gettimeofday(tp, NULL);
	152	break;
	153	#endif
	154	#ifdef CONFIG_CLOCK_GETTIME
	155	case CS_CGETTIME: {
	156	struct timespec ts;
	157
	158	if (fill_clock_gettime(&ts) < 0) {
	159	log_err("fio: clock_gettime fails\n");
	160	assert(0);
	161	}
	162
	163	tp->tv_sec = ts.tv_sec;
	164	tp->tv_usec = ts.tv_nsec / 1000;
	165	break;
	166	}
	167	#endif
	168	#ifdef ARCH_HAVE_CPU_CLOCK
	169	case CS_CPUCLOCK: {
	170	uint64_t usecs, t;
	171
	172	t = get_cpu_clock();
	173	if (t < cycles_start && !cycles_wrap)
	174	cycles_wrap = 1;
	175	else if (cycles_wrap && t >= cycles_start) {
	176	if (!tv->warned) {
	177	log_err("fio: double CPU clock wrap\n");
	178	tv->warned = 1;
	179	}
	180	}
	181
	182	t -= cycles_start;
	183	tv->last_cycles = t;
	184	tv->last_tv_valid = 1;
	185	#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
	186	usecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC;
	187	#else
	188	if (t < max_cycles_for_mult)
	189	usecs = (t * inv_cycles_per_usec) / 16777216UL;
	190	else
	191	usecs = t / cycles_per_usec;
	192	#endif
	193	tp->tv_sec = usecs / 1000000;
	194	tp->tv_usec = usecs % 1000000;
	195	break;
	196	}
	197	#endif
	198	default:
	199	log_err("fio: invalid clock source %d\n", fio_clock_source);
	200	break;
	201	}
	202	}
	203
	204	#ifdef FIO_DEBUG_TIME
	205	void fio_gettime(struct timeval tp, void caller)
	206	#else
	207	void fio_gettime(struct timeval tp, void fio_unused caller)
	208	#endif
	209	{
	210	#ifdef FIO_DEBUG_TIME
	211	if (!caller)
	212	caller = __builtin_return_address(0);
	213
	214	gtod_log_caller(caller);
	215	#endif
	216	if (fio_unlikely(fio_gettime_offload(tp)))
	217	return;
	218
	219	__fio_gettime(tp);
	220	}
	221
	222	#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
	223	static unsigned long get_cycles_per_usec(void)
	224	{
	225	struct timeval s, e;
	226	uint64_t c_s, c_e;
	227	enum fio_cs old_cs = fio_clock_source;
	228
	229	#ifdef CONFIG_CLOCK_GETTIME
	230	fio_clock_source = CS_CGETTIME;
	231	#else
	232	fio_clock_source = CS_GTOD;
	233	#endif
	234	__fio_gettime(&s);
	235
	236	c_s = get_cpu_clock();
	237	do {
	238	uint64_t elapsed;
	239
	240	__fio_gettime(&e);
	241
	242	elapsed = utime_since(&s, &e);
	243	if (elapsed >= 1280) {
	244	c_e = get_cpu_clock();
	245	break;
	246	}
	247	} while (1);
	248
	249	fio_clock_source = old_cs;
	250	return (c_e - c_s + 127) >> 7;
	251	}
	252
	253	#define NR_TIME_ITERS 50
	254
	255	static int calibrate_cpu_clock(void)
	256	{
	257	double delta, mean, S;
	258	uint64_t avg, cycles[NR_TIME_ITERS];
	259	int i, samples;
	260
	261	cycles[0] = get_cycles_per_usec();
	262	S = delta = mean = 0.0;
	263	for (i = 0; i < NR_TIME_ITERS; i++) {
	264	cycles[i] = get_cycles_per_usec();
	265	delta = cycles[i] - mean;
	266	if (delta) {
	267	mean += delta / (i + 1.0);
	268	S += delta * (cycles[i] - mean);
	269	}
	270	}
	271
	272	/*
	273	* The most common platform clock breakage is returning zero
	274	* indefinitely. Check for that and return failure.
	275	*/
	276	if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
	277	return 1;
	278
	279	S = sqrt(S / (NR_TIME_ITERS - 1.0));
	280
	281	samples = avg = 0;
	282	for (i = 0; i < NR_TIME_ITERS; i++) {
	283	double this = cycles[i];
	284
	285	if ((fmax(this, mean) - fmin(this, mean)) > S)
	286	continue;
	287	samples++;
	288	avg += this;
	289	}
	290
	291	S /= (double) NR_TIME_ITERS;
	292	mean /= 10.0;
	293
	294	for (i = 0; i < NR_TIME_ITERS; i++)
	295	dprint(FD_TIME, "cycles[%d]=%llu\n", i,
	296	(unsigned long long) cycles[i] / 10);
	297
	298	avg /= samples;
	299	avg = (avg + 5) / 10;
	300	dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
	301	dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);
	302
	303	cycles_per_usec = avg;
	304	inv_cycles_per_usec = 16777216UL / cycles_per_usec;
	305	max_cycles_for_mult = ~0ULL / inv_cycles_per_usec;
	306	dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
	307	cycles_start = get_cpu_clock();
	308	dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
	309	return 0;
	310	}
	311	#else
	312	static int calibrate_cpu_clock(void)
	313	{
	314	#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
	315	return 0;
	316	#else
	317	return 1;
	318	#endif
	319	}
	320	#endif // ARCH_HAVE_CPU_CLOCK
	321
	322	#ifndef CONFIG_TLS_THREAD
	323	void fio_local_clock_init(int is_thread)
	324	{
	325	struct tv_valid *t;
	326
	327	t = calloc(1, sizeof(*t));
	328	if (pthread_setspecific(tv_tls_key, t)) {
	329	log_err("fio: can't set TLS key\n");
	330	assert(0);
	331	}
	332	}
	333
	334	static void kill_tv_tls_key(void *data)
	335	{
	336	free(data);
	337	}
	338	#else
	339	void fio_local_clock_init(int is_thread)
	340	{
	341	}
	342	#endif
	343
	344	void fio_clock_init(void)
	345	{
	346	if (fio_clock_source == fio_clock_source_inited)
	347	return;
	348
	349	#ifndef CONFIG_TLS_THREAD
	350	if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
	351	log_err("fio: can't create TLS key\n");
	352	#endif
	353
	354	fio_clock_source_inited = fio_clock_source;
	355
	356	if (calibrate_cpu_clock())
	357	tsc_reliable = 0;
	358
	359	/*
	360	* If the arch sets tsc_reliable != 0, then it must be good enough
	361	* to use as THE clock source. For x86 CPUs, this means the TSC
	362	* runs at a constant rate and is synced across CPU cores.
	363	*/
	364	if (tsc_reliable) {
	365	if (!fio_clock_source_set)
	366	fio_clock_source = CS_CPUCLOCK;
	367	} else if (fio_clock_source == CS_CPUCLOCK)
	368	log_info("fio: clocksource=cpu may not be reliable\n");
	369	}
	370
	371	uint64_t utime_since(const struct timeval s, const struct timeval e)
	372	{
	373	long sec, usec;
	374	uint64_t ret;
	375
	376	sec = e->tv_sec - s->tv_sec;
	377	usec = e->tv_usec - s->tv_usec;
	378	if (sec > 0 && usec < 0) {
	379	sec--;
	380	usec += 1000000;
	381	}
	382
	383	/*
	384	* time warp bug on some kernels?
	385	*/
	386	if (sec < 0 \|\| (sec == 0 && usec < 0))
	387	return 0;
	388
	389	ret = sec * 1000000ULL + usec;
	390
	391	return ret;
	392	}
	393
	394	uint64_t utime_since_now(const struct timeval *s)
	395	{
	396	struct timeval t;
	397
	398	fio_gettime(&t, NULL);
	399	return utime_since(s, &t);
	400	}
	401
	402	uint64_t mtime_since(const struct timeval s, const struct timeval e)
	403	{
	404	long sec, usec, ret;
	405
	406	sec = e->tv_sec - s->tv_sec;
	407	usec = e->tv_usec - s->tv_usec;
	408	if (sec > 0 && usec < 0) {
	409	sec--;
	410	usec += 1000000;
	411	}
	412
	413	if (sec < 0 \|\| (sec == 0 && usec < 0))
	414	return 0;
	415
	416	sec *= 1000UL;
	417	usec /= 1000UL;
	418	ret = sec + usec;
	419
	420	return ret;
	421	}
	422
	423	uint64_t mtime_since_now(const struct timeval *s)
	424	{
	425	struct timeval t;
	426	void *p = __builtin_return_address(0);
	427
	428	fio_gettime(&t, p);
	429	return mtime_since(s, &t);
	430	}
	431
	432	uint64_t time_since_now(const struct timeval *s)
	433	{
	434	return mtime_since_now(s) / 1000;
	435	}
	436
	437	#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
	438	defined(CONFIG_SFAA)
	439
	440	#define CLOCK_ENTRIES 100000
	441
	442	struct clock_entry {
	443	uint32_t seq;
	444	uint32_t cpu;
	445	uint64_t tsc;
	446	};
	447
	448	struct clock_thread {
	449	pthread_t thread;
	450	int cpu;
	451	pthread_mutex_t lock;
	452	pthread_mutex_t started;
	453	uint32_t *seq;
	454	struct clock_entry *entries;
	455	};
	456
	457	static inline uint32_t atomic32_inc_return(uint32_t *seq)
	458	{
	459	return 1 + __sync_fetch_and_add(seq, 1);
	460	}
	461
	462	static void clock_thread_fn(void data)
	463	{
	464	struct clock_thread *t = data;
	465	struct clock_entry *c;
	466	os_cpu_mask_t cpu_mask;
	467	uint32_t last_seq;
	468	int i;
	469
	470	memset(&cpu_mask, 0, sizeof(cpu_mask));
	471	fio_cpu_set(&cpu_mask, t->cpu);
	472
	473	if (fio_setaffinity(gettid(), cpu_mask) == -1) {
	474	log_err("clock setaffinity failed\n");
	475	return (void *) 1;
	476	}
	477
	478	pthread_mutex_lock(&t->lock);
	479	pthread_mutex_unlock(&t->started);
	480
	481	last_seq = 0;
	482	c = &t->entries[0];
	483	for (i = 0; i < CLOCK_ENTRIES; i++, c++) {
	484	uint32_t seq;
	485	uint64_t tsc;
	486
	487	c->cpu = t->cpu;
	488	do {
	489	seq = atomic32_inc_return(t->seq);
	490	if (seq < last_seq)
	491	break;
	492	tsc = get_cpu_clock();
	493	} while (seq != *t->seq);
	494
	495	c->seq = seq;
	496	c->tsc = tsc;
	497	}
	498
	499	log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu,
	500	(unsigned long long) t->entries[i - 1].tsc - t->entries[0].tsc);
	501
	502	/*
	503	* The most common platform clock breakage is returning zero
	504	* indefinitely. Check for that and return failure.
	505	*/
	506	if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
	507	return (void *) 1;
	508
	509	return NULL;
	510	}
	511
	512	static int clock_cmp(const void p1, const void p2)
	513	{
	514	const struct clock_entry *c1 = p1;
	515	const struct clock_entry *c2 = p2;
	516
	517	if (c1->seq == c2->seq)
	518	log_err("cs: bug in atomic sequence!\n");
	519
	520	return c1->seq - c2->seq;
	521	}
	522
	523	int fio_monotonic_clocktest(void)
	524	{
	525	struct clock_thread *cthreads;
	526	unsigned int nr_cpus = cpus_online();
	527	struct clock_entry *entries;
	528	unsigned long tentries, failed = 0;
	529	struct clock_entry prev, this;
	530	uint32_t seq = 0;
	531	unsigned int i;
	532
	533	log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
	534
	535	#ifdef FIO_INC_DEBUG
	536	fio_debug \|= 1U << FD_TIME;
	537	#endif
	538	calibrate_cpu_clock();
	539	#ifdef FIO_INC_DEBUG
	540	fio_debug &= ~(1U << FD_TIME);
	541	#endif
	542
	543	cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
	544	tentries = CLOCK_ENTRIES * nr_cpus;
	545	entries = malloc(tentries * sizeof(struct clock_entry));
	546
	547	log_info("cs: Testing %u CPUs\n", nr_cpus);
	548
	549	for (i = 0; i < nr_cpus; i++) {
	550	struct clock_thread *t = &cthreads[i];
	551
	552	t->cpu = i;
	553	t->seq = &seq;
	554	t->entries = &entries[i * CLOCK_ENTRIES];
	555	pthread_mutex_init(&t->lock, NULL);
	556	pthread_mutex_init(&t->started, NULL);
	557	pthread_mutex_lock(&t->lock);
	558	if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
	559	failed++;
	560	nr_cpus = i;
	561	break;
	562	}
	563	}
	564
	565	for (i = 0; i < nr_cpus; i++) {
	566	struct clock_thread *t = &cthreads[i];
	567
	568	pthread_mutex_lock(&t->started);
	569	}
	570
	571	for (i = 0; i < nr_cpus; i++) {
	572	struct clock_thread *t = &cthreads[i];
	573
	574	pthread_mutex_unlock(&t->lock);
	575	}
	576
	577	for (i = 0; i < nr_cpus; i++) {
	578	struct clock_thread *t = &cthreads[i];
	579	void *ret;
	580
	581	pthread_join(t->thread, &ret);
	582	if (ret)
	583	failed++;
	584	}
	585	free(cthreads);
	586
	587	if (failed) {
	588	log_err("Clocksource test: %lu threads failed\n", failed);
	589	goto err;
	590	}
	591
	592	qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
	593
	594	for (failed = i = 0; i < tentries; i++) {
	595	this = &entries[i];
	596
	597	if (!i) {
	598	prev = this;
	599	continue;
	600	}
	601
	602	if (prev->tsc > this->tsc) {
	603	uint64_t diff = prev->tsc - this->tsc;
	604
	605	log_info("cs: CPU clock mismatch (diff=%llu):\n",
	606	(unsigned long long) diff);
	607	log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
	608	log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
	609	failed++;
	610	}
	611
	612	prev = this;
	613	}
	614
	615	if (failed)
	616	log_info("cs: Failed: %lu\n", failed);
	617	else
	618	log_info("cs: Pass!\n");
	619
	620	err:
	621	free(entries);
	622	return !!failed;
	623	}
	624
	625	#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
	626
	627	int fio_monotonic_clocktest(void)
	628	{
	629	log_info("cs: current platform does not support CPU clocks\n");
	630	return 0;
	631	}
	632
	633	#endif