[fio.git] / gettime.c

/*
 * Clock functions
 */

#include <unistd.h>
#include <math.h>
#include <sys/time.h>
#include <time.h>

#include "fio.h"
#include "smalloc.h"

#include "hash.h"
#include "os/os.h"

#ifdef ARCH_HAVE_CPU_CLOCK
static unsigned long cycles_per_usec;
static unsigned long inv_cycles_per_usec;
#endif
int tsc_reliable = 0;

struct tv_valid {
	struct timeval last_tv;
	uint64_t last_cycles;
	int last_tv_valid;
};
#ifdef CONFIG_TLS_THREAD
static struct tv_valid __thread static_tv_valid;
#else
static pthread_key_t tv_tls_key;
#endif

enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
int fio_clock_source_set = 0;
enum fio_cs fio_clock_source_inited = CS_INVAL;

#ifdef FIO_DEBUG_TIME

#define HASH_BITS	8
#define HASH_SIZE	(1 << HASH_BITS)

static struct flist_head hash[HASH_SIZE];
static int gtod_inited;

struct gtod_log {
	struct flist_head list;
	void *caller;
	unsigned long calls;
};

static struct gtod_log *find_hash(void *caller)
{
	unsigned long h = hash_ptr(caller, HASH_BITS);
	struct flist_head *entry;

	flist_for_each(entry, &hash[h]) {
		struct gtod_log *log = flist_entry(entry, struct gtod_log,
									list);

		if (log->caller == caller)
			return log;
	}

	return NULL;
}

static struct gtod_log *find_log(void *caller)
{
	struct gtod_log *log = find_hash(caller);

	if (!log) {
		unsigned long h;

		log = malloc(sizeof(*log));
		INIT_FLIST_HEAD(&log->list);
		log->caller = caller;
		log->calls = 0;

		h = hash_ptr(caller, HASH_BITS);
		flist_add_tail(&log->list, &hash[h]);
	}

	return log;
}

static void gtod_log_caller(void *caller)
{
	if (gtod_inited) {
		struct gtod_log *log = find_log(caller);

		log->calls++;
	}
}

static void fio_exit fio_dump_gtod(void)
{
	unsigned long total_calls = 0;
	int i;

	for (i = 0; i < HASH_SIZE; i++) {
		struct flist_head *entry;
		struct gtod_log *log;

		flist_for_each(entry, &hash[i]) {
			log = flist_entry(entry, struct gtod_log, list);

			printf("function %p, calls %lu\n", log->caller,
								log->calls);
			total_calls += log->calls;
		}
	}

	printf("Total %lu gettimeofday\n", total_calls);
}

static void fio_init gtod_init(void)
{
	int i;

	for (i = 0; i < HASH_SIZE; i++)
		INIT_FLIST_HEAD(&hash[i]);

	gtod_inited = 1;
}

#endif /* FIO_DEBUG_TIME */

#ifdef CONFIG_CLOCK_GETTIME
static int fill_clock_gettime(struct timespec *ts)
{
#ifdef CONFIG_CLOCK_MONOTONIC
	return clock_gettime(CLOCK_MONOTONIC, ts);
#else
	return clock_gettime(CLOCK_REALTIME, ts);
#endif
}
#endif

static void *__fio_gettime(struct timeval *tp)
{
	struct tv_valid *tv;

#ifdef CONFIG_TLS_THREAD
	tv = &static_tv_valid;
#else
	tv = pthread_getspecific(tv_tls_key);
#endif

	switch (fio_clock_source) {
#ifdef CONFIG_GETTIMEOFDAY
	case CS_GTOD:
		gettimeofday(tp, NULL);
		break;
#endif
#ifdef CONFIG_CLOCK_GETTIME
	case CS_CGETTIME: {
		struct timespec ts;

		if (fill_clock_gettime(&ts) < 0) {
			log_err("fio: clock_gettime fails\n");
			assert(0);
		}

		tp->tv_sec = ts.tv_sec;
		tp->tv_usec = ts.tv_nsec / 1000;
		break;
		}
#endif
#ifdef ARCH_HAVE_CPU_CLOCK
	case CS_CPUCLOCK: {
		uint64_t usecs, t;

		t = get_cpu_clock();
		if (tv && t < tv->last_cycles) {
			dprint(FD_TIME, "CPU clock going back in time\n");
			t = tv->last_cycles;
		} else if (tv)
			tv->last_cycles = t;

		usecs = (t * inv_cycles_per_usec) / 16777216UL;
		tp->tv_sec = usecs / 1000000;
		tp->tv_usec = usecs % 1000000;
		break;
		}
#endif
	default:
		log_err("fio: invalid clock source %d\n", fio_clock_source);
		break;
	}

	return tv;
}

#ifdef FIO_DEBUG_TIME
void fio_gettime(struct timeval *tp, void *caller)
#else
void fio_gettime(struct timeval *tp, void fio_unused *caller)
#endif
{
	struct tv_valid *tv;

#ifdef FIO_DEBUG_TIME
	if (!caller)
		caller = __builtin_return_address(0);

	gtod_log_caller(caller);
#endif
	if (fio_tv) {
		memcpy(tp, fio_tv, sizeof(*tp));
		return;
	}

	tv = __fio_gettime(tp);

	/*
	 * If Linux is using the tsc clock on non-synced processors,
	 * sometimes time can appear to drift backwards. Fix that up.
	 */
	if (tv) {
		if (tv->last_tv_valid) {
			if (tp->tv_sec < tv->last_tv.tv_sec)
				tp->tv_sec = tv->last_tv.tv_sec;
			else if (tv->last_tv.tv_sec == tp->tv_sec &&
				 tp->tv_usec < tv->last_tv.tv_usec)
				tp->tv_usec = tv->last_tv.tv_usec;
		}
		tv->last_tv_valid = 1;
		memcpy(&tv->last_tv, tp, sizeof(*tp));
	}
}

#ifdef ARCH_HAVE_CPU_CLOCK
static unsigned long get_cycles_per_usec(void)
{
	struct timeval s, e;
	uint64_t c_s, c_e;
	enum fio_cs old_cs = fio_clock_source;

#ifdef CONFIG_CLOCK_GETTIME
	fio_clock_source = CS_CGETTIME;
#else
	fio_clock_source = CS_GTOD;
#endif
	__fio_gettime(&s);

	c_s = get_cpu_clock();
	do {
		uint64_t elapsed;

		__fio_gettime(&e);

		elapsed = utime_since(&s, &e);
		if (elapsed >= 1280) {
			c_e = get_cpu_clock();
			break;
		}
	} while (1);

	fio_clock_source = old_cs;
	return (c_e - c_s + 127) >> 7;
}

#define NR_TIME_ITERS	50

static void calibrate_cpu_clock(void)
{
	double delta, mean, S;
	uint64_t avg, cycles[NR_TIME_ITERS];
	int i, samples;

	cycles[0] = get_cycles_per_usec();
	S = delta = mean = 0.0;
	for (i = 0; i < NR_TIME_ITERS; i++) {
		cycles[i] = get_cycles_per_usec();
		delta = cycles[i] - mean;
		if (delta) {
			mean += delta / (i + 1.0);
			S += delta * (cycles[i] - mean);
		}
	}

	S = sqrt(S / (NR_TIME_ITERS - 1.0));

	samples = avg = 0;
	for (i = 0; i < NR_TIME_ITERS; i++) {
		double this = cycles[i];

		if ((fmax(this, mean) - fmin(this, mean)) > S)
			continue;
		samples++;
		avg += this;
	}

	S /= (double) NR_TIME_ITERS;
	mean /= 10.0;

	for (i = 0; i < NR_TIME_ITERS; i++)
		dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10);

	avg /= samples;
	avg = (avg + 5) / 10;
	dprint(FD_TIME, "avg: %lu\n", avg);
	dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);

	cycles_per_usec = avg;
	inv_cycles_per_usec = 16777216UL / cycles_per_usec;
	dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
}
#else
static void calibrate_cpu_clock(void)
{
}
#endif

#ifndef CONFIG_TLS_THREAD
void fio_local_clock_init(int is_thread)
{
	struct tv_valid *t;

	t = calloc(sizeof(*t), 1);
	if (pthread_setspecific(tv_tls_key, t))
		log_err("fio: can't set TLS key\n");
}

static void kill_tv_tls_key(void *data)
{
	free(data);
}
#else
void fio_local_clock_init(int is_thread)
{
}
#endif

void fio_clock_init(void)
{
	if (fio_clock_source == fio_clock_source_inited)
		return;

#ifndef CONFIG_TLS_THREAD
	if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
		log_err("fio: can't create TLS key\n");
#endif

	/*
	 * Probably an AMD issue, will need to investigate. Until that
	 * is done, disable the CPU clock.
	 */
#if FIO_OS == os_solaris
	tsc_reliable = 0;
#endif

	fio_clock_source_inited = fio_clock_source;
	calibrate_cpu_clock();

	/*
	 * If the arch sets tsc_reliable != 0, then it must be good enough
	 * to use as THE clock source. For x86 CPUs, this means the TSC
	 * runs at a constant rate and is synced across CPU cores.
	 */
	if (tsc_reliable) {
		if (!fio_clock_source_set)
			fio_clock_source = CS_CPUCLOCK;
	} else if (fio_clock_source == CS_CPUCLOCK)
		log_info("fio: clocksource=cpu may not be reliable\n");
}

uint64_t utime_since(struct timeval *s, struct timeval *e)
{
	long sec, usec;
	uint64_t ret;

	sec = e->tv_sec - s->tv_sec;
	usec = e->tv_usec - s->tv_usec;
	if (sec > 0 && usec < 0) {
		sec--;
		usec += 1000000;
	}

	/*
	 * time warp bug on some kernels?
	 */
	if (sec < 0 || (sec == 0 && usec < 0))
		return 0;

	ret = sec * 1000000ULL + usec;

	return ret;
}

uint64_t utime_since_now(struct timeval *s)
{
	struct timeval t;

	fio_gettime(&t, NULL);
	return utime_since(s, &t);
}

uint64_t mtime_since(struct timeval *s, struct timeval *e)
{
	long sec, usec, ret;

	sec = e->tv_sec - s->tv_sec;
	usec = e->tv_usec - s->tv_usec;
	if (sec > 0 && usec < 0) {
		sec--;
		usec += 1000000;
	}

	if (sec < 0 || (sec == 0 && usec < 0))
		return 0;

	sec *= 1000UL;
	usec /= 1000UL;
	ret = sec + usec;

	return ret;
}

uint64_t mtime_since_now(struct timeval *s)
{
	struct timeval t;
	void *p = __builtin_return_address(0);

	fio_gettime(&t, p);
	return mtime_since(s, &t);
}

uint64_t time_since_now(struct timeval *s)
{
	return mtime_since_now(s) / 1000;
}

#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \
    defined(CONFIG_SFAA)

#define CLOCK_ENTRIES	100000

struct clock_entry {
	uint64_t seq;
	uint64_t tsc;
	uint64_t cpu;
};

struct clock_thread {
	pthread_t thread;
	int cpu;
	pthread_mutex_t lock;
	pthread_mutex_t started;
	uint64_t *seq;
	struct clock_entry *entries;
};

static inline uint64_t atomic64_inc_return(uint64_t *seq)
{
	return 1 + __sync_fetch_and_add(seq, 1);
}

static void *clock_thread_fn(void *data)
{
	struct clock_thread *t = data;
	struct clock_entry *c;
	os_cpu_mask_t cpu_mask;
	int i;

	memset(&cpu_mask, 0, sizeof(cpu_mask));
	fio_cpu_set(&cpu_mask, t->cpu);

	if (fio_setaffinity(gettid(), cpu_mask) == -1) {
		log_err("clock setaffinity failed\n");
		return (void *) 1;
	}

	pthread_mutex_lock(&t->lock);
	pthread_mutex_unlock(&t->started);

	c = &t->entries[0];
	for (i = 0; i < CLOCK_ENTRIES; i++, c++) {
		uint64_t seq, tsc;

		c->cpu = t->cpu;
		do {
			seq = atomic64_inc_return(t->seq);
			tsc = get_cpu_clock();
		} while (seq != *t->seq);

		c->seq = seq;
		c->tsc = tsc;
	}

	log_info("cs: cpu%3d: %lu clocks seen\n", t->cpu, t->entries[CLOCK_ENTRIES - 1].tsc - t->entries[0].tsc);
	return NULL;
}

static int clock_cmp(const void *p1, const void *p2)
{
	const struct clock_entry *c1 = p1;
	const struct clock_entry *c2 = p2;

	if (c1->seq == c2->seq)
		log_err("cs: bug in atomic sequence!\n");

	return c1->seq - c2->seq;
}

int fio_monotonic_clocktest(void)
{
	struct clock_thread *threads;
	unsigned int nr_cpus = cpus_online();
	struct clock_entry *entries;
	unsigned long tentries, failed;
	uint64_t seq = 0;
	int i;

	log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");

	fio_debug |= 1U << FD_TIME;
	calibrate_cpu_clock();
	fio_debug &= ~(1U << FD_TIME);

	threads = malloc(nr_cpus * sizeof(struct clock_thread));
	tentries = CLOCK_ENTRIES * nr_cpus;
	entries = malloc(tentries * sizeof(struct clock_entry));

	log_info("cs: Testing %u CPUs\n", nr_cpus);

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &threads[i];

		t->cpu = i;
		t->seq = &seq;
		t->entries = &entries[i * CLOCK_ENTRIES];
		pthread_mutex_init(&t->lock, NULL);
		pthread_mutex_init(&t->started, NULL);
		pthread_mutex_lock(&t->lock);
		pthread_create(&t->thread, NULL, clock_thread_fn, t);
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &threads[i];

		pthread_mutex_lock(&t->started);
	}

	for (i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &threads[i];

		pthread_mutex_unlock(&t->lock);
	}

	for (failed = i = 0; i < nr_cpus; i++) {
		struct clock_thread *t = &threads[i];
		void *ret;

		pthread_join(t->thread, &ret);
		if (ret)
			failed++;
	}
	free(threads);

	if (failed) {
		log_err("Clocksource test: %u threads failed\n", failed);
		goto err;
	}

	qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);

	for (failed = i = 0; i < tentries; i++) {
		struct clock_entry *prev, *this = &entries[i];

		if (!i) {
			prev = this;
			continue;
		}

		if (prev->tsc > this->tsc) {
			uint64_t diff = prev->tsc - this->tsc;

			log_info("cs: CPU clock mismatch (diff=%lu):\n", diff);
			log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", prev->cpu, prev->tsc, prev->seq);
			log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", this->cpu, this->tsc, this->seq);
			failed++;
		}

		prev = this;
	}

	if (failed)
		log_info("cs: Failed: %lu\n", failed);
	else
		log_info("cs: Pass!\n");

err:
	free(entries);
	return !!failed;
}

#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */

int fio_monotonic_clocktest(void)
{
	log_info("cs: current platform does not support CPU clocks\n");
	return 0;
}

#endif
Commit	Line	Data
02bcaa8c	1	/*
f5cc024a	2	* Clock functions
02bcaa8c	3	*/
f5cc024a	4
02bcaa8c	5	#include <unistd.h>
c223da83	6	#include <math.h>
02bcaa8c	7	#include <sys/time.h>
03e20d68	8	#include <time.h>
02bcaa8c JA	9
02bcaa8c JA	10	#include "fio.h"
be4ecfdf	11	#include "smalloc.h"
02bcaa8c JA	12
02bcaa8c JA	13	#include "hash.h"
7d11f871	14	#include "os/os.h"
02bcaa8c	15
09a32402	16	#ifdef ARCH_HAVE_CPU_CLOCK
c223da83	17	static unsigned long cycles_per_usec;
71339117	18	static unsigned long inv_cycles_per_usec;
09a32402	19	#endif
4de98eb0	20	int tsc_reliable = 0;
5d879392 JA	21
	22	struct tv_valid {
	23	struct timeval last_tv;
ba458c2f	24	uint64_t last_cycles;
5d879392 JA	25	int last_tv_valid;
5d879392 JA	26	};
67bf9823 JA	27	#ifdef CONFIG_TLS_THREAD
	28	static struct tv_valid __thread static_tv_valid;
	29	#else
5d879392	30	static pthread_key_t tv_tls_key;
67bf9823	31	#endif
02bcaa8c	32
16de1bf9	33	enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
fa80feae	34	int fio_clock_source_set = 0;
01423eae	35	enum fio_cs fio_clock_source_inited = CS_INVAL;
c223da83	36
02bcaa8c JA	37	#ifdef FIO_DEBUG_TIME
	38
	39	#define HASH_BITS 8
	40	#define HASH_SIZE (1 << HASH_BITS)
	41
01743ee1	42	static struct flist_head hash[HASH_SIZE];
02bcaa8c JA	43	static int gtod_inited;
	44
	45	struct gtod_log {
01743ee1	46	struct flist_head list;
02bcaa8c JA	47	void *caller;
	48	unsigned long calls;
	49	};
	50
	51	static struct gtod_log find_hash(void caller)
	52	{
	53	unsigned long h = hash_ptr(caller, HASH_BITS);
01743ee1	54	struct flist_head *entry;
02bcaa8c	55
01743ee1 JA	56	flist_for_each(entry, &hash[h]) {
	57	struct gtod_log *log = flist_entry(entry, struct gtod_log,
	58	list);
02bcaa8c JA	59
	60	if (log->caller == caller)
	61	return log;
	62	}
	63
	64	return NULL;
	65	}
	66
	67	static struct gtod_log find_log(void caller)
	68	{
	69	struct gtod_log *log = find_hash(caller);
	70
	71	if (!log) {
	72	unsigned long h;
	73
	74	log = malloc(sizeof(*log));
01743ee1	75	INIT_FLIST_HEAD(&log->list);
02bcaa8c JA	76	log->caller = caller;
	77	log->calls = 0;
	78
	79	h = hash_ptr(caller, HASH_BITS);
01743ee1	80	flist_add_tail(&log->list, &hash[h]);
02bcaa8c JA	81	}
	82
	83	return log;
	84	}
	85
	86	static void gtod_log_caller(void *caller)
	87	{
	88	if (gtod_inited) {
	89	struct gtod_log *log = find_log(caller);
	90
	91	log->calls++;
	92	}
	93	}
	94
	95	static void fio_exit fio_dump_gtod(void)
	96	{
	97	unsigned long total_calls = 0;
	98	int i;
	99
	100	for (i = 0; i < HASH_SIZE; i++) {
01743ee1	101	struct flist_head *entry;
02bcaa8c JA	102	struct gtod_log *log;
02bcaa8c JA	103
01743ee1 JA	104	flist_for_each(entry, &hash[i]) {
01743ee1 JA	105	log = flist_entry(entry, struct gtod_log, list);
02bcaa8c	106
5ec10eaa JA	107	printf("function %p, calls %lu\n", log->caller,
5ec10eaa JA	108	log->calls);
02bcaa8c JA	109	total_calls += log->calls;
	110	}
	111	}
	112
	113	printf("Total %lu gettimeofday\n", total_calls);
	114	}
	115
	116	static void fio_init gtod_init(void)
	117	{
	118	int i;
	119
	120	for (i = 0; i < HASH_SIZE; i++)
01743ee1	121	INIT_FLIST_HEAD(&hash[i]);
02bcaa8c JA	122
	123	gtod_inited = 1;
	124	}
	125
	126	#endif /* FIO_DEBUG_TIME */
	127
67bf9823	128	#ifdef CONFIG_CLOCK_GETTIME
9ff1c070 JA	129	static int fill_clock_gettime(struct timespec *ts)
9ff1c070 JA	130	{
67bf9823	131	#ifdef CONFIG_CLOCK_MONOTONIC
9ff1c070 JA	132	return clock_gettime(CLOCK_MONOTONIC, ts);
	133	#else
	134	return clock_gettime(CLOCK_REALTIME, ts);
	135	#endif
	136	}
1e97cce9	137	#endif
67bf9823 JA	138
67bf9823 JA	139	static void __fio_gettime(struct timeval tp)
02bcaa8c	140	{
93f0b09a	141	struct tv_valid *tv;
5d879392	142
67bf9823 JA	143	#ifdef CONFIG_TLS_THREAD
	144	tv = &static_tv_valid;
	145	#else
93f0b09a	146	tv = pthread_getspecific(tv_tls_key);
67bf9823	147	#endif
93f0b09a	148
c223da83	149	switch (fio_clock_source) {
67bf9823	150	#ifdef CONFIG_GETTIMEOFDAY
c223da83	151	case CS_GTOD:
02bcaa8c	152	gettimeofday(tp, NULL);
c223da83	153	break;
67bf9823 JA	154	#endif
67bf9823 JA	155	#ifdef CONFIG_CLOCK_GETTIME
c223da83	156	case CS_CGETTIME: {
02bcaa8c JA	157	struct timespec ts;
02bcaa8c JA	158
9ff1c070	159	if (fill_clock_gettime(&ts) < 0) {
c223da83 JA	160	log_err("fio: clock_gettime fails\n");
c223da83 JA	161	assert(0);
02bcaa8c JA	162	}
	163
	164	tp->tv_sec = ts.tv_sec;
	165	tp->tv_usec = ts.tv_nsec / 1000;
c223da83 JA	166	break;
c223da83 JA	167	}
67bf9823	168	#endif
c223da83 JA	169	#ifdef ARCH_HAVE_CPU_CLOCK
c223da83 JA	170	case CS_CPUCLOCK: {
ba458c2f	171	uint64_t usecs, t;
c223da83 JA	172
c223da83 JA	173	t = get_cpu_clock();
93f0b09a	174	if (tv && t < tv->last_cycles) {
c223da83	175	dprint(FD_TIME, "CPU clock going back in time\n");
93f0b09a JA	176	t = tv->last_cycles;
	177	} else if (tv)
	178	tv->last_cycles = t;
c223da83	179
71339117	180	usecs = (t * inv_cycles_per_usec) / 16777216UL;
c223da83 JA	181	tp->tv_sec = usecs / 1000000;
c223da83 JA	182	tp->tv_usec = usecs % 1000000;
c223da83 JA	183	break;
	184	}
	185	#endif
	186	default:
	187	log_err("fio: invalid clock source %d\n", fio_clock_source);
	188	break;
02bcaa8c	189	}
3e488920	190
67bf9823 JA	191	return tv;
	192	}
	193
	194	#ifdef FIO_DEBUG_TIME
	195	void fio_gettime(struct timeval tp, void caller)
	196	#else
	197	void fio_gettime(struct timeval tp, void fio_unused caller)
	198	#endif
	199	{
	200	struct tv_valid *tv;
	201
	202	#ifdef FIO_DEBUG_TIME
	203	if (!caller)
	204	caller = __builtin_return_address(0);
	205
	206	gtod_log_caller(caller);
	207	#endif
	208	if (fio_tv) {
	209	memcpy(tp, fio_tv, sizeof(*tp));
	210	return;
	211	}
	212
	213	tv = __fio_gettime(tp);
	214
3e488920 JA	215	/*
	216	* If Linux is using the tsc clock on non-synced processors,
	217	* sometimes time can appear to drift backwards. Fix that up.
	218	*/
93f0b09a JA	219	if (tv) {
	220	if (tv->last_tv_valid) {
	221	if (tp->tv_sec < tv->last_tv.tv_sec)
	222	tp->tv_sec = tv->last_tv.tv_sec;
	223	else if (tv->last_tv.tv_sec == tp->tv_sec &&
	224	tp->tv_usec < tv->last_tv.tv_usec)
	225	tp->tv_usec = tv->last_tv.tv_usec;
5d879392	226	}
93f0b09a JA	227	tv->last_tv_valid = 1;
93f0b09a JA	228	memcpy(&tv->last_tv, tp, sizeof(*tp));
3e488920	229	}
02bcaa8c	230	}
be4ecfdf	231
09a32402	232	#ifdef ARCH_HAVE_CPU_CLOCK
c223da83 JA	233	static unsigned long get_cycles_per_usec(void)
	234	{
	235	struct timeval s, e;
ba458c2f	236	uint64_t c_s, c_e;
67bf9823	237	enum fio_cs old_cs = fio_clock_source;
c223da83	238
67bf9823 JA	239	#ifdef CONFIG_CLOCK_GETTIME
	240	fio_clock_source = CS_CGETTIME;
	241	#else
	242	fio_clock_source = CS_GTOD;
	243	#endif
	244	__fio_gettime(&s);
9ff1c070	245
c223da83 JA	246	c_s = get_cpu_clock();
c223da83 JA	247	do {
ba458c2f	248	uint64_t elapsed;
c223da83	249
67bf9823	250	__fio_gettime(&e);
9ff1c070	251
c223da83	252	elapsed = utime_since(&s, &e);
486332e5	253	if (elapsed >= 1280) {
c223da83 JA	254	c_e = get_cpu_clock();
	255	break;
	256	}
	257	} while (1);
	258
67bf9823	259	fio_clock_source = old_cs;
c011000d	260	return (c_e - c_s + 127) >> 7;
c223da83 JA	261	}
c223da83 JA	262
fa80feae JA	263	#define NR_TIME_ITERS 50
fa80feae JA	264
09a32402	265	static void calibrate_cpu_clock(void)
c223da83 JA	266	{
c223da83 JA	267	double delta, mean, S;
ba458c2f	268	uint64_t avg, cycles[NR_TIME_ITERS];
c223da83 JA	269	int i, samples;
c223da83 JA	270
c223da83 JA	271	cycles[0] = get_cycles_per_usec();
c223da83 JA	272	S = delta = mean = 0.0;
fa80feae	273	for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83 JA	274	cycles[i] = get_cycles_per_usec();
	275	delta = cycles[i] - mean;
	276	if (delta) {
	277	mean += delta / (i + 1.0);
	278	S += delta * (cycles[i] - mean);
	279	}
	280	}
	281
fa80feae	282	S = sqrt(S / (NR_TIME_ITERS - 1.0));
c223da83 JA	283
c223da83 JA	284	samples = avg = 0;
fa80feae	285	for (i = 0; i < NR_TIME_ITERS; i++) {
c223da83 JA	286	double this = cycles[i];
c223da83 JA	287
03e20d68	288	if ((fmax(this, mean) - fmin(this, mean)) > S)
c223da83 JA	289	continue;
	290	samples++;
	291	avg += this;
	292	}
	293
fa80feae	294	S /= (double) NR_TIME_ITERS;
89db727d	295	mean /= 10.0;
c223da83	296
fa80feae	297	for (i = 0; i < NR_TIME_ITERS; i++)
c223da83 JA	298	dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10);
c223da83 JA	299
d7abad3d	300	avg /= samples;
b0ff22d7	301	avg = (avg + 5) / 10;
c223da83 JA	302	dprint(FD_TIME, "avg: %lu\n", avg);
	303	dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);
	304
	305	cycles_per_usec = avg;
71339117 JA	306	inv_cycles_per_usec = 16777216UL / cycles_per_usec;
71339117 JA	307	dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
09a32402 JA	308	}
	309	#else
	310	static void calibrate_cpu_clock(void)
	311	{
	312	}
	313	#endif
	314
67bf9823	315	#ifndef CONFIG_TLS_THREAD
5d879392 JA	316	void fio_local_clock_init(int is_thread)
	317	{
	318	struct tv_valid *t;
	319
	320	t = calloc(sizeof(*t), 1);
	321	if (pthread_setspecific(tv_tls_key, t))
	322	log_err("fio: can't set TLS key\n");
	323	}
	324
	325	static void kill_tv_tls_key(void *data)
	326	{
	327	free(data);
	328	}
67bf9823 JA	329	#else
	330	void fio_local_clock_init(int is_thread)
	331	{
	332	}
	333	#endif
5d879392	334
09a32402 JA	335	void fio_clock_init(void)
09a32402 JA	336	{
01423eae JA	337	if (fio_clock_source == fio_clock_source_inited)
	338	return;
	339
67bf9823	340	#ifndef CONFIG_TLS_THREAD
5d879392 JA	341	if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
5d879392 JA	342	log_err("fio: can't create TLS key\n");
67bf9823	343	#endif
5d879392	344
372fd659 JA	345	/*
	346	* Probably an AMD issue, will need to investigate. Until that
	347	* is done, disable the CPU clock.
	348	*/
	349	#if FIO_OS == os_solaris
	350	tsc_reliable = 0;
	351	#endif
	352
01423eae	353	fio_clock_source_inited = fio_clock_source;
09a32402	354	calibrate_cpu_clock();
fa80feae JA	355
	356	/*
	357	* If the arch sets tsc_reliable != 0, then it must be good enough
	358	* to use as THE clock source. For x86 CPUs, this means the TSC
	359	* runs at a constant rate and is synced across CPU cores.
	360	*/
	361	if (tsc_reliable) {
	362	if (!fio_clock_source_set)
	363	fio_clock_source = CS_CPUCLOCK;
	364	} else if (fio_clock_source == CS_CPUCLOCK)
	365	log_info("fio: clocksource=cpu may not be reliable\n");
c223da83 JA	366	}
c223da83 JA	367
aa60bc58	368	uint64_t utime_since(struct timeval s, struct timeval e)
be4ecfdf	369	{
39ab7da2	370	long sec, usec;
aa60bc58	371	uint64_t ret;
39ab7da2 JA	372
	373	sec = e->tv_sec - s->tv_sec;
	374	usec = e->tv_usec - s->tv_usec;
	375	if (sec > 0 && usec < 0) {
	376	sec--;
	377	usec += 1000000;
	378	}
	379
	380	/*
	381	* time warp bug on some kernels?
	382	*/
	383	if (sec < 0 \|\| (sec == 0 && usec < 0))
	384	return 0;
	385
	386	ret = sec * 1000000ULL + usec;
	387
	388	return ret;
be4ecfdf JA	389	}
be4ecfdf JA	390
aa60bc58	391	uint64_t utime_since_now(struct timeval *s)
be4ecfdf	392	{
39ab7da2 JA	393	struct timeval t;
	394
	395	fio_gettime(&t, NULL);
	396	return utime_since(s, &t);
be4ecfdf	397	}
783a3eb1	398
aa60bc58	399	uint64_t mtime_since(struct timeval s, struct timeval e)
783a3eb1	400	{
39ab7da2	401	long sec, usec, ret;
783a3eb1	402
39ab7da2 JA	403	sec = e->tv_sec - s->tv_sec;
	404	usec = e->tv_usec - s->tv_usec;
	405	if (sec > 0 && usec < 0) {
	406	sec--;
	407	usec += 1000000;
783a3eb1 JA	408	}
783a3eb1 JA	409
39ab7da2 JA	410	if (sec < 0 \|\| (sec == 0 && usec < 0))
	411	return 0;
	412
	413	sec *= 1000UL;
	414	usec /= 1000UL;
	415	ret = sec + usec;
	416
	417	return ret;
783a3eb1 JA	418	}
783a3eb1 JA	419
aa60bc58	420	uint64_t mtime_since_now(struct timeval *s)
783a3eb1	421	{
39ab7da2 JA	422	struct timeval t;
39ab7da2 JA	423	void *p = __builtin_return_address(0);
783a3eb1	424
39ab7da2 JA	425	fio_gettime(&t, p);
	426	return mtime_since(s, &t);
	427	}
783a3eb1	428
aa60bc58	429	uint64_t time_since_now(struct timeval *s)
39ab7da2 JA	430	{
39ab7da2 JA	431	return mtime_since_now(s) / 1000;
783a3eb1	432	}
7d11f871	433
67bf9823 JA	434	#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
67bf9823 JA	435	defined(CONFIG_SFAA)
7d11f871 JA	436
	437	#define CLOCK_ENTRIES 100000
	438
	439	struct clock_entry {
ba458c2f JA	440	uint64_t seq;
	441	uint64_t tsc;
	442	uint64_t cpu;
7d11f871 JA	443	};
	444
	445	struct clock_thread {
	446	pthread_t thread;
	447	int cpu;
	448	pthread_mutex_t lock;
	449	pthread_mutex_t started;
	450	uint64_t *seq;
	451	struct clock_entry *entries;
	452	};
	453
	454	static inline uint64_t atomic64_inc_return(uint64_t *seq)
	455	{
	456	return 1 + __sync_fetch_and_add(seq, 1);
	457	}
	458
	459	static void clock_thread_fn(void data)
	460	{
	461	struct clock_thread *t = data;
	462	struct clock_entry *c;
	463	os_cpu_mask_t cpu_mask;
	464	int i;
	465
	466	memset(&cpu_mask, 0, sizeof(cpu_mask));
	467	fio_cpu_set(&cpu_mask, t->cpu);
	468
	469	if (fio_setaffinity(gettid(), cpu_mask) == -1) {
	470	log_err("clock setaffinity failed\n");
	471	return (void *) 1;
	472	}
	473
7d11f871	474	pthread_mutex_lock(&t->lock);
b9b3498e	475	pthread_mutex_unlock(&t->started);
7d11f871 JA	476
	477	c = &t->entries[0];
	478	for (i = 0; i < CLOCK_ENTRIES; i++, c++) {
	479	uint64_t seq, tsc;
	480
	481	c->cpu = t->cpu;
	482	do {
	483	seq = atomic64_inc_return(t->seq);
	484	tsc = get_cpu_clock();
	485	} while (seq != *t->seq);
	486
	487	c->seq = seq;
	488	c->tsc = tsc;
	489	}
	490
	491	log_info("cs: cpu%3d: %lu clocks seen\n", t->cpu, t->entries[CLOCK_ENTRIES - 1].tsc - t->entries[0].tsc);
	492	return NULL;
	493	}
	494
	495	static int clock_cmp(const void p1, const void p2)
	496	{
	497	const struct clock_entry *c1 = p1;
	498	const struct clock_entry *c2 = p2;
	499
b9b3498e JA	500	if (c1->seq == c2->seq)
	501	log_err("cs: bug in atomic sequence!\n");
	502
7d11f871 JA	503	return c1->seq - c2->seq;
	504	}
	505
	506	int fio_monotonic_clocktest(void)
	507	{
	508	struct clock_thread *threads;
	509	unsigned int nr_cpus = cpus_online();
	510	struct clock_entry *entries;
	511	unsigned long tentries, failed;
	512	uint64_t seq = 0;
	513	int i;
	514
d5e3f5d8 JA	515	log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
d5e3f5d8 JA	516
4f1d43c2 JA	517	fio_debug \|= 1U << FD_TIME;
	518	calibrate_cpu_clock();
	519	fio_debug &= ~(1U << FD_TIME);
	520
7d11f871 JA	521	threads = malloc(nr_cpus * sizeof(struct clock_thread));
	522	tentries = CLOCK_ENTRIES * nr_cpus;
	523	entries = malloc(tentries * sizeof(struct clock_entry));
	524
	525	log_info("cs: Testing %u CPUs\n", nr_cpus);
	526
	527	for (i = 0; i < nr_cpus; i++) {
	528	struct clock_thread *t = &threads[i];
	529
	530	t->cpu = i;
	531	t->seq = &seq;
	532	t->entries = &entries[i * CLOCK_ENTRIES];
	533	pthread_mutex_init(&t->lock, NULL);
	534	pthread_mutex_init(&t->started, NULL);
	535	pthread_mutex_lock(&t->lock);
	536	pthread_create(&t->thread, NULL, clock_thread_fn, t);
	537	}
	538
	539	for (i = 0; i < nr_cpus; i++) {
	540	struct clock_thread *t = &threads[i];
	541
	542	pthread_mutex_lock(&t->started);
	543	}
	544
	545	for (i = 0; i < nr_cpus; i++) {
	546	struct clock_thread *t = &threads[i];
	547
	548	pthread_mutex_unlock(&t->lock);
	549	}
	550
	551	for (failed = i = 0; i < nr_cpus; i++) {
	552	struct clock_thread *t = &threads[i];
	553	void *ret;
	554
	555	pthread_join(t->thread, &ret);
	556	if (ret)
	557	failed++;
	558	}
	559	free(threads);
	560
	561	if (failed) {
	562	log_err("Clocksource test: %u threads failed\n", failed);
	563	goto err;
	564	}
	565
	566	qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
	567
	568	for (failed = i = 0; i < tentries; i++) {
	569	struct clock_entry prev, this = &entries[i];
	570
	571	if (!i) {
	572	prev = this;
	573	continue;
	574	}
	575
	576	if (prev->tsc > this->tsc) {
	577	uint64_t diff = prev->tsc - this->tsc;
	578
	579	log_info("cs: CPU clock mismatch (diff=%lu):\n", diff);
	580	log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", prev->cpu, prev->tsc, prev->seq);
	581	log_info("\t CPU%3lu: TSC=%lu, SEQ=%lu\n", this->cpu, this->tsc, this->seq);
	582	failed++;
	583	}
	584
585	prev = this;
586	}
587
588	if (failed)
589	log_info("cs: Failed: %lu\n", failed);
590	else
591	log_info("cs: Pass!\n");
592
593	err:
594	free(entries);
595	return !!failed;
596	}
597
598	#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
599
600	int fio_monotonic_clocktest(void)
601	{
602	log_info("cs: current platform does not support CPU clocks\n");
603	return 0;
604	}
605
606	#endif