From fa80feae51331fb170e784459fa1359d7ec3a963 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 9 Dec 2012 20:29:00 +0100 Subject: [PATCH] Add check for invariant TSC on x86 and use TSC is default clock if reliable TSC is by far the fastest clock we can use. Check the CPUID bits for whether it is both constant rate AND synced across cores. If it is, we can use it as our default clock source. Fio will default to this clock source on x86 if no other clock source is specifically given with clocksource= in the job file. Signed-off-by: Jens Axboe --- arch/arch-x86-common.h | 43 ++++++++++++++++++++++++++++++++++++++++++ arch/arch-x86.h | 2 ++ arch/arch-x86_64.h | 2 ++ crc/crc32c-intel.c | 16 ---------------- fio.c | 2 ++ fio.h | 1 + gettime.c | 30 +++++++++++++++++++++-------- options.c | 2 +- 8 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 arch/arch-x86-common.h diff --git a/arch/arch-x86-common.h b/arch/arch-x86-common.h new file mode 100644 index 00000000..1e623544 --- /dev/null +++ b/arch/arch-x86-common.h @@ -0,0 +1,43 @@ +#ifndef FIO_ARCH_X86_COMMON +#define FIO_ARCH_X86_COMMON + +static inline void do_cpuid(unsigned int *eax, unsigned int *ebx, + unsigned int *ecx, unsigned int *edx) +{ + unsigned int id = *eax; + + asm("movl %4, %%eax;" + "cpuid;" + "movl %%eax, %0;" + "movl %%ebx, %1;" + "movl %%ecx, %2;" + "movl %%edx, %3;" + : "=r" (*eax), "=r" (*ebx), "=r" (*ecx), "=r" (*edx) + : "r" (id) + : "eax", "ebx", "ecx", "edx"); +} + +#define ARCH_HAVE_INIT +extern int tsc_reliable; +static inline int arch_init(char *envp[]) +{ + unsigned int eax, ebx, ecx, edx; + + /* + * Check for TSC + */ + eax = 1; + do_cpuid(&eax, &ebx, &ecx, &edx); + if (!(edx & (1U << 4))) + return 0; + + /* + * Check for constant rate and synced (across cores) TSC + */ + eax = 0x80000007; + do_cpuid(&eax, &ebx, &ecx, &edx); + tsc_reliable = edx & (1U << 8); + return 0; +} + +#endif diff --git a/arch/arch-x86.h b/arch/arch-x86.h index 1ededd84..48030060 100644 --- a/arch/arch-x86.h +++ b/arch/arch-x86.h @@ -1,6 +1,8 @@ #ifndef ARCH_X86_H #define ARCH_X86_H +#include "arch-x86-common.h" + #define FIO_ARCH (arch_i386) #ifndef __NR_ioprio_set diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h index 29e681f2..d8b0933b 100644 --- a/arch/arch-x86_64.h +++ b/arch/arch-x86_64.h @@ -1,6 +1,8 @@ #ifndef ARCH_X86_64_h #define ARCH_X86_64_h +#include "arch-x86-common.h" + #define FIO_ARCH (arch_x86_64) #ifndef __NR_ioprio_set diff --git a/crc/crc32c-intel.c b/crc/crc32c-intel.c index 8a6e6dce..8e1cd587 100644 --- a/crc/crc32c-intel.c +++ b/crc/crc32c-intel.c @@ -78,22 +78,6 @@ uint32_t crc32c_intel(unsigned char const *data, unsigned long length) return crc; } -static void do_cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, - unsigned int *edx) -{ - int id = *eax; - - asm("movl %4, %%eax;" - "cpuid;" - "movl %%eax, %0;" - "movl %%ebx, %1;" - "movl %%ecx, %2;" - "movl %%edx, %3;" - : "=r" (*eax), "=r" (*ebx), "=r" (*ecx), "=r" (*edx) - : "r" (id) - : "eax", "ebx", "ecx", "edx"); -} - void crc32c_intel_probe(void) { if (!crc32c_probed) { diff --git a/fio.c b/fio.c index ac026fba..f44273f4 100644 --- a/fio.c +++ b/fio.c @@ -104,6 +104,8 @@ int main(int argc, char *argv[], char *envp[]) if (parse_options(argc, argv)) return 1; + fio_time_init(); + if (nr_clients) return fio_handle_clients(); else diff --git a/fio.h b/fio.h index 3de3f000..de4ca4da 100644 --- a/fio.h +++ b/fio.h @@ -598,6 +598,7 @@ extern char *job_section; extern int fio_gtod_offload; extern int fio_gtod_cpu; extern enum fio_cs fio_clock_source; +extern int fio_clock_source_set; extern int warnings_fatal; extern int terse_version; extern int is_backend; diff --git a/gettime.c b/gettime.c index 35d685e1..89a3a016 100644 --- a/gettime.c +++ b/gettime.c @@ -15,11 +15,13 @@ #ifdef ARCH_HAVE_CPU_CLOCK static unsigned long cycles_per_usec; static unsigned long last_cycles; +int tsc_reliable = 0; #endif static struct timeval last_tv; static int last_tv_valid; enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE; +int fio_clock_source_set = 0; #ifdef FIO_DEBUG_TIME @@ -208,15 +210,17 @@ static unsigned long get_cycles_per_usec(void) return c_e - c_s; } +#define NR_TIME_ITERS 50 + static void calibrate_cpu_clock(void) { double delta, mean, S; - unsigned long avg, cycles[10]; + unsigned long avg, cycles[NR_TIME_ITERS]; int i, samples; cycles[0] = get_cycles_per_usec(); S = delta = mean = 0.0; - for (i = 0; i < 10; i++) { + for (i = 0; i < NR_TIME_ITERS; i++) { cycles[i] = get_cycles_per_usec(); delta = cycles[i] - mean; if (delta) { @@ -225,10 +229,10 @@ static void calibrate_cpu_clock(void) } } - S = sqrt(S / (10 - 1.0)); + S = sqrt(S / (NR_TIME_ITERS - 1.0)); samples = avg = 0; - for (i = 0; i < 10; i++) { + for (i = 0; i < NR_TIME_ITERS; i++) { double this = cycles[i]; if ((fmax(this, mean) - fmin(this, mean)) > S) @@ -237,10 +241,10 @@ static void calibrate_cpu_clock(void) avg += this; } - S /= 10.0; - mean /= 10.0; + S /= (double) NR_TIME_ITERS; + mean /= (double) NR_TIME_ITERS; - for (i = 0; i < 10; i++) + for (i = 0; i < NR_TIME_ITERS; i++) dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10); avg /= (samples * 10); @@ -248,7 +252,6 @@ static void calibrate_cpu_clock(void) dprint(FD_TIME, "mean=%f, S=%f\n", mean, S); cycles_per_usec = avg; - } #else static void calibrate_cpu_clock(void) @@ -260,6 +263,17 @@ void fio_clock_init(void) { last_tv_valid = 0; calibrate_cpu_clock(); + + /* + * If the arch sets tsc_reliable != 0, then it must be good enough + * to use as THE clock source. For x86 CPUs, this means the TSC + * runs at a constant rate and is synced across CPU cores. + */ + if (tsc_reliable) { + if (!fio_clock_source_set) + fio_clock_source = CS_CPUCLOCK; + } else if (fio_clock_source == CS_CPUCLOCK) + log_info("fio: clocksource=cpu may not be reliable\n"); } unsigned long long utime_since(struct timeval *s, struct timeval *e) diff --git a/options.c b/options.c index d46bcbbc..738c78bb 100644 --- a/options.c +++ b/options.c @@ -370,7 +370,7 @@ static int fio_clock_source_cb(void *data, const char *str) struct thread_data *td = data; fio_clock_source = td->o.clocksource; - fio_time_init(); + fio_clock_source_set = 1; return 0; } -- 2.25.1