Add check for invariant TSC on x86 and use TSC is default clock if reliable

TSC is by far the fastest clock we can use. Check the CPUID bits for
whether it is both constant rate AND synced across cores. If it is,
we can use it as our default clock source.

Fio will default to this clock source on x86 if no other clock source
is specifically given with clocksource= in the job file.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/arch/arch-x86-common.h b/arch/arch-x86-common.h
new file mode 100644 (file)
index 0000000..1e62354
--- /dev/null
+++ b/arch/arch-x86-common.h
@@ -0,0 +1,43 @@
+#ifndef FIO_ARCH_X86_COMMON
+#define FIO_ARCH_X86_COMMON
+
+static inline void do_cpuid(unsigned int *eax, unsigned int *ebx,
+                           unsigned int *ecx, unsigned int *edx)
+{
+       unsigned int id = *eax;
+
+       asm("movl %4, %%eax;"
+           "cpuid;"
+           "movl %%eax, %0;"
+           "movl %%ebx, %1;"
+           "movl %%ecx, %2;"
+           "movl %%edx, %3;"
+               : "=r" (*eax), "=r" (*ebx), "=r" (*ecx), "=r" (*edx)
+               : "r" (id)
+               : "eax", "ebx", "ecx", "edx");
+}
+
+#define ARCH_HAVE_INIT
+extern int tsc_reliable;
+static inline int arch_init(char *envp[])
+{
+       unsigned int eax, ebx, ecx, edx;
+
+       /*
+        * Check for TSC
+        */
+       eax = 1;
+       do_cpuid(&eax, &ebx, &ecx, &edx);
+       if (!(edx & (1U << 4)))
+               return 0;
+
+       /*
+        * Check for constant rate and synced (across cores) TSC
+        */
+       eax = 0x80000007;
+       do_cpuid(&eax, &ebx, &ecx, &edx);
+       tsc_reliable = edx & (1U << 8);
+       return 0;
+}
+
+#endif

diff --git a/arch/arch-x86.h b/arch/arch-x86.h
index 1ededd845a869b8751c64ed0a06e5bd41aad3273..48030060041bfc48391ec90f998bbace4556741e 100644 (file)
--- a/arch/arch-x86.h
+++ b/arch/arch-x86.h
@@ -1,6 +1,8 @@
 #ifndef ARCH_X86_H
 #define ARCH_X86_H
 
+#include "arch-x86-common.h"
+
 #define FIO_ARCH       (arch_i386)
 
 #ifndef __NR_ioprio_set

diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h
index 29e681f2f8533926451e731460b69a499fc0aa53..d8b0933bdadda5e3c7c076b176d9655022bf93ec 100644 (file)
--- a/arch/arch-x86_64.h
+++ b/arch/arch-x86_64.h
@@ -1,6 +1,8 @@
 #ifndef ARCH_X86_64_h
 #define ARCH_X86_64_h
 
+#include "arch-x86-common.h"
+
 #define FIO_ARCH       (arch_x86_64)
 
 #ifndef __NR_ioprio_set

diff --git a/crc/crc32c-intel.c b/crc/crc32c-intel.c
index 8a6e6dcefb749b41ed5ecddb7035ae6a8c247adf..8e1cd587dcec85adc2a75ec4bd824e75457514b5 100644 (file)
--- a/crc/crc32c-intel.c
+++ b/crc/crc32c-intel.c
@@ -78,22 +78,6 @@ uint32_t crc32c_intel(unsigned char const *data, unsigned long length)
        return crc;
 }
 
-static void do_cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx,
-                    unsigned int *edx)
-{
-       int id = *eax;
-
-       asm("movl %4, %%eax;"
-           "cpuid;"
-           "movl %%eax, %0;"
-           "movl %%ebx, %1;"
-           "movl %%ecx, %2;"
-           "movl %%edx, %3;"
-               : "=r" (*eax), "=r" (*ebx), "=r" (*ecx), "=r" (*edx)
-               : "r" (id)
-               : "eax", "ebx", "ecx", "edx");
-}
-
 void crc32c_intel_probe(void)
 {
        if (!crc32c_probed) {

diff --git a/fio.c b/fio.c
index ac026fba8117ed09995ba4cfd398ab1f10bf19d3..f44273f483c59103ece4a3d4681496e663698f66 100644 (file)
--- a/fio.c
+++ b/fio.c
@@ -104,6 +104,8 @@ int main(int argc, char *argv[], char *envp[])
        if (parse_options(argc, argv))
                return 1;
 
+       fio_time_init();
+
        if (nr_clients)
                return fio_handle_clients();
        else

diff --git a/fio.h b/fio.h
index 3de3f000983a5621ee65c2e8d975e250d0ace7cf..de4ca4daefa2ebc6bceb770e345be87a8d2389aa 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -598,6 +598,7 @@ extern char *job_section;
 extern int fio_gtod_offload;
 extern int fio_gtod_cpu;
 extern enum fio_cs fio_clock_source;
+extern int fio_clock_source_set;
 extern int warnings_fatal;
 extern int terse_version;
 extern int is_backend;

diff --git a/gettime.c b/gettime.c
index 35d685e1576149974d5e77f18db2b1fb180f3a9e..89a3a016e4c871a767365f96318392a65ec30ba6 100644 (file)
--- a/gettime.c
+++ b/gettime.c
@@ -15,11 +15,13 @@
 #ifdef ARCH_HAVE_CPU_CLOCK
 static unsigned long cycles_per_usec;
 static unsigned long last_cycles;
+int tsc_reliable = 0;
 #endif
 static struct timeval last_tv;
 static int last_tv_valid;
 
 enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
+int fio_clock_source_set = 0;
 
 #ifdef FIO_DEBUG_TIME
 
@@ -208,15 +210,17 @@ static unsigned long get_cycles_per_usec(void)
        return c_e - c_s;
 }
 
+#define NR_TIME_ITERS  50
+
 static void calibrate_cpu_clock(void)
 {
        double delta, mean, S;
-       unsigned long avg, cycles[10];
+       unsigned long avg, cycles[NR_TIME_ITERS];
        int i, samples;
 
        cycles[0] = get_cycles_per_usec();
        S = delta = mean = 0.0;
-       for (i = 0; i < 10; i++) {
+       for (i = 0; i < NR_TIME_ITERS; i++) {
                cycles[i] = get_cycles_per_usec();
                delta = cycles[i] - mean;
                if (delta) {
@@ -225,10 +229,10 @@ static void calibrate_cpu_clock(void)
                }
        }
 
-       S = sqrt(S / (10 - 1.0));
+       S = sqrt(S / (NR_TIME_ITERS - 1.0));
 
        samples = avg = 0;
-       for (i = 0; i < 10; i++) {
+       for (i = 0; i < NR_TIME_ITERS; i++) {
                double this = cycles[i];
 
                if ((fmax(this, mean) - fmin(this, mean)) > S)
@@ -237,10 +241,10 @@ static void calibrate_cpu_clock(void)
                avg += this;
        }
 
-       S /= 10.0;
-       mean /= 10.0;
+       S /= (double) NR_TIME_ITERS;
+       mean /= (double) NR_TIME_ITERS;
 
-       for (i = 0; i < 10; i++)
+       for (i = 0; i < NR_TIME_ITERS; i++)
                dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10);
 
        avg /= (samples * 10);
@@ -248,7 +252,6 @@ static void calibrate_cpu_clock(void)
        dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);
 
        cycles_per_usec = avg;
-
 }
 #else
 static void calibrate_cpu_clock(void)
@@ -260,6 +263,17 @@ void fio_clock_init(void)
 {
        last_tv_valid = 0;
        calibrate_cpu_clock();
+
+       /*
+        * If the arch sets tsc_reliable != 0, then it must be good enough
+        * to use as THE clock source. For x86 CPUs, this means the TSC
+        * runs at a constant rate and is synced across CPU cores.
+        */
+       if (tsc_reliable) {
+               if (!fio_clock_source_set)
+                       fio_clock_source = CS_CPUCLOCK;
+       } else if (fio_clock_source == CS_CPUCLOCK)
+               log_info("fio: clocksource=cpu may not be reliable\n");
 }
 
 unsigned long long utime_since(struct timeval *s, struct timeval *e)

diff --git a/options.c b/options.c
index d46bcbbcf6c9cab8fc1c41e8800bc5071429ccd9..738c78bb7daea3cfead7a2e1090b8af595f8c564 100644 (file)
--- a/options.c
+++ b/options.c
@@ -370,7 +370,7 @@ static int fio_clock_source_cb(void *data, const char *str)
        struct thread_data *td = data;
 
        fio_clock_source = td->o.clocksource;
-       fio_time_init();
+       fio_clock_source_set = 1;
        return 0;
 }