gettime: improve cpu clock test

We're missing a synchronization before reading and comparing
with the current *t->seq in the loop. Rewrite the loop to
use compare-and-exchange for the increment, and ensure that
we have the proper synchronization inserted. This should both
be faster and more reliable.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/configure b/configure
index 2b46ab836711c8dee292e4d3a030eb577f194f93..d34c000656bf7f6970d6a2eb917b8715761bbc69 100755 (executable)
--- a/configure
+++ b/configure
@@ -343,6 +343,8 @@ CYGWIN*)
   # Flags below are still necessary mostly for MinGW.
   socklen_t="yes"
   sfaa="yes"
   # Flags below are still necessary mostly for MinGW.
   socklen_t="yes"
   sfaa="yes"

+  sync_sync="yes"
+  cmp_swap="yes"

   rusage_thread="yes"
   fdatasync="yes"
   clock_gettime="yes" # clock_monotonic probe has dependency on this
   rusage_thread="yes"
   fdatasync="yes"
   clock_gettime="yes" # clock_monotonic probe has dependency on this


@@ -706,6 +708,44 @@ if compile_prog "" "" "__sync_fetch_and_add()" ; then
 fi
 print_config "__sync_fetch_and_add" "$sfaa"
 
 fi
 print_config "__sync_fetch_and_add" "$sfaa"
 

+##########################################
+# __sync_synchronize() test
+if test "$sync_sync" != "yes" ; then
+  sync_sync="no"
+fi
+cat > $TMPC << EOF
+#include <inttypes.h>
+
+int main(int argc, char **argv)
+{
+  __sync_synchronize();
+  return 0;
+}
+EOF
+if compile_prog "" "" "__sync_synchronize()" ; then
+    sync_sync="yes"
+fi
+print_config "__sync_synchronize" "$sync_sync"
+
+##########################################
+# __sync_val_compare_and_swap() test
+if test "$cmp_swap" != "yes" ; then
+  cmp_swap="no"
+fi
+cat > $TMPC << EOF
+#include <inttypes.h>
+
+int main(int argc, char **argv)
+{
+  int x = 0;
+  return __sync_val_compare_and_swap(&x, 1, 2);
+}
+EOF
+if compile_prog "" "" "__sync_val_compare_and_swap()" ; then
+    cmp_swap="yes"
+fi
+print_config "__sync_val_compare_and_swap" "$cmp_swap"
+

 ##########################################
 # libverbs probe
 if test "$libverbs" != "yes" ; then
 ##########################################
 # libverbs probe
 if test "$libverbs" != "yes" ; then


@@ -2108,6 +2148,12 @@ fi
 if test "$sfaa" = "yes" ; then
   output_sym "CONFIG_SFAA"
 fi
 if test "$sfaa" = "yes" ; then
   output_sym "CONFIG_SFAA"
 fi

+if test "$sync_sync" = "yes" ; then
+  output_sym "CONFIG_SYNC_SYNC"
+fi
+if test "$cmp_swap" = "yes" ; then
+  output_sym "CONFIG_CMP_SWAP"
+fi

 if test "$libverbs" = "yes" -a "$rdmacm" = "yes" ; then
   output_sym "CONFIG_RDMA"
 fi
 if test "$libverbs" = "yes" -a "$rdmacm" = "yes" ; then
   output_sym "CONFIG_RDMA"
 fi

diff --git a/gettime.c b/gettime.c
index 1cbef84b0d765f224309214067fa92f1048c7fed..c256a96c16c8f26253a253c82d23d23f4a5d914c 100644 (file)
--- a/gettime.c
+++ b/gettime.c
@@ -548,7 +548,7 @@ uint64_t time_since_now(const struct timespec *s)
 }
 
 #if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \
 }
 
 #if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \

-    defined(CONFIG_SFAA)
+    defined(CONFIG_SYNC_SYNC) && defined(CONFIG_CMP_SWAP)

 
 #define CLOCK_ENTRIES_DEBUG    100000
 #define CLOCK_ENTRIES_TEST     1000
 
 #define CLOCK_ENTRIES_DEBUG    100000
 #define CLOCK_ENTRIES_TEST     1000


@@ -570,9 +570,10 @@ struct clock_thread {
        struct clock_entry *entries;
 };
 
        struct clock_entry *entries;
 };
 

-static inline uint32_t atomic32_inc_return(uint32_t *seq)
+static inline uint32_t atomic32_compare_and_swap(uint32_t *ptr, uint32_t old,
+                                                uint32_t new)

 {
 {

-       return 1 + __sync_fetch_and_add(seq, 1);
+       return __sync_val_compare_and_swap(ptr, old, new);

 }
 
 static void *clock_thread_fn(void *data)
 }
 
 static void *clock_thread_fn(void *data)


@@ -580,7 +581,6 @@ static void *clock_thread_fn(void *data)
        struct clock_thread *t = data;
        struct clock_entry *c;
        os_cpu_mask_t cpu_mask;
        struct clock_thread *t = data;
        struct clock_entry *c;
        os_cpu_mask_t cpu_mask;

-       uint32_t last_seq;

        unsigned long long first;
        int i;
 
        unsigned long long first;
        int i;
 


@@ -604,7 +604,6 @@ static void *clock_thread_fn(void *data)
        pthread_mutex_unlock(&t->started);
 
        first = get_cpu_clock();
        pthread_mutex_unlock(&t->started);
 
        first = get_cpu_clock();

-       last_seq = 0;

        c = &t->entries[0];
        for (i = 0; i < t->nr_entries; i++, c++) {
                uint32_t seq;
        c = &t->entries[0];
        for (i = 0; i < t->nr_entries; i++, c++) {
                uint32_t seq;


@@ -612,11 +611,15 @@ static void *clock_thread_fn(void *data)
 
                c->cpu = t->cpu;
                do {
 
                c->cpu = t->cpu;
                do {

-                       seq = atomic32_inc_return(t->seq);
-                       if (seq < last_seq)
+                       seq = *t->seq;
+                       if (seq == UINT_MAX)

                                break;
                                break;

+                       __sync_synchronize();

                        tsc = get_cpu_clock();
                        tsc = get_cpu_clock();

-               } while (seq != *t->seq);
+               } while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1));
+
+               if (seq == UINT_MAX)
+                       break;

 
                c->seq = seq;
                c->tsc = tsc;
 
                c->seq = seq;
                c->tsc = tsc;


@@ -634,7 +637,7 @@ static void *clock_thread_fn(void *data)
         * The most common platform clock breakage is returning zero
         * indefinitely. Check for that and return failure.
         */
         * The most common platform clock breakage is returning zero
         * indefinitely. Check for that and return failure.
         */

-       if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
+       if (i > 1 && !t->entries[i - 1].tsc && !t->entries[0].tsc)

                goto err;
 
        fio_cpuset_exit(&cpu_mask);
                goto err;
 
        fio_cpuset_exit(&cpu_mask);