fio: fix aio trim completion latencies

[fio.git] / gettime.c

diff --git a/gettime.c b/gettime.c
index 1cbef84b0d765f224309214067fa92f1048c7fed..272a3e6244527c27d59a06ff99fb4871424a063b 100644 (file)
--- a/gettime.c
+++ b/gettime.c
@@ -2,15 +2,9 @@
  * Clock functions
  */
 
-#include <unistd.h>
 #include <math.h>
-#include <sys/time.h>
-#include <time.h>
 
 #include "fio.h"
-#include "smalloc.h"
-
-#include "hash.h"
 #include "os/os.h"
 
 #if defined(ARCH_HAVE_CPU_CLOCK)
@@ -243,16 +237,15 @@ static unsigned long get_cycles_per_msec(void)
        c_s = get_cpu_clock();
        do {
                __fio_gettime(&e);
+               c_e = get_cpu_clock();
 
-               elapsed = utime_since(&s, &e);
-               if (elapsed >= 1280) {
-                       c_e = get_cpu_clock();
+               elapsed = ntime_since(&s, &e);
+               if (elapsed >= 1280000)
                        break;
-               }
        } while (1);
 
        fio_clock_source = old_cs;
-       return (c_e - c_s) * 1000 / elapsed;
+       return (c_e - c_s) * 1000000 / elapsed;
 }
 
 #define NR_TIME_ITERS  50
@@ -305,10 +298,10 @@ static int calibrate_cpu_clock(void)
 
        avg /= samples;
        cycles_per_msec = avg;
-       dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
-       dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f\n",
+       dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f, N=%d\n",
                        (unsigned long long) minc,
-                       (unsigned long long) maxc, mean, S);
+                       (unsigned long long) maxc, mean, S, NR_TIME_ITERS);
+       dprint(FD_TIME, "trimmed mean=%llu, N=%d\n", (unsigned long long) avg, samples);
 
        max_ticks = MAX_CLOCK_SEC * cycles_per_msec * 1000ULL;
        max_mult = ULLONG_MAX / max_ticks;
@@ -379,7 +372,7 @@ static int calibrate_cpu_clock(void)
 #endif // ARCH_HAVE_CPU_CLOCK
 
 #ifndef CONFIG_TLS_THREAD
-void fio_local_clock_init(int is_thread)
+void fio_local_clock_init(void)
 {
        struct tv_valid *t;
 
@@ -395,7 +388,7 @@ static void kill_tv_tls_key(void *data)
        free(data);
 }
 #else
-void fio_local_clock_init(int is_thread)
+void fio_local_clock_init(void)
 {
 }
 #endif
@@ -548,7 +541,7 @@ uint64_t time_since_now(const struct timespec *s)
 }
 
 #if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK)  && \
-    defined(CONFIG_SFAA)
+    defined(CONFIG_SYNC_SYNC) && defined(CONFIG_CMP_SWAP)
 
 #define CLOCK_ENTRIES_DEBUG    100000
 #define CLOCK_ENTRIES_TEST     1000
@@ -563,16 +556,16 @@ struct clock_thread {
        pthread_t thread;
        int cpu;
        int debug;
-       pthread_mutex_t lock;
-       pthread_mutex_t started;
+       struct fio_sem lock;
        unsigned long nr_entries;
        uint32_t *seq;
        struct clock_entry *entries;
 };
 
-static inline uint32_t atomic32_inc_return(uint32_t *seq)
+static inline uint32_t atomic32_compare_and_swap(uint32_t *ptr, uint32_t old,
+                                                uint32_t new)
 {
-       return 1 + __sync_fetch_and_add(seq, 1);
+       return __sync_val_compare_and_swap(ptr, old, new);
 }
 
 static void *clock_thread_fn(void *data)
@@ -580,7 +573,6 @@ static void *clock_thread_fn(void *data)
        struct clock_thread *t = data;
        struct clock_entry *c;
        os_cpu_mask_t cpu_mask;
-       uint32_t last_seq;
        unsigned long long first;
        int i;
 
@@ -600,11 +592,9 @@ static void *clock_thread_fn(void *data)
                goto err;
        }
 
-       pthread_mutex_lock(&t->lock);
-       pthread_mutex_unlock(&t->started);
+       fio_sem_down(&t->lock);
 
        first = get_cpu_clock();
-       last_seq = 0;
        c = &t->entries[0];
        for (i = 0; i < t->nr_entries; i++, c++) {
                uint32_t seq;
@@ -612,11 +602,15 @@ static void *clock_thread_fn(void *data)
 
                c->cpu = t->cpu;
                do {
-                       seq = atomic32_inc_return(t->seq);
-                       if (seq < last_seq)
+                       seq = *t->seq;
+                       if (seq == UINT_MAX)
                                break;
+                       __sync_synchronize();
                        tsc = get_cpu_clock();
-               } while (seq != *t->seq);
+               } while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1));
+
+               if (seq == UINT_MAX)
+                       break;
 
                c->seq = seq;
                c->tsc = tsc;
@@ -634,7 +628,7 @@ static void *clock_thread_fn(void *data)
         * The most common platform clock breakage is returning zero
         * indefinitely. Check for that and return failure.
         */
-       if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
+       if (i > 1 && !t->entries[i - 1].tsc && !t->entries[0].tsc)
                goto err;
 
        fio_cpuset_exit(&cpu_mask);
@@ -699,9 +693,7 @@ int fio_monotonic_clocktest(int debug)
                t->seq = &seq;
                t->nr_entries = nr_entries;
                t->entries = &entries[i * nr_entries];
-               pthread_mutex_init(&t->lock, NULL);
-               pthread_mutex_init(&t->started, NULL);
-               pthread_mutex_lock(&t->lock);
+               __fio_sem_init(&t->lock, FIO_SEM_LOCKED);
                if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
                        failed++;
                        nr_cpus = i;
@@ -712,13 +704,7 @@ int fio_monotonic_clocktest(int debug)
        for (i = 0; i < nr_cpus; i++) {
                struct clock_thread *t = &cthreads[i];
 
-               pthread_mutex_lock(&t->started);
-       }
-
-       for (i = 0; i < nr_cpus; i++) {
-               struct clock_thread *t = &cthreads[i];
-
-               pthread_mutex_unlock(&t->lock);
+               fio_sem_up(&t->lock);
        }
 
        for (i = 0; i < nr_cpus; i++) {
@@ -728,6 +714,7 @@ int fio_monotonic_clocktest(int debug)
                pthread_join(t->thread, &ret);
                if (ret)
                        failed++;
+               __fio_sem_remove(&t->lock);
        }
        free(cthreads);