syslet: add proper read barrier between user_tail and completion read

Also fixup a bug with ring indexing, it needs to use the real ring size
mask, not the io depth.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/arch/arch-alpha.h b/arch/arch-alpha.h
index b899494fec7b41b359b75a2a66a39567cafb8578..a476891aecf4cbe4268a4c617e41c4aa33526905 100644 (file)
--- a/arch/arch-alpha.h
+++ b/arch/arch-alpha.h
@@ -14,5 +14,6 @@
 
 #define nop            do { } while (0)
 #define fio_ffz(v)     generic_ffz((v))
 
 #define nop            do { } while (0)
 #define fio_ffz(v)     generic_ffz((v))

+#define read_barrier() __asm__ __volatile__("mb": : :"memory")

 
 #endif
 
 #endif

diff --git a/arch/arch-ia64.h b/arch/arch-ia64.h
index 7baccca509feaf0ba168cea082275117151a10d4..3268acc51ea02fe1528e1f2477ff3bc42186d5e1 100644 (file)
--- a/arch/arch-ia64.h
+++ b/arch/arch-ia64.h
@@ -18,7 +18,8 @@
 #define __NR_sys_vmsplice      1302
 #endif
 
 #define __NR_sys_vmsplice      1302
 #endif
 

-#define nop    asm volatile ("hint @pause" ::: "memory");
+#define nop            asm volatile ("hint @pause" ::: "memory");
+#define read_barrier() asm volatile ("mf" ::: "memory")

 
 #define ia64_popcnt(x)                                                 \
 ({                                                                     \
 
 #define ia64_popcnt(x)                                                 \
 ({                                                                     \

diff --git a/arch/arch-ppc.h b/arch/arch-ppc.h
index 0a23c01a56d2c07325930c433e50c6dded20d949..978313196d4c885690d47a7a6a67542b523b6d60 100644 (file)
--- a/arch/arch-ppc.h
+++ b/arch/arch-ppc.h
@@ -20,6 +20,14 @@
 
 #define nop    do { } while (0)
 
 
 #define nop    do { } while (0)
 

+#ifdef __powerpc64__
+#define read_barrier() \
+       __asm__ __volatile__ ("lwsync" : : : "memory")
+#else
+#define read_barrier() \
+       __asm__ __volatile__ ("sync" : : : "memory")
+#endif
+

 static inline int __ilog2(unsigned long bitmask)
 {
        int lz;
 static inline int __ilog2(unsigned long bitmask)
 {
        int lz;

diff --git a/arch/arch-s390.h b/arch/arch-s390.h
index ed5d462d1d969b7e2db4cb994a88115813b24b44..5e13155951111876c6878b8a707c037970ece2ea 100644 (file)
--- a/arch/arch-s390.h
+++ b/arch/arch-s390.h
@@ -14,5 +14,6 @@
 
 #define nop            asm volatile ("diag 0,0,68" : : : "memory")
 #define fio_ffz(v)     generic_ffz((v))
 
 #define nop            asm volatile ("diag 0,0,68" : : : "memory")
 #define fio_ffz(v)     generic_ffz((v))

+#define read_barrier() asm volatile("bcr 15,0" : : : "memory")

 
 #endif
 
 #endif

diff --git a/arch/arch-x86.h b/arch/arch-x86.h
index 7bb8ffb9849bcda446a7bcfa89c81d3a451cd121..343e51ab5b0853a5ea94f3f1f3a6f5e4033ad922 100644 (file)
--- a/arch/arch-x86.h
+++ b/arch/arch-x86.h
@@ -29,7 +29,8 @@
 
 #define FIO_HAVE_SYSLET
 
 
 #define FIO_HAVE_SYSLET
 

-#define nop    __asm__ __volatile__("rep;nop": : :"memory")
+#define nop            __asm__ __volatile__("rep;nop": : :"memory")
+#define read_barrier() asm volatile ("": : :"memory")

 
 static inline unsigned long fio_ffz(unsigned long bitmask)
 {
 
 static inline unsigned long fio_ffz(unsigned long bitmask)
 {

diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h
index 4f75addc1be14fd35d46e9631a57b50978bb6023..979c3206185f87334488d60179ff9918e13642cf 100644 (file)
--- a/arch/arch-x86_64.h
+++ b/arch/arch-x86_64.h
@@ -29,7 +29,8 @@
 
 #define FIO_HAVE_SYSLET
 
 
 #define FIO_HAVE_SYSLET
 

-#define nop    __asm__ __volatile__("rep;nop": : :"memory")
+#define nop            __asm__ __volatile__("rep;nop": : :"memory")
+#define read_barrier() asm volatile("lfence":::"memory")

 
 static inline unsigned long fio_ffz(unsigned long bitmask)
 {
 
 static inline unsigned long fio_ffz(unsigned long bitmask)
 {

diff --git a/engines/syslet-rw.c b/engines/syslet-rw.c
index e3be4859503bc9e897c1f7bd29e118f07d0d3734..49bbc712c86a013765e33d0504f758a232667df5 100644 (file)
--- a/engines/syslet-rw.c
+++ b/engines/syslet-rw.c
@@ -32,6 +32,7 @@ struct syslet_data {
        unsigned int nr_events;
        
        struct syslet_ring *ring;
        unsigned int nr_events;
        
        struct syslet_ring *ring;

+       unsigned int ring_mask;

        void *stack;
 };
 
        void *stack;
 };
 


@@ -46,10 +47,13 @@ static void fio_syslet_add_event(struct thread_data *td, struct io_u *io_u)
 static void fio_syslet_add_events(struct thread_data *td, unsigned int nr)
 {
        struct syslet_data *sd = td->io_ops->data;
 static void fio_syslet_add_events(struct thread_data *td, unsigned int nr)
 {
        struct syslet_data *sd = td->io_ops->data;

-       unsigned int i;
+       unsigned int i, uidx;
+
+       uidx = sd->ring->user_tail;
+       read_barrier();

 
        for (i = 0; i < nr; i++) {
 
        for (i = 0; i < nr; i++) {

-               unsigned int idx = (i + sd->ring->user_tail) % td->o.iodepth;
+               unsigned int idx = (i + uidx) & sd->ring_mask;

                struct syslet_completion *comp = &sd->ring->comp[idx];
                struct io_u *io_u = (struct io_u *) (long) comp->caller_data;
                long ret;
                struct syslet_completion *comp = &sd->ring->comp[idx];
                struct io_u *io_u = (struct io_u *) (long) comp->caller_data;
                long ret;


@@ -262,6 +266,7 @@ static int fio_syslet_init(struct thread_data *td)
                goto err_mem;
 
        sd->ring = ring;
                goto err_mem;
 
        sd->ring = ring;

+       sd->ring_mask = ring_nr - 1;

        sd->stack = stack;
 
        memset(sd->ring, 0, ring_size);
        sd->stack = stack;
 
        memset(sd->ring, 0, ring_size);