rdma: use private random state

[fio.git] / engines / rdma.c

diff --git a/engines/rdma.c b/engines/rdma.c
index fa94865f6284c7e6ff9117e41237937bb387b37c..87b061a2e61788563fff7ac05d69e06b1b8e3f29 100644 (file)
--- a/engines/rdma.c
+++ b/engines/rdma.c
@@ -7,14 +7,25 @@
  *
  * This I/O engine is disabled by default. To enable it, execute:
  *
- * $ export EXTFLAGS="-DFIO_HAVE_RDMA"
- * $ export EXTLIBS="-libverbs -lrdmacm"
+ * $ export EXTFLAGS+=" -DFIO_HAVE_RDMA "
+ * $ export EXTLIBS+=" -libverbs -lrdmacm "
  *
  * before running make. You will need the Linux RDMA software as well, either
  * from your Linux distributor or directly from openfabrics.org:
  *
  * http://www.openfabrics.org/downloads/OFED/
  *
+ * Exchanging steps of RDMA ioengine control messages:
+ *     1. client side sends test mode (RDMA_WRITE/RDMA_READ/SEND)
+ *        to server side.
+ *     2. server side parses test mode, and sends back confirmation
+ *        to client side. In RDMA WRITE/READ test, this confirmation
+ *        includes memory information, such as rkey, address.
+ *     3. client side initiates test loop.
+ *     4. In RDMA WRITE/READ test, client side sends a completion
+ *        notification to server side. Server side updates its
+ *        td->done as true.
+ *
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -41,7 +52,7 @@
 #include <rdma/rdma_cma.h>
 #include <infiniband/arch.h>
 
-#define FIO_RDMA_MAX_IO_DEPTH    128
+#define FIO_RDMA_MAX_IO_DEPTH    512
 
 enum rdma_io_mode {
        FIO_RDMA_UNKNOWN = 0,
@@ -108,6 +119,8 @@ struct rdmaio_data {
        int io_u_flight_nr;
        struct io_u **io_us_completed;
        int io_u_completed_nr;
+
+       struct frand_state rand_state;
 };
 
 static int client_recv(struct thread_data *td, struct ibv_wc *wc)
@@ -205,7 +218,7 @@ static int cq_event_handler(struct thread_data *td, enum ibv_wc_opcode opcode)
                                }
                        }
                        if (i == rd->io_u_flight_nr)
-                               log_err("fio: recv wr %ld not found\n",
+                               log_err("fio: recv wr %" PRId64 " not found\n",
                                        wc.wr_id);
                        else {
                                /* put the last one into middle of the list */
@@ -234,7 +247,7 @@ static int cq_event_handler(struct thread_data *td, enum ibv_wc_opcode opcode)
                                }
                        }
                        if (i == rd->io_u_flight_nr)
-                               log_err("fio: send wr %ld not found\n",
+                               log_err("fio: send wr %" PRId64 " not found\n",
                                        wc.wr_id);
                        else {
                                /* put the last one into middle of the list */
@@ -587,9 +600,12 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
 {
        struct rdmaio_data *rd = td->io_ops->data;
        struct ibv_send_wr *bad_wr;
+#if 0
        enum ibv_wc_opcode comp_opcode;
        comp_opcode = IBV_WC_RDMA_WRITE;
-       int i, index;
+#endif
+       int i;
+       long index;
        struct rdma_io_u_data *r_io_u_d;
 
        r_io_u_d = NULL;
@@ -600,7 +616,10 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
                case FIO_RDMA_MEM_WRITE:
                        /* compose work request */
                        r_io_u_d = io_us[i]->engine_data;
-                       index = rand() % rd->rmt_nr;
+                       if (td->o.use_os_rand)
+                               index = os_random_long(&td->random_state) % rd->rmt_nr;
+                       else
+                               index = __rand(&rd->rand_state) % rd->rmt_nr;
                        r_io_u_d->sq_wr.opcode = IBV_WR_RDMA_WRITE;
                        r_io_u_d->sq_wr.wr.rdma.rkey = rd->rmt_us[index].rkey;
                        r_io_u_d->sq_wr.wr.rdma.remote_addr = \
@@ -610,7 +629,10 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
                case FIO_RDMA_MEM_READ:
                        /* compose work request */
                        r_io_u_d = io_us[i]->engine_data;
-                       index = rand() % rd->rmt_nr;
+                       if (td->o.use_os_rand)
+                               index = os_random_long(&td->random_state) % rd->rmt_nr;
+                       else
+                               index = __rand(&rd->rand_state) % rd->rmt_nr;
                        r_io_u_d->sq_wr.opcode = IBV_WR_RDMA_READ;
                        r_io_u_d->sq_wr.wr.rdma.rkey = rd->rmt_us[index].rkey;
                        r_io_u_d->sq_wr.wr.rdma.remote_addr = \
@@ -672,7 +694,8 @@ static int fio_rdmaio_recv(struct thread_data *td, struct io_u **io_us,
                rdma_poll_wait(td, IBV_WC_RECV);
 
                dprint(FD_IO, "fio: recv FINISH message\n");
-               exit(0);
+               td->done = 1;
+               return 0;
        }
 
        return i;
@@ -787,6 +810,16 @@ static int fio_rdmaio_connect(struct thread_data *td, struct fio_file *f)
        /* wait for remote MR info from server side */
        rdma_poll_wait(td, IBV_WC_RECV);
 
+       /* In SEND/RECV test, it's a good practice to setup the iodepth of
+        * of the RECV side deeper than that of the SEND side to
+        * avoid RNR (receiver not ready) error. The
+        * SEND side may send so many unsolicited message before 
+        * RECV side commits sufficient recv buffers into recv queue.
+        * This may lead to RNR error. Here, SEND side pauses for a while
+        * during which RECV side commits sufficient recv buffers.
+        */
+       usleep(500000);
+
        return 0;
 }
 
@@ -869,8 +902,8 @@ static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
         return 1;
     }*/
 
-       ibv_destroy_qp(rd->qp);
        ibv_destroy_cq(rd->cq);
+       ibv_destroy_qp(rd->qp);
 
        if (rd->is_client == 1)
                rdma_destroy_id(rd->cm_id);
@@ -1018,8 +1051,9 @@ static int fio_rdmaio_init(struct thread_data *td)
        /* soft limit */
        if ((rl.rlim_cur != RLIM_INFINITY)
            && (rl.rlim_cur < td->orig_buffer_size)) {
-               log_err("fio: soft RLIMIT_MEMLOCK is: %ld\n", rl.rlim_cur);
-               log_err("fio: total block size is:    %ld\n",
+               log_err("fio: soft RLIMIT_MEMLOCK is: %" PRId64 "\n",
+                       rl.rlim_cur);
+               log_err("fio: total block size is:    %zd\n",
                        td->orig_buffer_size);
                /* try to set larger RLIMIT_MEMLOCK */
                rl.rlim_cur = rl.rlim_max;
@@ -1178,6 +1212,7 @@ static int fio_rdmaio_setup(struct thread_data *td)
                rd = malloc(sizeof(*rd));;
 
                memset(rd, 0, sizeof(*rd));
+               init_rand_seed(&rd->rand_state, GOLDEN_RATIO_PRIME);
                td->io_ops->data = rd;
        }
 
@@ -1225,8 +1260,8 @@ static int fio_rdmaio_init(struct thread_data fio_unused * td)
        log_err("     make sure OFED is installed,\n");
        log_err("     $ ofed_info\n");
        log_err("     then try to make fio as follows:\n");
-       log_err("     $ export EXTFLAGS=\"-DFIO_HAVE_RDMA\"\n");
-       log_err("     $ export EXTLIBS=\"-libverbs -lrdmacm\"\n");
+       log_err("     $ export EXTFLAGS+=\" -DFIO_HAVE_RDMA \"\n");
+       log_err("     $ export EXTLIBS+=\" -libverbs -lrdmacm \"\n");
        log_err("     $ make clean && make\n");
        return 1;
 }