Update for RDMA io engine's compatibility

[fio.git] / engines / rdma.c

diff --git a/engines/rdma.c b/engines/rdma.c
index 9b1830158a652f6b48399c323faa11130cd4bc03..81f167367e90df6f9e2dd439e2ddfdc41f7c08e1 100644 (file)
--- a/engines/rdma.c
+++ b/engines/rdma.c
@@ -5,12 +5,7 @@
  * Supports both RDMA memory semantics and channel semantics
  *   for the InfiniBand, RoCE and iWARP protocols.
  *
  * Supports both RDMA memory semantics and channel semantics
  *   for the InfiniBand, RoCE and iWARP protocols.
  *

- * This I/O engine is disabled by default. To enable it, execute:
- *
- * $ export EXTFLAGS+=" -DFIO_HAVE_RDMA "
- * $ export EXTLIBS+=" -libverbs -lrdmacm "
- *
- * before running make. You will need the Linux RDMA software as well, either
+ * You will need the Linux RDMA software installed, either

  * from your Linux distributor or directly from openfabrics.org:
  *
  * http://www.openfabrics.org/downloads/OFED/
  * from your Linux distributor or directly from openfabrics.org:
  *
  * http://www.openfabrics.org/downloads/OFED/


@@ -41,15 +36,12 @@
 #include <sys/time.h>
 #include <sys/resource.h>
 
 #include <sys/time.h>
 #include <sys/resource.h>
 

-#include <byteswap.h>

 #include <pthread.h>
 #include <inttypes.h>
 
 #include "../fio.h"
 #include "../hash.h"
 
 #include <pthread.h>
 #include <inttypes.h>
 
 #include "../fio.h"
 #include "../hash.h"
 

-#ifdef FIO_HAVE_RDMA
-

 #include <rdma/rdma_cma.h>
 #include <infiniband/arch.h>
 
 #include <rdma/rdma_cma.h>
 #include <infiniband/arch.h>
 


@@ -536,10 +528,10 @@ static int fio_rdmaio_getevents(struct thread_data *td, unsigned int min,
 {
        struct rdmaio_data *rd = td->io_ops->data;
        enum ibv_wc_opcode comp_opcode;
 {
        struct rdmaio_data *rd = td->io_ops->data;
        enum ibv_wc_opcode comp_opcode;

-       comp_opcode = IBV_WC_RDMA_WRITE;

        struct ibv_cq *ev_cq;
        void *ev_ctx;
        int ret, r = 0;
        struct ibv_cq *ev_cq;
        void *ev_ctx;
        int ret, r = 0;

+       comp_opcode = IBV_WC_RDMA_WRITE;

 
        switch (rd->rdma_protocol) {
        case FIO_RDMA_MEM_WRITE:
 
        switch (rd->rdma_protocol) {
        case FIO_RDMA_MEM_WRITE:


@@ -1020,26 +1012,11 @@ static int fio_rdmaio_setup_listen(struct thread_data *td, short port)
        return 0;
 }
 
        return 0;
 }
 

-static int fio_rdmaio_init(struct thread_data *td)
+static int check_set_rlimits(struct thread_data *td)

 {
 {

-       struct rdmaio_data *rd = td->io_ops->data;
-       struct flist_head *entry;
-       unsigned int max_bs;
-       unsigned int port;
-       char host[64], buf[128];
-       char *sep, *portp, *modep;
-       int ret, i = 0;
+#ifdef CONFIG_RLIMIT_MEMLOCK

        struct rlimit rl;
 
        struct rlimit rl;
 

-       if (td_rw(td)) {
-               log_err("fio: rdma connections must be read OR write\n");
-               return 1;
-       }
-       if (td_random(td)) {
-               log_err("fio: RDMA network IO can't be random\n");
-               return 1;
-       }
-

        /* check RLIMIT_MEMLOCK */
        if (getrlimit(RLIMIT_MEMLOCK, &rl) != 0) {
                log_err("fio: getrlimit fail: %d(%s)\n",
        /* check RLIMIT_MEMLOCK */
        if (getrlimit(RLIMIT_MEMLOCK, &rl) != 0) {
                log_err("fio: getrlimit fail: %d(%s)\n",


@@ -1064,6 +1041,31 @@ static int fio_rdmaio_init(struct thread_data *td)
                        return 1;
                }
        }
                        return 1;
                }
        }

+#endif
+
+       return 0;
+}
+
+static int fio_rdmaio_init(struct thread_data *td)
+{
+       struct rdmaio_data *rd = td->io_ops->data;
+       unsigned int max_bs;
+       unsigned int port;
+       char host[64], buf[128];
+       char *sep, *portp, *modep;
+       int ret, i;
+
+       if (td_rw(td)) {
+               log_err("fio: rdma connections must be read OR write\n");
+               return 1;
+       }
+       if (td_random(td)) {
+               log_err("fio: RDMA network IO can't be random\n");
+               return 1;
+       }
+
+       if (check_set_rlimits(td))
+               return 1;

 
        strcpy(buf, td->o.filename);
 
 
        strcpy(buf, td->o.filename);
 


@@ -1150,8 +1152,8 @@ static int fio_rdmaio_init(struct thread_data *td)
 
        max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
        /* register each io_u in the free list */
 
        max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
        /* register each io_u in the free list */

-       flist_for_each(entry, &td->io_u_freelist) {
-               struct io_u *io_u = flist_entry(entry, struct io_u, list);
+       for (i = 0; i < td->io_u_freelist.nr; i++) {
+               struct io_u *io_u = td->io_u_freelist.io_us[i];

 
                io_u->engine_data = malloc(sizeof(struct rdma_io_u_data));
                memset(io_u->engine_data, 0, sizeof(struct rdma_io_u_data));
 
                io_u->engine_data = malloc(sizeof(struct rdma_io_u_data));
                memset(io_u->engine_data, 0, sizeof(struct rdma_io_u_data));


@@ -1174,7 +1176,6 @@ static int fio_rdmaio_init(struct thread_data *td)
 #if 0
                log_info("fio: Send rkey %x addr %" PRIx64 " len %d to client\n", io_u->mr->rkey, io_u->buf, max_bs); */
 #endif
 #if 0
                log_info("fio: Send rkey %x addr %" PRIx64 " len %d to client\n", io_u->mr->rkey, io_u->buf, max_bs); */
 #endif

-               i++;

        }
 
        rd->send_buf.nr = htonl(i);
        }
 
        rd->send_buf.nr = htonl(i);


@@ -1224,49 +1225,6 @@ static struct ioengine_ops ioengine_rw = {
        .flags          = FIO_DISKLESSIO | FIO_UNIDIR | FIO_PIPEIO,
 };
 
        .flags          = FIO_DISKLESSIO | FIO_UNIDIR | FIO_PIPEIO,
 };
 

-#else /* FIO_HAVE_RDMA */
-
-static int fio_rdmaio_open_file(struct thread_data *td, struct fio_file *f)
-{
-       return 0;
-}
-
-static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
-{
-       return 0;
-}
-
-static int fio_rdmaio_queue(struct thread_data *td, struct io_u *io_u)
-{
-       return FIO_Q_COMPLETED;
-}
-
-static int fio_rdmaio_init(struct thread_data fio_unused * td)
-{
-       log_err("fio: rdma(librdmacm libibverbs) not available\n");
-       log_err("     You haven't compiled rdma ioengine into fio.\n");
-       log_err("     If you want to try rdma ioengine,\n");
-       log_err("     make sure OFED is installed,\n");
-       log_err("     $ ofed_info\n");
-       log_err("     then try to make fio as follows:\n");
-       log_err("     $ export EXTFLAGS+=\" -DFIO_HAVE_RDMA \"\n");
-       log_err("     $ export EXTLIBS+=\" -libverbs -lrdmacm \"\n");
-       log_err("     $ make clean && make\n");
-       return 1;
-}
-
-static struct ioengine_ops ioengine_rw = {
-       .name           = "rdma",
-       .version        = FIO_IOOPS_VERSION,
-       .init           = fio_rdmaio_init,
-       .queue          = fio_rdmaio_queue,
-       .open_file      = fio_rdmaio_open_file,
-       .close_file     = fio_rdmaio_close_file,
-       .flags          = FIO_SYNCIO | FIO_DISKLESSIO | FIO_UNIDIR | FIO_PIPEIO,
-};
-
-#endif
-

 static void fio_init fio_rdmaio_register(void)
 {
        register_ioengine(&ioengine_rw);
 static void fio_init fio_rdmaio_register(void)
 {
        register_ioengine(&ioengine_rw);