Introduce enum fio_q_status

[fio.git] / engines / rdma.c

diff --git a/engines/rdma.c b/engines/rdma.c
index fbe8434406b72e37fd88cf20ca46114b0b001914..67c5aa7e23eb1f65b47ea5ad4b93886b630f2e82 100644 (file)
--- a/engines/rdma.c
+++ b/engines/rdma.c
@@ -30,7 +30,7 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <netdb.h>
-#include <sys/poll.h>
+#include <poll.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/time.h>
@@ -44,7 +44,6 @@
 #include "../optgroup.h"
 
 #include <rdma/rdma_cma.h>
-#include <infiniband/arch.h>
 
 #define FIO_RDMA_MAX_IO_DEPTH    512
 
@@ -60,6 +59,7 @@ struct rdmaio_options {
        struct thread_data *td;
        unsigned int port;
        enum rdma_io_mode verb;
+       char *bindname;
 };
 
 static int str_hostname_cb(void *data, const char *input)
@@ -82,6 +82,16 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_RDMA,
        },
+       {
+               .name   = "bindname",
+               .lname  = "rdma engine bindname",
+               .type   = FIO_OPT_STR_STORE,
+               .off1   = offsetof(struct rdmaio_options, bindname),
+               .help   = "Bind for RDMA IO engine",
+               .def    = "",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_RDMA,
+       },
        {
                .name   = "port",
                .lname  = "rdma engine port",
@@ -216,7 +226,7 @@ static int client_recv(struct thread_data *td, struct ibv_wc *wc)
                rd->rmt_nr = ntohl(rd->recv_buf.nr);
 
                for (i = 0; i < rd->rmt_nr; i++) {
-                       rd->rmt_us[i].buf = ntohll(rd->recv_buf.rmt_us[i].buf);
+                       rd->rmt_us[i].buf = be64_to_cpu(rd->recv_buf.rmt_us[i].buf);
                        rd->rmt_us[i].rkey = ntohl(rd->recv_buf.rmt_us[i].rkey);
                        rd->rmt_us[i].size = ntohl(rd->recv_buf.rmt_us[i].size);
 
@@ -781,7 +791,8 @@ static int fio_rdmaio_recv(struct thread_data *td, struct io_u **io_us,
        return i;
 }
 
-static int fio_rdmaio_queue(struct thread_data *td, struct io_u *io_u)
+static enum fio_q_status
+fio_rdmaio_queue(struct thread_data *td, struct io_u *io_u)
 {
        struct rdmaio_data *rd = td->io_ops_data;
 
@@ -802,7 +813,7 @@ static void fio_rdmaio_queued(struct thread_data *td, struct io_u **io_us,
                              unsigned int nr)
 {
        struct rdmaio_data *rd = td->io_ops_data;
-       struct timeval now;
+       struct timespec now;
        unsigned int i;
 
        if (!fio_fill_issue_time(td))
@@ -881,7 +892,7 @@ static int fio_rdmaio_connect(struct thread_data *td, struct fio_file *f)
        rd->send_buf.nr = htonl(td->o.iodepth);
 
        if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) {
-               log_err("fio: ibv_post_send fail: %m");
+               log_err("fio: ibv_post_send fail: %m\n");
                return 1;
        }
 
@@ -932,7 +943,7 @@ static int fio_rdmaio_accept(struct thread_data *td, struct fio_file *f)
        ret = rdma_poll_wait(td, IBV_WC_RECV) < 0;
 
        if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) {
-               log_err("fio: ibv_post_send fail: %m");
+               log_err("fio: ibv_post_send fail: %m\n");
                return 1;
        }
 
@@ -965,7 +976,7 @@ static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
                                     || (rd->rdma_protocol ==
                                         FIO_RDMA_MEM_READ))) {
                if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) {
-                       log_err("fio: ibv_post_send fail: %m");
+                       log_err("fio: ibv_post_send fail: %m\n");
                        return 1;
                }
 
@@ -1005,30 +1016,53 @@ static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
        return 0;
 }
 
+static int aton(struct thread_data *td, const char *host,
+                    struct sockaddr_in *addr)
+{
+       if (inet_aton(host, &addr->sin_addr) != 1) {
+               struct hostent *hent;
+
+               hent = gethostbyname(host);
+               if (!hent) {
+                       td_verror(td, errno, "gethostbyname");
+                       return 1;
+               }
+
+               memcpy(&addr->sin_addr, hent->h_addr, 4);
+       }
+       return 0;
+}
+
 static int fio_rdmaio_setup_connect(struct thread_data *td, const char *host,
                                    unsigned short port)
 {
        struct rdmaio_data *rd = td->io_ops_data;
+       struct rdmaio_options *o = td->eo;
+       struct sockaddr_storage addrb;
        struct ibv_recv_wr *bad_wr;
        int err;
 
        rd->addr.sin_family = AF_INET;
        rd->addr.sin_port = htons(port);
 
-       if (inet_aton(host, &rd->addr.sin_addr) != 1) {
-               struct hostent *hent;
+       err = aton(td, host, &rd->addr);
+       if (err)
+               return err;
 
-               hent = gethostbyname(host);
-               if (!hent) {
-                       td_verror(td, errno, "gethostbyname");
-                       return 1;
-               }
+       /* resolve route */
+       if (strcmp(o->bindname, "") != 0) {
+               addrb.ss_family = AF_INET;
+               err = aton(td, o->bindname, (struct sockaddr_in *)&addrb);
+               if (err)
+                       return err;
+               err = rdma_resolve_addr(rd->cm_id, (struct sockaddr *)&addrb,
+                                       (struct sockaddr *)&rd->addr, 2000);
 
-               memcpy(&rd->addr.sin_addr, hent->h_addr, 4);
+       } else {
+               err = rdma_resolve_addr(rd->cm_id, NULL,
+                                       (struct sockaddr *)&rd->addr, 2000);
        }
 
-       /* resolve route */
-       err = rdma_resolve_addr(rd->cm_id, NULL, (struct sockaddr *)&rd->addr, 2000);
        if (err != 0) {
                log_err("fio: rdma_resolve_addr: %d\n", err);
                return 1;
@@ -1073,15 +1107,20 @@ static int fio_rdmaio_setup_connect(struct thread_data *td, const char *host,
 static int fio_rdmaio_setup_listen(struct thread_data *td, short port)
 {
        struct rdmaio_data *rd = td->io_ops_data;
+       struct rdmaio_options *o = td->eo;
        struct ibv_recv_wr *bad_wr;
        int state = td->runstate;
 
        td_set_runstate(td, TD_SETTING_UP);
 
        rd->addr.sin_family = AF_INET;
-       rd->addr.sin_addr.s_addr = htonl(INADDR_ANY);
        rd->addr.sin_port = htons(port);
 
+       if (strcmp(o->bindname, "") == 0)
+               rd->addr.sin_addr.s_addr = htonl(INADDR_ANY);
+       else
+               rd->addr.sin_addr.s_addr = htonl(*o->bindname);
+
        /* rdma_listen */
        if (rdma_bind_addr(rd->cm_id, (struct sockaddr *)&rd->addr) != 0) {
                log_err("fio: rdma_bind_addr fail: %m\n");
@@ -1156,7 +1195,8 @@ static int compat_options(struct thread_data *td)
 {
        // The original RDMA engine had an ugly / seperator
        // on the filename for it's options. This function
-       // retains backwards compatibility with it.100
+       // retains backwards compatibility with it. Note we do not
+       // support setting the bindname option is this legacy mode.
 
        struct rdmaio_options *o = td->eo;
        char *modep, *portp;
@@ -1300,7 +1340,7 @@ static int fio_rdmaio_init(struct thread_data *td)
                }
 
                rd->send_buf.rmt_us[i].buf =
-                   htonll((uint64_t) (unsigned long)io_u->buf);
+                   cpu_to_be64((uint64_t) (unsigned long)io_u->buf);
                rd->send_buf.rmt_us[i].rkey = htonl(io_u->mr->rkey);
                rd->send_buf.rmt_us[i].size = htonl(max_bs);