fio: add libiscsi engine
authorKyle Zhang <kyle@smartx.com>
Wed, 10 Apr 2019 11:37:41 +0000 (19:37 +0800)
committerKyle Zhang <kyle@smartx.com>
Sun, 21 Apr 2019 02:37:48 +0000 (10:37 +0800)
Adding a new engine that access iscsi lun with libiscsi.

You could find example fio configuration in examples/libiscsi.fio .

Signed-off-by: Kyle Zhang <kyle@smartx.com>
HOWTO
Makefile
configure
engines/libiscsi.c [new file with mode: 0644]
examples/libiscsi.fio [new file with mode: 0644]
fio.1
optgroup.h

diff --git a/HOWTO b/HOWTO
index 468772d..4fd4da1 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -1991,6 +1991,8 @@ I/O engine
                        Asynchronous read and write using DDN's Infinite Memory Engine (IME).
                        This engine will try to stack as much IOs as possible by creating
                        requests for IME. FIO will then decide when to commit these requests.
+               **libiscsi**
+                       Read and write iscsi lun with libiscsi.
 
 I/O engine specific parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
index fd138dd..d7e5fca 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -59,6 +59,12 @@ ifdef CONFIG_LIBHDFS
   SOURCE += engines/libhdfs.c
 endif
 
+ifdef CONFIG_LIBISCSI
+  CFLAGS += $(LIBISCSI_CFLAGS)
+  LIBS += $(LIBISCSI_LIBS)
+  SOURCE += engines/libiscsi.c
+endif
+
 ifdef CONFIG_64BIT
   CFLAGS += -DBITS_PER_LONG=64
 endif
index 3c882f0..c7a7c0a 100755 (executable)
--- a/configure
+++ b/configure
@@ -148,6 +148,7 @@ disable_lex=""
 disable_pmem="no"
 disable_native="no"
 march_set="no"
+libiscsi="no"
 prefix=/usr/local
 
 # parse options
@@ -204,6 +205,8 @@ for opt do
   ;;
   --with-ime=*) ime_path="$optarg"
   ;;
+  --enable-libiscsi) libiscsi="yes"
+  ;;
   --help)
     show_help="yes"
     ;;
@@ -239,6 +242,7 @@ if test "$show_help" = "yes" ; then
   echo "--enable-cuda           Enable GPUDirect RDMA support"
   echo "--disable-native        Don't build for native host"
   echo "--with-ime=             Install path for DDN's Infinite Memory Engine"
+  echo "--enable-libiscsi       Enable iscsi support"
   exit $exit_val
 fi
 
@@ -1970,6 +1974,22 @@ if compile_prog "-I${ime_path}/include" "-L${ime_path}/lib -lim_client" "libime"
 fi
 print_config "DDN's Infinite Memory Engine" "$libime"
 
+##########################################
+# Check if we have required environment variables configured for libiscsi
+if test "$libiscsi" = "yes" ; then
+  if $(pkg-config --atleast-version=1.9.0 libiscsi); then
+    libiscsi="yes"
+    libiscsi_cflags=$(pkg-config --cflags libiscsi)
+    libiscsi_libs=$(pkg-config --libs libiscsi)
+  else
+    if test "$libiscsi" = "yes" ; then
+      echo "libiscsi" "Install libiscsi >= 1.9.0"
+    fi
+    libiscsi="no"
+  fi
+fi
+print_config "iscsi engine" "$libiscsi"
+
 ##########################################
 # Check if we have lex/yacc available
 yacc="no"
@@ -2543,7 +2563,7 @@ if test "$libhdfs" = "yes" ; then
   echo "JAVA_HOME=$JAVA_HOME" >> $config_host_mak
   echo "FIO_LIBHDFS_INCLUDE=$FIO_LIBHDFS_INCLUDE" >> $config_host_mak
   echo "FIO_LIBHDFS_LIB=$FIO_LIBHDFS_LIB" >> $config_host_mak
- fi
+fi
 if test "$mtd" = "yes" ; then
   output_sym "CONFIG_MTD"
 fi
@@ -2627,6 +2647,12 @@ fi
 if test "$thp" = "yes" ; then
   output_sym "CONFIG_HAVE_THP"
 fi
+if test "$libiscsi" = "yes" ; then
+  output_sym "CONFIG_LIBISCSI"
+  echo "CONFIG_LIBISCSI=m" >> $config_host_mak
+  echo "LIBISCSI_CFLAGS=$libiscsi_cflags" >> $config_host_mak
+  echo "LIBISCSI_LIBS=$libiscsi_libs" >> $config_host_mak
+fi
 
 echo "LIBS+=$LIBS" >> $config_host_mak
 echo "GFIO_LIBS+=$GFIO_LIBS" >> $config_host_mak
diff --git a/engines/libiscsi.c b/engines/libiscsi.c
new file mode 100644 (file)
index 0000000..e4eb0ba
--- /dev/null
@@ -0,0 +1,407 @@
+/*
+ * libiscsi engine
+ *
+ * this engine read/write iscsi lun with libiscsi.
+ */
+
+
+#include "../fio.h"
+#include "../optgroup.h"
+
+#include <stdlib.h>
+#include <iscsi/iscsi.h>
+#include <iscsi/scsi-lowlevel.h>
+#include <poll.h>
+
+struct iscsi_lun;
+struct iscsi_info;
+
+struct iscsi_task {
+       struct scsi_task        *scsi_task;
+       struct iscsi_lun        *iscsi_lun;
+       struct io_u             *io_u;
+};
+
+struct iscsi_lun {
+       struct iscsi_info       *iscsi_info;
+       struct iscsi_context    *iscsi;
+       struct iscsi_url        *url;
+       int                      block_size;
+       uint64_t                 num_blocks;
+};
+
+struct iscsi_info {
+       struct iscsi_lun        **luns;
+       int                       nr_luns;
+       struct pollfd            *pfds;
+       struct iscsi_task       **complete_events;
+       int                       nr_events;
+};
+
+struct iscsi_options {
+       void    *pad;
+       char    *initiator;
+};
+
+static struct fio_option options[] = {
+       {
+               .name     = "initiator",
+               .lname    = "initiator",
+               .type     = FIO_OPT_STR_STORE,
+               .off1     = offsetof(struct iscsi_options, initiator),
+               .def      = "iqn.2019-04.org.fio:fio",
+               .help     = "initiator name",
+               .category = FIO_OPT_C_ENGINE,
+               .group    = FIO_OPT_G_ISCSI,
+       },
+
+       {
+               .name = NULL,
+       },
+};
+
+static int fio_iscsi_setup_lun(struct iscsi_info *iscsi_info,
+                              char *initiator, struct fio_file *f, int i)
+{
+       struct iscsi_lun                *iscsi_lun  = NULL;
+       struct scsi_task                *task       = NULL;
+       struct scsi_readcapacity16      *rc16       = NULL;
+       int                              ret        = 0;
+
+       iscsi_lun = malloc(sizeof(struct iscsi_lun));
+       memset(iscsi_lun, 0, sizeof(struct iscsi_lun));
+
+       iscsi_lun->iscsi_info = iscsi_info;
+
+       iscsi_lun->url = iscsi_parse_full_url(NULL, f->file_name);
+       if (iscsi_lun->url == NULL) {
+               log_err("iscsi: failed to parse url: %s\n", f->file_name);
+               ret = EINVAL;
+               goto out;
+       }
+
+       iscsi_lun->iscsi = iscsi_create_context(initiator);
+       if (iscsi_lun->iscsi == NULL) {
+               log_err("iscsi: failed to create iscsi context.\n");
+               ret = 1;
+               goto out;
+       }
+
+       if (iscsi_set_targetname(iscsi_lun->iscsi, iscsi_lun->url->target)) {
+               log_err("iscsi: failed to set target name.\n");
+               ret = EINVAL;
+               goto out;
+       }
+
+       if (iscsi_set_session_type(iscsi_lun->iscsi, ISCSI_SESSION_NORMAL) != 0) {
+               log_err("iscsi: failed to set session type.\n");
+               ret = EINVAL;
+               goto out;
+       }
+
+       if (iscsi_set_header_digest(iscsi_lun->iscsi,
+                                   ISCSI_HEADER_DIGEST_NONE_CRC32C) != 0) {
+               log_err("iscsi: failed to set header digest.\n");
+               ret = EINVAL;
+               goto out;
+       }
+
+       if (iscsi_full_connect_sync(iscsi_lun->iscsi,
+                                   iscsi_lun->url->portal,
+                                   iscsi_lun->url->lun)) {
+               log_err("sicsi: failed to connect to LUN : %s\n",
+                       iscsi_get_error(iscsi_lun->iscsi));
+               ret = EINVAL;
+               goto out;
+       }
+
+       task = iscsi_readcapacity16_sync(iscsi_lun->iscsi, iscsi_lun->url->lun);
+       if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+               log_err("iscsi: failed to send readcapacity command\n");
+               ret = EINVAL;
+               goto out;
+       }
+
+       rc16 = scsi_datain_unmarshall(task);
+       if (rc16 == NULL) {
+               log_err("iscsi: failed to unmarshal readcapacity16 data.\n");
+               ret = EINVAL;
+               goto out;
+       }
+
+       iscsi_lun->block_size = rc16->block_length;
+       iscsi_lun->num_blocks = rc16->returned_lba + 1;
+
+       scsi_free_scsi_task(task);
+       task = NULL;
+
+       f->real_file_size = iscsi_lun->num_blocks * iscsi_lun->block_size;
+       f->engine_data    = iscsi_lun;
+
+       iscsi_info->luns[i]    = iscsi_lun;
+       iscsi_info->pfds[i].fd = iscsi_get_fd(iscsi_lun->iscsi);
+
+out:
+       if (task) {
+               scsi_free_scsi_task(task);
+       }
+
+       if (ret && iscsi_lun) {
+               if (iscsi_lun->iscsi != NULL) {
+                       if (iscsi_is_logged_in(iscsi_lun->iscsi)) {
+                               iscsi_logout_sync(iscsi_lun->iscsi);
+                       }
+                       iscsi_destroy_context(iscsi_lun->iscsi);
+               }
+               free(iscsi_lun);
+       }
+
+       return ret;
+}
+
+static int fio_iscsi_setup(struct thread_data *td)
+{
+       struct iscsi_options    *options    = td->eo;
+       struct iscsi_info       *iscsi_info = NULL;
+       int                      ret        = 0;
+       struct fio_file         *f;
+       int                      i;
+
+       iscsi_info          = malloc(sizeof(struct iscsi_info));
+       iscsi_info->nr_luns = td->o.nr_files;
+       iscsi_info->luns    = calloc(iscsi_info->nr_luns, sizeof(struct iscsi_lun*));
+       iscsi_info->pfds    = calloc(iscsi_info->nr_luns, sizeof(struct pollfd));
+
+       iscsi_info->nr_events       = 0;
+       iscsi_info->complete_events = calloc(td->o.iodepth, sizeof(struct iscsi_task*));
+
+       td->io_ops_data = iscsi_info;
+
+       for_each_file(td, f, i) {
+               ret = fio_iscsi_setup_lun(iscsi_info, options->initiator, f, i);
+               if (ret < 0) break;
+       }
+
+       return ret;
+}
+
+static int fio_iscsi_init(struct thread_data *td) {
+       return 0;
+}
+
+static void fio_iscsi_cleanup_lun(struct iscsi_lun *iscsi_lun) {
+       if (iscsi_lun->iscsi != NULL) {
+               if (iscsi_is_logged_in(iscsi_lun->iscsi)) {
+                       iscsi_logout_sync(iscsi_lun->iscsi);
+               }
+               iscsi_destroy_context(iscsi_lun->iscsi);
+       }
+       free(iscsi_lun);
+}
+
+static void fio_iscsi_cleanup(struct thread_data *td)
+{
+       struct iscsi_info *iscsi_info = td->io_ops_data;
+
+       for (int i = 0; i < iscsi_info->nr_luns; i++) {
+               if (iscsi_info->luns[i]) {
+                       fio_iscsi_cleanup_lun(iscsi_info->luns[i]);
+                       iscsi_info->luns[i] = NULL;
+               }
+       }
+
+       free(iscsi_info->luns);
+       free(iscsi_info->pfds);
+       free(iscsi_info->complete_events);
+       free(iscsi_info);
+}
+
+static int fio_iscsi_prep(struct thread_data *td, struct io_u *io_u)
+{
+       return 0;
+}
+
+static int fio_iscsi_open_file(struct thread_data *td, struct fio_file *f)
+{
+       return 0;
+}
+
+static int fio_iscsi_close_file(struct thread_data *td, struct fio_file *f)
+{
+       return 0;
+}
+
+static void iscsi_cb(struct iscsi_context *iscsi, int status,
+                    void *command_data, void *private_data)
+{
+       struct iscsi_task       *iscsi_task = (struct iscsi_task*)private_data;
+       struct iscsi_lun        *iscsi_lun  = iscsi_task->iscsi_lun;
+       struct iscsi_info       *iscsi_info = iscsi_lun->iscsi_info;
+       struct io_u             *io_u       = iscsi_task->io_u;
+
+       if (status == SCSI_STATUS_GOOD) {
+               io_u->error = 0;
+       } else {
+               log_err("iscsi: request failed with error %s.\n",
+                       iscsi_get_error(iscsi_lun->iscsi));
+
+               io_u->error = 1;
+               io_u->resid = io_u->xfer_buflen;
+       }
+
+       iscsi_info->complete_events[iscsi_info->nr_events] = iscsi_task;
+       iscsi_info->nr_events++;
+}
+
+static enum fio_q_status fio_iscsi_queue(struct thread_data *td,
+                                        struct io_u *io_u)
+{
+       struct iscsi_lun        *iscsi_lun  = io_u->file->engine_data;
+       struct scsi_task        *scsi_task  = NULL;
+       struct iscsi_task       *iscsi_task = malloc(sizeof(struct iscsi_task));
+       int                      ret        = -1;
+
+       if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) {
+               if (io_u->offset % iscsi_lun->block_size != 0) {
+                       log_err("iscsi: offset is not align to block size.\n");
+                       ret = -1;
+                       goto out;
+               }
+
+               if (io_u->xfer_buflen % iscsi_lun->block_size != 0) {
+                       log_err("iscsi: buflen is not align to block size.\n");
+                       ret = -1;
+                       goto out;
+               }
+       }
+
+       if (io_u->ddir == DDIR_READ) {
+               scsi_task = scsi_cdb_read16(io_u->offset / iscsi_lun->block_size,
+                                           io_u->xfer_buflen,
+                                           iscsi_lun->block_size,
+                                           0, 0, 0, 0, 0);
+               ret = scsi_task_add_data_in_buffer(scsi_task, io_u->xfer_buflen,
+                                                  io_u->xfer_buf);
+               if (ret < 0) {
+                       log_err("iscsi: failed to add data in buffer.\n");
+                       goto out;
+               }
+       } else if (io_u->ddir == DDIR_WRITE) {
+               scsi_task = scsi_cdb_write16(io_u->offset / iscsi_lun->block_size,
+                                            io_u->xfer_buflen,
+                                            iscsi_lun->block_size,
+                                            0, 0, 0, 0, 0);
+               ret = scsi_task_add_data_out_buffer(scsi_task, io_u->xfer_buflen,
+                                                   io_u->xfer_buf);
+               if (ret < 0) {
+                       log_err("iscsi: failed to add data out buffer.\n");
+                       goto out;
+               }
+       } else if (ddir_sync(io_u->ddir)) {
+               scsi_task = scsi_cdb_synchronizecache16(
+                       0, iscsi_lun->num_blocks * iscsi_lun->block_size, 0, 0);
+       } else {
+               log_err("iscsi: invalid I/O operation: %d\n", io_u->ddir);
+               ret = EINVAL;
+               goto out;
+       }
+
+       iscsi_task->scsi_task = scsi_task;
+       iscsi_task->iscsi_lun = iscsi_lun;
+       iscsi_task->io_u      = io_u;
+
+       ret = iscsi_scsi_command_async(iscsi_lun->iscsi, iscsi_lun->url->lun,
+                                      scsi_task, iscsi_cb, NULL, iscsi_task);
+       if (ret < 0) {
+               log_err("iscsi: failed to send scsi command.\n");
+               goto out;
+       }
+
+       return FIO_Q_QUEUED;
+
+out:
+       if (iscsi_task) {
+               free(iscsi_task);
+       }
+
+       if (scsi_task) {
+               scsi_free_scsi_task(scsi_task);
+       }
+
+       if (ret) {
+               io_u->error = ret;
+       }
+       return FIO_Q_COMPLETED;
+}
+
+static int fio_iscsi_getevents(struct thread_data *td, unsigned int min,
+                              unsigned int max, const struct timespec *t)
+{
+       struct iscsi_info       *iscsi_info = td->io_ops_data;
+       int                      ret        = 0;
+
+       iscsi_info->nr_events = 0;
+
+       while (iscsi_info->nr_events < min) {
+               for (int i = 0; i < iscsi_info->nr_luns; i++) {
+                       int events = iscsi_which_events(iscsi_info->luns[i]->iscsi);
+                       iscsi_info->pfds[i].events = events;
+               }
+
+               ret = poll(iscsi_info->pfds, iscsi_info->nr_luns, -1);
+               if (ret < 0) {
+                       log_err("iscsi: failed to poll events: %s.\n",
+                               strerror(errno));
+                       break;
+               }
+
+               for (int i = 0; i < iscsi_info->nr_luns; i++) {
+                       ret = iscsi_service(iscsi_info->luns[i]->iscsi,
+                                           iscsi_info->pfds[i].revents);
+                       assert(ret >= 0);
+               }
+       }
+
+       return ret < 0 ? ret : iscsi_info->nr_events;
+}
+
+static struct io_u *fio_iscsi_event(struct thread_data *td, int event)
+{
+       struct iscsi_info       *iscsi_info = (struct iscsi_info*)td->io_ops_data;
+       struct iscsi_task       *iscsi_task = iscsi_info->complete_events[event];
+       struct io_u             *io_u       = iscsi_task->io_u;
+
+       iscsi_info->complete_events[event] = NULL;
+
+       scsi_free_scsi_task(iscsi_task->scsi_task);
+       free(iscsi_task);
+
+       return io_u;
+}
+
+static struct ioengine_ops ioengine_iscsi = {
+       .name               = "libiscsi",
+       .version            = FIO_IOOPS_VERSION,
+       .flags              = FIO_SYNCIO | FIO_DISKLESSIO | FIO_NODISKUTIL,
+       .setup              = fio_iscsi_setup,
+       .init               = fio_iscsi_init,
+       .prep               = fio_iscsi_prep,
+       .queue              = fio_iscsi_queue,
+       .getevents          = fio_iscsi_getevents,
+       .event              = fio_iscsi_event,
+       .cleanup            = fio_iscsi_cleanup,
+       .open_file          = fio_iscsi_open_file,
+       .close_file         = fio_iscsi_close_file,
+       .option_struct_size = sizeof(struct iscsi_options),
+       .options            = options,
+};
+
+static void fio_init fio_iscsi_register(void)
+{
+       register_ioengine(&ioengine_iscsi);
+}
+
+static void fio_exit fio_iscsi_unregister(void)
+{
+       unregister_ioengine(&ioengine_iscsi);
+}
diff --git a/examples/libiscsi.fio b/examples/libiscsi.fio
new file mode 100644 (file)
index 0000000..565604d
--- /dev/null
@@ -0,0 +1,3 @@
+[iscsi]
+ioengine=libiscsi
+filename=iscsi\://127.0.0.1/iqn.2016-02.com.fio\:system\:fio/1
diff --git a/fio.1 b/fio.1
index ed49268..2708b50 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -1751,6 +1751,9 @@ are "contiguous" and the IO depth is not exceeded) before issuing a call to IME.
 Asynchronous read and write using DDN's Infinite Memory Engine (IME). This
 engine will try to stack as much IOs as possible by creating requests for IME.
 FIO will then decide when to commit these requests.
+.TP
+.B libiscsi
+Read and write iscsi lun with libiscsi.
 .SS "I/O engine specific parameters"
 In addition, there are some parameters which are only valid when a specific
 \fBioengine\fR is in use. These are used identically to normal parameters,
index bf1bb03..483addd 100644 (file)
@@ -63,6 +63,7 @@ enum opt_category_group {
        __FIO_OPT_G_SG,
        __FIO_OPT_G_MMAP,
        __FIO_OPT_G_NR,
+       __FIO_OPT_G_ISCSI,
 
        FIO_OPT_G_RATE          = (1ULL << __FIO_OPT_G_RATE),
        FIO_OPT_G_ZONE          = (1ULL << __FIO_OPT_G_ZONE),
@@ -100,6 +101,7 @@ enum opt_category_group {
        FIO_OPT_G_SG            = (1ULL << __FIO_OPT_G_SG),
        FIO_OPT_G_MMAP          = (1ULL << __FIO_OPT_G_MMAP),
        FIO_OPT_G_INVALID       = (1ULL << __FIO_OPT_G_NR),
+       FIO_OPT_G_ISCSI         = (1ULL << __FIO_OPT_G_ISCSI),
 };
 
 extern const struct opt_group *opt_group_from_mask(uint64_t *mask);