Update libhdfs engine documention and options

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/HOWTO b/HOWTO
index d7283535db0d8ddff7908422185fe5cdb9480b76..a0b89c8071d15b45aaef71222611625e38022178 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -694,7 +694,21 @@ ioengine=str       Defines how the job issues io to the file. The following
                                having to go through FUSE. This ioengine
                                defines engine specific options.
 
                                having to go through FUSE. This ioengine
                                defines engine specific options.
 

-                       hdfs    Read and write through Hadoop (HDFS).
+                       libhdfs Read and write through Hadoop (HDFS).
+                               The 'filename' option is used to specify host,
+                               port of the hdfs name-node to connect. This
+                               engine interprets offsets a little
+                               differently. In HDFS, files once created
+                               cannot be modified. So random writes are not
+                               possible. To imitate this, libhdfs engine
+                               expects bunch of small files to be created
+                               over HDFS, and engine will randomly pick a
+                               file out of those files based on the offset
+                               generated by fio backend. (see the example
+                               job file to create such files, use rw=write
+                               option). Please note, you might want to set
+                               necessary environment variables to work with
+                               hdfs/libhdfs properly.

 
                        external Prefix to specify loading an external
                                IO engine object file. Append the engine
 
                        external Prefix to specify loading an external
                                IO engine object file. Append the engine

diff --git a/examples/libhdfs.fio b/examples/libhdfs.fio
new file mode 100644 (file)
index 0000000..d5c0ba6
--- /dev/null
+++ b/examples/libhdfs.fio
@@ -0,0 +1,8 @@
+[global]
+runtime=300
+
+[hdfs]
+filename=dfs-perftest-base.dfs-perftest-base,9000
+ioengine=libhdfs
+rw=read
+bs=256k

diff --git a/fio.1 b/fio.1
index b5ff3ccbc4633dd0cf92ad42a59efe660d64e848..c61948bb52814a2a6bac248c5db0063406225a48 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -613,8 +613,16 @@ Using Glusterfs libgfapi async interface to direct access to Glusterfs volumes w
 having to go through FUSE. This ioengine defines engine specific
 options.
 .TP
 having to go through FUSE. This ioengine defines engine specific
 options.
 .TP

-.B hdfs
-Read and write through Hadoop (HDFS)
+.B libhdfs
+Read and write through Hadoop (HDFS).  The \fBfilename\fR option is used to
+specify host,port of the hdfs name-node to connect. This engine interprets
+offsets a little differently. In HDFS, files once created cannot be modified.
+So random writes are not possible. To imitate this, libhdfs engine expects
+bunch of small files to be created over HDFS, and engine will randomly pick a
+file out of those files based on the offset generated by fio backend. (see the
+example job file to create such files, use rw=write option). Please note, you
+might want to set necessary environment variables to work with hdfs/libhdfs
+properly.

 .RE
 .P
 .RE
 .RE
 .P
 .RE

diff --git a/options.c b/options.c
index 484efc1a2ebe8dd438dceff5a2a1307c8eaf7c0b..3acfdc86f197a0db37df2e91a3feae477ba25d91 100644 (file)
--- a/options.c
+++ b/options.c
@@ -672,7 +672,7 @@ static int str_numa_mpol_cb(void *data, char *input)
                }
                td->o.numa_memnodes = strdup(nodelist);
                numa_free_nodemask(verify_bitmask);
                }
                td->o.numa_memnodes = strdup(nodelist);
                numa_free_nodemask(verify_bitmask);

-                
+

                break;
        case MPOL_LOCAL:
        case MPOL_DEFAULT:
                break;
        case MPOL_LOCAL:
        case MPOL_DEFAULT:


@@ -1542,7 +1542,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                          },
 #endif
 #ifdef CONFIG_LIBHDFS
                          },
 #endif
 #ifdef CONFIG_LIBHDFS

-                         { .ival = "hdfs",
+                         { .ival = "libhdfs",

                            .help = "Hadoop Distributed Filesystem (HDFS) engine"
                          },
 #endif
                            .help = "Hadoop Distributed Filesystem (HDFS) engine"
                          },
 #endif