ioengines: implement dircreate, dirstat, dirdelete engines to fileoperations.c

Similar to file operation, directory operation performance is an
important benchmark to file system in practice.

* dircreate engine measures directories create performance
* dirstat engine measures directories lookup performance
* dirdelete engine measures directories delete performance

Signed-off-by: friendy-su <friendy.su@sony.com>
Signed-off-by: Vincent Fu <vincent.fu@samsung.com>

diff --git a/HOWTO.rst b/HOWTO.rst
index 4c8ac3314e711d020223dfac201d59c4007e1f01..fb067fe52844cb4fb3c1ad3bf192807b8007e5dc 100644 (file)
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2192,6 +2192,21 @@ I/O engine
                        and 'nrfiles', so that the files will be created.
                        This engine is to measure file delete.
 
+               **dircreate**
+                       Simply create the directories and do no I/O to them.  You still need to
+                       set  `filesize` so that all the accounting still occurs, but no
+                       actual I/O will be done other than creating the directories.
+
+               **dirstat**
+                       Simply do stat() and do no I/O to the directories. You need to set 'filesize'
+                       and 'nrfiles', so that directories will be created.
+                       This engine is to measure directory lookup and meta data access.
+
+               **dirdelete**
+                       Simply delete the directories by rmdir() and do no I/O to them. You need to set 'filesize'
+                       and 'nrfiles', so that the directories will be created.
+                       This engine is to measure directory delete.
+
                **libpmem**
                        Read and write using mmap I/O to a file on a filesystem
                        mounted with DAX on a persistent memory device through the PMDK

diff --git a/engines/fileoperations.c b/engines/fileoperations.c
index 1db60da1814cd9fabf69cd1627b1fcdb5e930393..a3e92e6c1efd957b6fe1f7a4e084dda977bba680 100644 (file)
--- a/engines/fileoperations.c
+++ b/engines/fileoperations.c
@@ -1,8 +1,8 @@
 /*
- * fileoperations engine
+ * file/directory operations engine
  *
- * IO engine that doesn't do any IO, just operates files and tracks the latency
- * of the file operation.
+ * IO engine that doesn't do any IO, just operates files/directories
+ * and tracks the latency of the operation.
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -15,9 +15,15 @@
 #include "../optgroup.h"
 #include "../oslib/statx.h"
 
+enum fio_engine {
+       UNKNOWN_OP_ENGINE = 0,
+       FILE_OP_ENGINE = 1,
+       DIR_OP_ENGINE = 2,
+};
 
 struct fc_data {
        enum fio_ddir stat_ddir;
+       enum fio_engine op_engine;
 };
 
 struct filestat_options {
@@ -61,6 +67,26 @@ static struct fio_option options[] = {
        },
 };
 
+static int setup_dirs(struct thread_data *td)
+{
+       int ret = 0;
+       int i;
+       struct fio_file *f;
+
+       for_each_file(td, f, i) {
+               dprint(FD_FILE, "setup directory %s\n", f->file_name);
+               ret = fio_mkdir(f->file_name, 0700);
+               if ((ret && errno != EEXIST)) {
+                       log_err("create directory %s failed with %d\n",
+                               f->file_name, errno);
+                       break;
+               }
+               ret = 0;
+       }
+       return ret;
+}
+
+
 
 static int open_file(struct thread_data *td, struct fio_file *f)
 {
@@ -81,7 +107,14 @@ static int open_file(struct thread_data *td, struct fio_file *f)
        if (do_lat)
                fio_gettime(&start, NULL);
 
-       f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+       if (((struct fc_data *)td->io_ops_data)->op_engine == FILE_OP_ENGINE)
+               f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+       else if (((struct fc_data *)td->io_ops_data)->op_engine == DIR_OP_ENGINE)
+               f->fd = fio_mkdir(f->file_name, S_IFDIR);
+       else {
+               log_err("fio: unknown file/directory operation engine\n");
+               return 1;
+       }
 
        if (f->fd == -1) {
                char buf[FIO_VERROR_SIZE];
@@ -195,7 +228,16 @@ static int delete_file(struct thread_data *td, struct fio_file *f)
        if (do_lat)
                fio_gettime(&start, NULL);
 
-       ret = unlink(f->file_name);
+       if (((struct fc_data *)td->io_ops_data)->op_engine == FILE_OP_ENGINE)
+               ret = unlink(f->file_name);
+       else if (((struct fc_data *)td->io_ops_data)->op_engine == DIR_OP_ENGINE)
+               ret = rmdir(f->file_name);
+       else {
+               log_err("fio: unknown file/directory operation engine\n");
+               return 1;
+       }
+
+
 
        if (ret == -1) {
                char buf[FIO_VERROR_SIZE];
@@ -250,6 +292,17 @@ static int init(struct thread_data *td)
        else if (td_write(td))
                data->stat_ddir = DDIR_WRITE;
 
+       data->op_engine = UNKNOWN_OP_ENGINE;
+
+       if (!strncmp(td->o.ioengine, "file", 4)) {
+               data->op_engine = FILE_OP_ENGINE;
+               dprint(FD_FILE, "Operate engine type: file\n");
+       }
+       if (!strncmp(td->o.ioengine, "dir", 3)) {
+               data->op_engine = DIR_OP_ENGINE;
+               dprint(FD_FILE, "Operate engine type: directory\n");
+       }
+
        td->io_ops_data = data;
        return 0;
 }
@@ -261,6 +314,12 @@ static void cleanup(struct thread_data *td)
        free(data);
 }
 
+static int remove_dir(struct thread_data *td, struct fio_file *f)
+{
+       dprint(FD_FILE, "remove directory %s\n", f->file_name);
+       return rmdir(f->file_name);
+}
+
 static struct ioengine_ops ioengine_filecreate = {
        .name           = "filecreate",
        .version        = FIO_IOOPS_VERSION,
@@ -302,12 +361,61 @@ static struct ioengine_ops ioengine_filedelete = {
                                FIO_NOSTATS | FIO_NOFILEHASH,
 };
 
+static struct ioengine_ops ioengine_dircreate = {
+       .name           = "dircreate",
+       .version        = FIO_IOOPS_VERSION,
+       .init           = init,
+       .cleanup        = cleanup,
+       .queue          = queue_io,
+       .get_file_size  = get_file_size,
+       .open_file      = open_file,
+       .close_file     = generic_close_file,
+       .unlink_file    = remove_dir,
+       .flags          = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+                               FIO_NOSTATS | FIO_NOFILEHASH,
+};
+
+static struct ioengine_ops ioengine_dirstat = {
+       .name           = "dirstat",
+       .version        = FIO_IOOPS_VERSION,
+       .setup          = setup_dirs,
+       .init           = init,
+       .cleanup        = cleanup,
+       .queue          = queue_io,
+       .invalidate     = invalidate_do_nothing,
+       .get_file_size  = generic_get_file_size,
+       .open_file      = stat_file,
+       .unlink_file    = remove_dir,
+       .flags          =  FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+                               FIO_NOSTATS | FIO_NOFILEHASH,
+       .options        = options,
+       .option_struct_size = sizeof(struct filestat_options),
+};
+
+static struct ioengine_ops ioengine_dirdelete = {
+       .name           = "dirdelete",
+       .version        = FIO_IOOPS_VERSION,
+       .setup          = setup_dirs,
+       .init           = init,
+       .invalidate     = invalidate_do_nothing,
+       .cleanup        = cleanup,
+       .queue          = queue_io,
+       .get_file_size  = get_file_size,
+       .open_file      = delete_file,
+       .unlink_file    = remove_dir,
+       .flags          = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+                               FIO_NOSTATS | FIO_NOFILEHASH,
+};
+
 
 static void fio_init fio_fileoperations_register(void)
 {
        register_ioengine(&ioengine_filecreate);
        register_ioengine(&ioengine_filestat);
        register_ioengine(&ioengine_filedelete);
+       register_ioengine(&ioengine_dircreate);
+       register_ioengine(&ioengine_dirstat);
+       register_ioengine(&ioengine_dirdelete);
 }
 
 static void fio_exit fio_fileoperations_unregister(void)
@@ -315,4 +423,7 @@ static void fio_exit fio_fileoperations_unregister(void)
        unregister_ioengine(&ioengine_filecreate);
        unregister_ioengine(&ioengine_filestat);
        unregister_ioengine(&ioengine_filedelete);
+       unregister_ioengine(&ioengine_dircreate);
+       unregister_ioengine(&ioengine_dirstat);
+       unregister_ioengine(&ioengine_dirdelete);
 }

diff --git a/examples/dircreate-ioengine.fio b/examples/dircreate-ioengine.fio
new file mode 100644 (file)
index 0000000..c89d9e4
--- /dev/null
+++ b/examples/dircreate-ioengine.fio
@@ -0,0 +1,25 @@
+# Example dircreate job
+#
+# create_on_open is needed so that the open happens during the run and not the
+# setup.
+#
+# openfiles needs to be set so that you do not exceed the maximum allowed open
+# files.
+#
+# filesize needs to be set to a non zero value so fio will actually run, but the
+# IO will not really be done and the write latency numbers will only reflect the
+# open times.
+[global]
+create_on_open=1
+nrfiles=30
+ioengine=dircreate
+fallocate=none
+filesize=4k
+openfiles=1
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]

diff --git a/examples/dirdelete-ioengine.fio b/examples/dirdelete-ioengine.fio
new file mode 100644 (file)
index 0000000..4e5b1e2
--- /dev/null
+++ b/examples/dirdelete-ioengine.fio
@@ -0,0 +1,18 @@
+# Example dirdelete job
+
+# 'filedelete' engine only do 'rmdir(dirname)'.
+# 'filesize' must be set, then directories will be created at setup stage.
+# 'unlink' is better set to 0, since the directory is deleted in measurement.
+# the options disabled completion latency output such as 'disable_clat' and 'gtod_reduce' must not set.
+[global]
+ioengine=dirdelete
+filesize=4k
+nrfiles=200
+unlink=0
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]

diff --git a/examples/dirstat-ioengine.fio b/examples/dirstat-ioengine.fio
new file mode 100644 (file)
index 0000000..1322dd2
--- /dev/null
+++ b/examples/dirstat-ioengine.fio
@@ -0,0 +1,18 @@
+# Example dirstat job
+
+# 'dirstat' engine only do 'stat(dirname)', file will not be open().
+# 'filesize' must be set, then files will be created at setup stage.
+
+[global]
+ioengine=dirstat
+numjobs=10
+filesize=4k
+nrfiles=5
+thread
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]

diff --git a/fio.1 b/fio.1
index 09c6b621c6ce875e8e82c43b9156bde616630751..63375c625726eb68f9bbafa8ca1bccce3ae68ba4 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -2004,6 +2004,21 @@ Simply delete files by unlink() and do no I/O to the file. You need to set 'file
 and 'nrfiles', so that files will be created.
 This engine is to measure file delete.
 .TP
+.B dircreate
+Simply create the directories and do no I/O to them.  You still need to set
+\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be
+done other than creating the directories.
+.TP
+.B dirstat
+Simply do stat() and do no I/O to the directory. You need to set 'filesize'
+and 'nrfiles', so that directories will be created.
+This engine is to measure directory lookup and meta data access.
+.TP
+.B dirdelete
+Simply delete directories by unlink() and do no I/O to the directory. You need to set 'filesize'
+and 'nrfiles', so that directories will be created.
+This engine is to measure directory delete.
+.TP
 .B libpmem
 Read and write using mmap I/O to a file on a filesystem
 mounted with DAX on a persistent memory device through the PMDK