fio: add multi directory support

This patch adds support for ':' seperated multiple directories at the
directory config statement in order to achieve an automatic distribution
of job clones (numjob) across directories.

That way people can distribute a load across these directories (usually
mount points of disks) automatically - changing numjob will be
sufficient to get all job clones evenly (optimal if dirs % numjobs = 0,
otherwise as good as possible) distributed at all times.

To avoid confused users old config Files will behave like they always
did, old fio binaries using new config files won't abort but just use
the first specified dir. If one specifies an explcit (non generated)
filename the distribution to many directories is also deactivated.

It also fixes an issue of events seeming out of order like when running
with --debug=file seeing the "..." message meaning "I created the
clones" prior to the last clone activities.  Now the clones are called
with index N-1 .. 1, zero being the base thread as before.

Signed-off-by: Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/engines/net.c b/engines/net.c
index 1dc55d5e7cb88d0b1a8ca9a60d2d24fc2f5eefdd..8b85a88604bde109f86f4cf5864ab045560f3be1 100644 (file)
--- a/engines/net.c
+++ b/engines/net.c
@@ -1194,7 +1194,7 @@ static int fio_netio_setup(struct thread_data *td)
        struct netio_data *nd;
 
        if (!td->files_index) {
-               add_file(td, td->o.filename ?: "net");
+               add_file(td, td->o.filename ?: "net", 0);
                td->o.nr_files = td->o.nr_files ?: 1;
        }
 

diff --git a/engines/rbd.c b/engines/rbd.c
index d089a41d35fa1de5ff1d1eef983caa505cd27343..39fa0ceb21c8f97c6c1ac1bcc3e83bcace424903 100644 (file)
--- a/engines/rbd.c
+++ b/engines/rbd.c
@@ -377,7 +377,7 @@ static int fio_rbd_setup(struct thread_data *td)
         * The size of the RBD is set instead of a artificial file.
         */
        if (!td->files_index) {
-               add_file(td, td->o.filename ? : "rbd");
+               add_file(td, td->o.filename ? : "rbd", 0);
                td->o.nr_files = td->o.nr_files ? : 1;
        }
        f = td->files[0];

diff --git a/file.h b/file.h
index c1d02a5bea57add8d9049aad117fb91592740459..d065a25096447e00cb218382c83e6458038d4b55 100644 (file)
--- a/file.h
+++ b/file.h
@@ -125,6 +125,11 @@ struct fio_file {
        struct disk_util *du;
 };
 
+struct file_name {
+       struct flist_head list;
+       char *filename;
+};
+
 #define FILE_FLAG_FNS(name)                                            \
 static inline void fio_file_set_##name(struct fio_file *f)             \
 {                                                                      \
@@ -162,7 +167,7 @@ extern int __must_check generic_close_file(struct thread_data *, struct fio_file
 extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *);
 extern int __must_check file_lookup_open(struct fio_file *f, int flags);
 extern int __must_check pre_read_files(struct thread_data *);
-extern int add_file(struct thread_data *, const char *);
+extern int add_file(struct thread_data *, const char *, int);
 extern int add_file_exclusive(struct thread_data *, const char *);
 extern void get_file(struct fio_file *);
 extern int __must_check put_file(struct thread_data *, struct fio_file *);
@@ -175,6 +180,7 @@ extern int init_random_map(struct thread_data *);
 extern void dup_files(struct thread_data *, struct thread_data *);
 extern int get_fileno(struct thread_data *, const char *);
 extern void free_release_files(struct thread_data *);
+extern void filesetup_mem_free(void);
 void fio_file_reset(struct thread_data *, struct fio_file *);
 int fio_files_done(struct thread_data *);
 

diff --git a/filesetup.c b/filesetup.c
index 544ecb1c53dce5f336faf3294c0acfc75ace53e3..f0e3b34fd8ae1e90424c04a37ec1cba4b71e4a38 100644 (file)
--- a/filesetup.c
+++ b/filesetup.c
@@ -11,6 +11,7 @@
 #include "fio.h"
 #include "smalloc.h"
 #include "filehash.h"
+#include "options.h"
 #include "os/os.h"
 #include "hash.h"
 #include "lib/axmap.h"
@@ -21,6 +22,8 @@
 
 static int root_warn;
 
+static FLIST_HEAD(filename_list);
+
 static inline void clear_error(struct thread_data *td)
 {
        td->error = 0;
@@ -1101,7 +1104,48 @@ static void get_file_type(struct fio_file *f)
        }
 }
 
-int add_file(struct thread_data *td, const char *fname)
+static void set_already_allocated(const char *fname) {
+       struct file_name *fn;
+
+       fn = malloc(sizeof(struct file_name));
+       fn->filename = strdup(fname);
+       flist_add_tail(&fn->list, &filename_list);
+}
+
+static int is_already_allocated(const char *fname)
+{
+       struct flist_head *entry;
+       char *filename;
+
+       if (!flist_empty(&filename_list))
+       {
+               flist_for_each(entry, &filename_list) {
+                       filename = flist_entry(entry, struct file_name, list)->filename;
+
+                       if (strcmp(filename, fname) == 0)
+                               return 1;
+               }
+       }
+
+       return 0;
+}
+
+static void free_already_allocated() {
+       struct flist_head *entry, *tmp;
+       struct file_name *fn;
+
+       if (!flist_empty(&filename_list))
+       {
+               flist_for_each_safe(entry, tmp, &filename_list) {
+                       fn = flist_entry(entry, struct file_name, list);
+                       free(fn->filename);
+                       flist_del(&fn->list);
+                       free(fn);
+               }
+       }
+}
+
+int add_file(struct thread_data *td, const char *fname, int numjob)
 {
        int cur_files = td->files_index;
        char file_name[PATH_MAX];
@@ -1110,6 +1154,15 @@ int add_file(struct thread_data *td, const char *fname)
 
        dprint(FD_FILE, "add file %s\n", fname);
 
+       if (td->o.directory)
+               len = set_name_idx(file_name, td->o.directory, numjob);
+
+       sprintf(file_name + len, "%s", fname);
+
+       /* clean cloned siblings using existing files */
+       if (numjob && is_already_allocated(file_name))
+               return 0;
+
        f = smalloc(sizeof(*f));
        if (!f) {
                log_err("fio: smalloc OOM\n");
@@ -1149,10 +1202,6 @@ int add_file(struct thread_data *td, const char *fname)
        if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO))
                f->real_file_size = -1ULL;
 
-       if (td->o.directory)
-               len = sprintf(file_name, "%s/", td->o.directory);
-
-       sprintf(file_name + len, "%s", fname);
        f->file_name = smalloc_strdup(file_name);
        if (!f->file_name) {
                log_err("fio: smalloc OOM\n");
@@ -1179,6 +1228,8 @@ int add_file(struct thread_data *td, const char *fname)
        if (f->filetype == FIO_TYPE_FILE)
                td->nr_normal_files++;
 
+       set_already_allocated(file_name);
+
        dprint(FD_FILE, "file %p \"%s\" added at %d\n", f, f->file_name,
                                                        cur_files);
 
@@ -1195,7 +1246,7 @@ int add_file_exclusive(struct thread_data *td, const char *fname)
                        return i;
        }
 
-       return add_file(td, fname);
+       return add_file(td, fname, 0);
 }
 
 void get_file(struct fio_file *f)
@@ -1304,7 +1355,7 @@ static int recurse_dir(struct thread_data *td, const char *dirname)
                }
 
                if (S_ISREG(sb.st_mode)) {
-                       add_file(td, full_path);
+                       add_file(td, full_path, 0);
                        td->o.nr_files++;
                        continue;
                }
@@ -1421,3 +1472,8 @@ int fio_files_done(struct thread_data *td)
 
        return 1;
 }
+
+/* free memory used in initialization phase only */
+void filesetup_mem_free() {
+       free_already_allocated();
+}

diff --git a/fio.1 b/fio.1
index 2320ba341fc5efdf839afb350981f185a3906327..1f3f4dd6b39af7241827fba2a918977af5ae2560 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -158,6 +158,12 @@ otherwise has no special purpose.
 .BI directory \fR=\fPstr
 Prefix filenames with this directory.  Used to place files in a location other
 than `./'.
+You can specify a number of directories by separating the names with a ':'
+character. These directories will be assigned equally distributed to job clones
+creates with \fInumjobs\fR as long as they are using generated filenames.
+If specific \fIfilename(s)\fR are set fio will use the first listed directory,
+and thereby matching the  \fIfilename\fR semantic which generates a file each
+clone if not specified, but let all clones use the same if set.
 .TP
 .BI filename \fR=\fPstr
 .B fio

diff --git a/init.c b/init.c
index 9d3c960870a89e0412094b5d5ee4ba6754462061..7f8b3178ebd26f36e47c434dbcf035a6b8ceac28 100644 (file)
--- a/init.c
+++ b/init.c
@@ -1020,10 +1020,10 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
                file_alloced = 1;
 
                if (o->nr_files == 1 && exists_and_not_file(jobname))
-                       add_file(td, jobname);
+                       add_file(td, jobname, job_add_num);
                else {
                        for (i = 0; i < o->nr_files; i++)
-                               add_file(td, make_filename(fname, o, jobname, td->thread_number, i));
+                               add_file(td, make_filename(fname, o, jobname, job_add_num, i), job_add_num);
                }
        }
 
@@ -1166,9 +1166,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
                        }
                }
 
-               job_add_num = numjobs - 1;
-
-               if (add_job(td_new, jobname, job_add_num, 1, client_type))
+               if (add_job(td_new, jobname, numjobs, 1, client_type))
                        goto err;
        }
 
@@ -2027,6 +2025,7 @@ int parse_options(int argc, char *argv[])
 
        free(ini_file);
        fio_options_free(&def_thread);
+       filesetup_mem_free();
 
        if (!thread_number) {
                if (parse_dryrun())

diff --git a/iolog.c b/iolog.c
index 5fd9416c036ec2454030b4ddf6772eab487efa30..eeaca291e21eb6503712f29a602ca5ad121a2615 100644 (file)
--- a/iolog.c
+++ b/iolog.c
@@ -324,7 +324,7 @@ static int read_iolog2(struct thread_data *td, FILE *f)
                        rw = DDIR_INVAL;
                        if (!strcmp(act, "add")) {
                                td->o.nr_files++;
-                               fileno = add_file(td, fname);
+                               fileno = add_file(td, fname, 0);
                                file_action = FIO_LOG_ADD_FILE;
                                continue;
                        } else if (!strcmp(act, "open")) {

diff --git a/options.c b/options.c
index 5b97ec413aae0ba43d8f911ccfda8dc24397553f..04fb506a6e9cfffd234e0b898eee08e600cb81c2 100644 (file)
--- a/options.c
+++ b/options.c
@@ -729,11 +729,11 @@ static int str_random_distribution_cb(void *data, const char *str)
 }
 
 /*
- * Return next file in the string. Files are separated with ':'. If the ':'
+ * Return next name in the string. Files are separated with ':'. If the ':'
  * is escaped with a '\', then that ':' is part of the filename and does not
  * indicate a new file.
  */
-static char *get_next_file_name(char **ptr)
+static char *get_next_name(char **ptr)
 {
        char *str = *ptr;
        char *p, *start;
@@ -774,6 +774,43 @@ static char *get_next_file_name(char **ptr)
        return start;
 }
 
+
+static int get_max_name_idx(char *input)
+{
+       unsigned int cur_idx;
+       char *str, *p;
+
+       p = str = strdup(input);
+       for (cur_idx = 0; ; cur_idx++)
+               if (get_next_name(&str) == NULL)
+                       break;
+
+       free(p);
+       return cur_idx;
+}
+
+/*
+ * Returns the directory at the index, indexes > entires will be
+ * assigned via modulo division of the index
+ */
+int set_name_idx(char *target, char *input, int index)
+{
+       unsigned int cur_idx;
+       int len;
+       char *fname, *str, *p;
+
+       p = str = strdup(input);
+
+       index %= get_max_name_idx(input);
+       for (cur_idx = 0; cur_idx <= index; cur_idx++)
+               fname = get_next_name(&str);
+
+       len = sprintf(target, "%s/", fname);
+       free(p);
+
+       return len;
+}
+
 static int str_filename_cb(void *data, const char *input)
 {
        struct thread_data *td = data;
@@ -787,10 +824,10 @@ static int str_filename_cb(void *data, const char *input)
        if (!td->files_index)
                td->o.nr_files = 0;
 
-       while ((fname = get_next_file_name(&str)) != NULL) {
+       while ((fname = get_next_name(&str)) != NULL) {
                if (!strlen(fname))
                        break;
-               add_file(td, fname);
+               add_file(td, fname, 0);
                td->o.nr_files++;
        }
 
@@ -798,27 +835,35 @@ static int str_filename_cb(void *data, const char *input)
        return 0;
 }
 
-static int str_directory_cb(void *data, const char fio_unused *str)
+static int str_directory_cb(void *data, const char fio_unused *unused)
 {
        struct thread_data *td = data;
        struct stat sb;
+       char *dirname, *str, *p;
+       int ret = 0;
 
        if (parse_dryrun())
                return 0;
 
-       if (lstat(td->o.directory, &sb) < 0) {
-               int ret = errno;
+       p = str = strdup(td->o.directory);
+       while ((dirname = get_next_name(&str)) != NULL) {
+               if (lstat(dirname, &sb) < 0) {
+                       ret = errno;
 
-               log_err("fio: %s is not a directory\n", td->o.directory);
-               td_verror(td, ret, "lstat");
-               return 1;
-       }
-       if (!S_ISDIR(sb.st_mode)) {
-               log_err("fio: %s is not a directory\n", td->o.directory);
-               return 1;
+                       log_err("fio: %s is not a directory\n", dirname);
+                       td_verror(td, ret, "lstat");
+                       goto out;
+               }
+               if (!S_ISDIR(sb.st_mode)) {
+                       log_err("fio: %s is not a directory\n", dirname);
+                       ret = 1;
+                       goto out;
+               }
        }
 
-       return 0;
+out:
+       free(p);
+       return ret;
 }
 
 static int str_lockfile_cb(void *data, const char fio_unused *str)

diff --git a/options.h b/options.h
index 3dc48a9788ab073a811bc29c78b26a0f65c4a4dc..de9f6109fa1b474e72a1b60df667f99f7455735d 100644 (file)
--- a/options.h
+++ b/options.h
@@ -17,6 +17,8 @@ void add_opt_posval(const char *, const char *, const char *);
 void del_opt_posval(const char *, const char *);
 struct thread_data;
 void fio_options_free(struct thread_data *);
+char *get_name_idx(char *, int);
+int set_name_idx(char *, char *, int);
 
 extern struct fio_option fio_options[FIO_MAX_OPTS];