Allow explicit setting of a number of files

We currently only allow filename=foo for one file, add the possibility
to specify any number of files by seperating with a colon.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/HOWTO b/HOWTO
index f9e3fc708530c7b963d00035a749869771d6f4f2..5df2607e874a35fa131db5f36a0095af2dfe917a 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -205,7 +205,11 @@ filename=str       Fio normally makes up a filename based on the job name,
                files between threads in a job or several jobs, specify
                a filename for each of them to override the default. If
                the ioengine used is 'net', the filename is the host and
-               port to connect to in the format of =host:port.
+               port to connect to in the format of =host:port. If the
+               ioengine is file based, you can specify a number of files
+               by seperating the names with a ':' colon. So if you wanted
+               a job to open /dev/sda and /dev/sdb as the two working files,
+               you would use filename=/dev/sda:/dev/sdb
 
 rw=str         Type of io pattern. Accepted values are:
 

diff --git a/engines/cpu.c b/engines/cpu.c
index 7a084e0acb96247dddb2af6a10b62fee2ce7a769..b6f9075d365be76fe944a33101cf6eff3b195b88 100644 (file)
--- a/engines/cpu.c
+++ b/engines/cpu.c
@@ -10,7 +10,7 @@ static int fio_cpuio_queue(struct thread_data *td, struct io_u fio_unused *io_u)
 static int fio_cpuio_setup(struct thread_data fio_unused *td)
 {
        struct fio_file *f;
-       int i;
+       unsigned int i;
 
        td->total_file_size = -1;
        td->io_size = td->total_file_size;

diff --git a/engines/net.c b/engines/net.c
index 2e136f3dc104007522f67142deafdad3eee12925..c2f45e5f1a870510af6ff09d18f0ccc355e5dbf6 100644 (file)
--- a/engines/net.c
+++ b/engines/net.c
@@ -223,8 +223,9 @@ static int fio_netio_init(struct thread_data *td)
        unsigned short port;
        struct fio_file *f;
        char host[64], buf[128];
+       unsigned int i;
        char *sep;
-       int ret, i;
+       int ret;
 
        if (!td->total_file_size) {
                log_err("fio: need size= set\n");

diff --git a/engines/null.c b/engines/null.c
index 67ac4535245f40c72905dfac07df3b6c9e182b33..42a780aa17fea77d0ce44d8765e810e3b7db85b9 100644 (file)
--- a/engines/null.c
+++ b/engines/null.c
@@ -61,7 +61,7 @@ static int fio_null_queue(struct thread_data fio_unused *td, struct io_u *io_u)
 static int fio_null_setup(struct thread_data *td)
 {
        struct fio_file *f;
-       int i;
+       unsigned int i;
 
        if (!td->total_file_size) {
                log_err("fio: need size= set\n");

diff --git a/engines/sg.c b/engines/sg.c
index 3a6a2486b758df41ea749482c88efa91b2378ae4..98ecbd1c25a0ca52a003c85e75fe2090ee8a7433 100644 (file)
--- a/engines/sg.c
+++ b/engines/sg.c
@@ -69,8 +69,9 @@ static int fio_sgio_getevents(struct thread_data *td, int min, int max,
         */
        struct fio_file *f = &td->files[0];
        struct sgio_data *sd = td->io_ops->data;
-       int left = max, ret, events, i, r = 0;
+       int left = max, ret, r = 0;
        void *buf = sd->sgbuf;
+       unsigned int i, events;
 
        /*
         * Fill in the file descriptors
@@ -191,7 +192,7 @@ static int fio_sgio_doio(struct thread_data *td, struct io_u *io_u, int sync)
 {
        struct fio_file *f = io_u->file;
 
-       if (td->filetype == FIO_TYPE_BD)
+       if (f->filetype == FIO_TYPE_BD)
                return fio_sgio_ioctl_doio(td, f, io_u);
 
        return fio_sgio_rw_doio(f, io_u, sync);
@@ -347,12 +348,12 @@ static int fio_sgio_type_check(struct thread_data *td, struct fio_file *f)
        struct sgio_data *sd = td->io_ops->data;
        unsigned int bs;
 
-       if (td->filetype == FIO_TYPE_BD) {
+       if (f->filetype == FIO_TYPE_BD) {
                if (ioctl(f->fd, BLKSSZGET, &bs) < 0) {
                        td_verror(td, errno, "ioctl");
                        return 1;
                }
-       } else if (td->filetype == FIO_TYPE_CHAR) {
+       } else if (f->filetype == FIO_TYPE_CHAR) {
                int version, ret;
 
                if (ioctl(f->fd, SG_GET_VERSION_NUM, &version) < 0) {
@@ -370,7 +371,7 @@ static int fio_sgio_type_check(struct thread_data *td, struct fio_file *f)
 
        sd->bs = bs;
 
-       if (td->filetype == FIO_TYPE_BD) {
+       if (f->filetype == FIO_TYPE_BD) {
                td->io_ops->getevents = NULL;
                td->io_ops->event = NULL;
        }

diff --git a/filesetup.c b/filesetup.c
index 48b034a6b4f83d4913e94a19a72611e8d1755e80..948cecda7fb56e70de64c0c00ef9723e7564be74 100644 (file)
--- a/filesetup.c
+++ b/filesetup.c
@@ -15,7 +15,7 @@ static int file_ok(struct thread_data *td, struct fio_file *f)
 {
        struct stat st;
 
-       if (td->filetype != FIO_TYPE_FILE ||
+       if (f->filetype != FIO_TYPE_FILE ||
            (td->io_ops->flags & FIO_DISKLESSIO))
                return 0;
 
@@ -98,7 +98,8 @@ err:
 static int create_files(struct thread_data *td)
 {
        struct fio_file *f;
-       int i, err, need_create, can_extend;
+       int err, need_create, can_extend;
+       unsigned int i;
 
        for_each_file(td, f, i)
                f->file_size = td->total_file_size / td->nr_files;
@@ -111,17 +112,20 @@ static int create_files(struct thread_data *td)
                return 0;
 
        need_create = 0;
-       if (td->filetype == FIO_TYPE_FILE) {
-               for_each_file(td, f, i) {
-                       int file_there = !file_ok(td, f);
+       for_each_file(td, f, i) {
+               int file_there;
+
+               if (f->filetype != FIO_TYPE_FILE)
+                       continue;
 
-                       if (file_there && td_write(td) && !td->overwrite) {
-                               unlink(f->file_name);
-                               file_there = 0;
-                       }
+               file_there = !file_ok(td, f);
 
-                       need_create += !file_there;
+               if (file_there && td_write(td) && !td->overwrite) {
+                       unlink(f->file_name);
+                       file_there = 0;
                }
+
+               need_create += !file_there;
        }
 
        if (!need_create)
@@ -204,9 +208,9 @@ static int get_file_size(struct thread_data *td, struct fio_file *f)
 {
        int ret = 0;
 
-       if (td->filetype == FIO_TYPE_FILE)
+       if (f->filetype == FIO_TYPE_FILE)
                ret = file_size(td, f);
-       else if (td->filetype == FIO_TYPE_BD)
+       else if (f->filetype == FIO_TYPE_BD)
                ret = bdev_size(td, f);
        else
                f->real_file_size = -1;
@@ -234,11 +238,11 @@ int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
         */
        if (f->mmap)
                ret = madvise(f->mmap, f->file_size, MADV_DONTNEED);
-       else if (td->filetype == FIO_TYPE_FILE) {
+       else if (f->filetype == FIO_TYPE_FILE) {
                ret = fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_DONTNEED);
-       } else if (td->filetype == FIO_TYPE_BD) {
+       } else if (f->filetype == FIO_TYPE_BD) {
                ret = blockdev_invalidate_cache(f->fd);
-       } else if (td->filetype == FIO_TYPE_CHAR)
+       } else if (f->filetype == FIO_TYPE_CHAR)
                ret = 0;
 
        if (ret < 0) {
@@ -267,12 +271,12 @@ int generic_open_file(struct thread_data *td, struct fio_file *f)
        if (td_write(td) || td_rw(td)) {
                flags |= O_RDWR;
 
-               if (td->filetype == FIO_TYPE_FILE)
+               if (f->filetype == FIO_TYPE_FILE)
                        flags |= O_CREAT;
 
                f->fd = open(f->file_name, flags, 0600);
        } else {
-               if (td->filetype == FIO_TYPE_CHAR)
+               if (f->filetype == FIO_TYPE_CHAR)
                        flags |= O_RDWR;
                else
                        flags |= O_RDONLY;
@@ -316,7 +320,8 @@ err:
 int open_files(struct thread_data *td)
 {
        struct fio_file *f;
-       int i, err = 0;
+       unsigned int i;
+       int err = 0;
 
        for_each_file(td, f, i) {
                err = td_io_open_file(td, f);
@@ -339,7 +344,8 @@ int open_files(struct thread_data *td)
 int setup_files(struct thread_data *td)
 {
        struct fio_file *f;
-       int err, i;
+       unsigned int i;
+       int err;
 
        /*
         * if ioengine defines a setup() method, it's responsible for
@@ -385,13 +391,12 @@ int setup_files(struct thread_data *td)
 void close_files(struct thread_data *td)
 {
        struct fio_file *f;
-       int i;
+       unsigned int i;
 
        for_each_file(td, f, i) {
                if (!td->filename && f->unlink &&
-                   td->filetype == FIO_TYPE_FILE) {
+                   f->filetype == FIO_TYPE_FILE) {
                        unlink(f->file_name);
-                       free(f->file_name);
                        f->file_name = NULL;
                }
 
@@ -406,3 +411,35 @@ void close_files(struct thread_data *td)
        td->files = NULL;
        td->nr_files = 0;
 }
+
+static void get_file_type(struct thread_data *td, struct fio_file *f)
+{
+       struct stat sb;
+
+       f->filetype = FIO_TYPE_FILE;
+
+       if (!lstat(td->filename, &sb)) {
+               if (S_ISBLK(sb.st_mode))
+                       f->filetype = FIO_TYPE_BD;
+               else if (S_ISCHR(sb.st_mode))
+                       f->filetype = FIO_TYPE_CHAR;
+       }
+}
+
+void add_file(struct thread_data *td, const char *fname)
+{
+       int cur_files = td->open_files;
+       struct fio_file *f;
+
+       td->files = realloc(td->files, (cur_files + 1) * sizeof(*f));
+
+       f = &td->files[cur_files];
+       memset(f, 0, sizeof(*f));
+       f->fd = -1;
+       f->file_name = fname;
+
+       get_file_type(td, f);
+
+       td->open_files++;
+       td->nr_uniq_files = td->open_files;
+}

diff --git a/fio.c b/fio.c
index 6222a4185c7105973b4080669e34a37303a7f281..4d628b9f522383880c5fde1dd34d91ff179eedf1 100644 (file)
--- a/fio.c
+++ b/fio.c
@@ -255,7 +255,8 @@ static void do_verify(struct thread_data *td)
 {
        struct fio_file *f;
        struct io_u *io_u;
-       int ret, i, min_events;
+       int ret, min_events;
+       unsigned int i;
 
        /*
         * sync io first and invalidate cache, to make sure we really
@@ -379,7 +380,8 @@ static void do_io(struct thread_data *td)
 {
        struct timeval s;
        unsigned long usec;
-       int i, ret = 0;
+       unsigned int i;
+       int ret = 0;
 
        td_set_runstate(td, TD_RUNNING);
 
@@ -650,7 +652,8 @@ static int switch_ioscheduler(struct thread_data *td)
 static int clear_io_state(struct thread_data *td)
 {
        struct fio_file *f;
-       int i, ret;
+       unsigned int i;
+       int ret;
 
        td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0;
        td->this_io_bytes[0] = td->this_io_bytes[1] = 0;

diff --git a/fio.h b/fio.h
index 0ca23cc0e6b09186b059e8bae86b266d22a6d5de..3b902a8da940ce20346a6e1d3a13c072735767b7 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -223,6 +223,8 @@ enum fio_ioengine_flags {
  * this structure holds state information for a single file.
  */
 struct fio_file {
+       enum fio_filetype filetype;
+
        /*
         * A file may not be a file descriptor, let the io engine decide
         */
@@ -230,7 +232,7 @@ struct fio_file {
                unsigned long file_data;
                int fd;
        };
-       char *file_name;
+       const char *file_name;
        void *mmap;
        unsigned long long file_size;
        unsigned long long real_file_size;
@@ -314,7 +316,6 @@ struct thread_data {
        int thread_number;
        int groupid;
        struct thread_stat ts;
-       enum fio_filetype filetype;
        struct fio_file *files;
        unsigned int nr_files;
        unsigned int nr_open_files;
@@ -648,6 +649,7 @@ extern int __must_check open_files(struct thread_data *);
 extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *);
 extern int __must_check generic_open_file(struct thread_data *, struct fio_file *);
 extern void generic_close_file(struct thread_data *, struct fio_file *);
+extern void add_file(struct thread_data *, const char *);
 
 /*
  * ETA/status stuff
@@ -764,7 +766,7 @@ extern void close_ioengine(struct thread_data *);
 #define for_each_td(td, i)     \
        for ((i) = 0, (td) = &threads[0]; (i) < (int) thread_number; (i)++, (td)++)
 #define for_each_file(td, f, i)        \
-       for ((i) = 0, (f) = &(td)->files[0]; (i) < (int) (td)->open_files; (i)++, (f)++)
+       for ((i) = 0, (f) = &(td)->files[0]; (i) < (td)->open_files; (i)++, (f)++)
 
 #define fio_assert(td, cond)   do {    \
        if (!(cond)) {                  \

diff --git a/init.c b/init.c
index 624d2842d69b2e576487357d9d6f44e4443f650c..f3c756e960dc4c33afe8a3503236921311b1baf6 100644 (file)
--- a/init.c
+++ b/init.c
@@ -31,6 +31,8 @@ static int str_prioclass_cb(void *, unsigned int *);
 static int str_exitall_cb(void);
 static int str_cpumask_cb(void *, unsigned int *);
 static int str_fst_cb(void *, const char *);
+static int str_filename_cb(void *, const char *);
+static int str_directory_cb(void *, const char *);
 
 #define __stringify_1(x)       #x
 #define __stringify(x)         __stringify_1(x)
@@ -55,13 +57,15 @@ static struct fio_option options[] = {
                .name   = "directory",
                .type   = FIO_OPT_STR_STORE,
                .off1   = td_var_offset(directory),
+               .cb     = str_directory_cb,
                .help   = "Directory to store files in",
        },
        {
                .name   = "filename",
                .type   = FIO_OPT_STR_STORE,
                .off1   = td_var_offset(filename),
-               .help   = "Force the use of a specific file",
+               .cb     = str_filename_cb,
+               .help   = "File(s) to use for the workload",
        },
        {
                .name   = "rw",
@@ -721,12 +725,6 @@ static void fixup_options(struct thread_data *td)
        if (td->bs_unaligned && (td->odirect || td->io_ops->flags & FIO_RAWIO))
                log_err("fio: bs_unaligned may not work with raw io\n");
 
-       /*
-        * O_DIRECT and char doesn't mix, clear that flag if necessary.
-        */
-       if (td->filetype == FIO_TYPE_CHAR && td->odirect)
-               td->odirect = 0;
-
        /*
         * thinktime_spin must be less than thinktime
         */
@@ -802,11 +800,11 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
 {
        const char *ddir_str[] = { NULL, "read", "write", "rw", NULL,
                                   "randread", "randwrite", "randrw" };
-       struct stat sb;
-       int numjobs, i;
+       unsigned int i;
        struct fio_file *f;
        const char *engine;
-       int fn_given;
+       char fname[PATH_MAX];
+       int numjobs;
 
        /*
         * the def_thread is just for options, it's not a real job
@@ -829,65 +827,23 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
        if (td->odirect)
                td->io_ops->flags |= FIO_RAWIO;
 
-       fn_given = (long) td->filename;
-       if (!td->filename)
+       if (!td->filename) {
                td->filename = strdup(jobname);
 
-       td->filetype = FIO_TYPE_FILE;
-       if (!lstat(td->filename, &sb)) {
-               if (S_ISBLK(sb.st_mode))
-                       td->filetype = FIO_TYPE_BD;
-               else if (S_ISCHR(sb.st_mode))
-                       td->filetype = FIO_TYPE_CHAR;
+               for (i = 0; i < td->nr_files; i++) {
+                       sprintf(fname, "%s.%d.%d", td->filename, td->thread_number, i);
+                       add_file(td, fname);
+               }
        }
 
        fixup_options(td);
 
-       if (fn_given)
-               td->nr_uniq_files = 1;
-       else
-               td->nr_uniq_files = td->open_files;
-
-       if (td->filetype == FIO_TYPE_FILE) {
-               char tmp[PATH_MAX];
-               int len = 0;
-
-               if (td->directory && td->directory[0] != '\0') {
-                       if (lstat(td->directory, &sb) < 0) {
-                               log_err("fio: %s is not a directory\n", td->directory);
-                               td_verror(td, errno, "lstat");
-                               return 1;
-                       }
-                       if (!S_ISDIR(sb.st_mode)) {
-                               log_err("fio: %s is not a directory\n", td->directory);
-                               return 1;
-                       }
-                       len = sprintf(tmp, "%s/", td->directory);
-               }
-
-               td->files = malloc(sizeof(struct fio_file) * td->open_files);
-
-               for_each_file(td, f, i) {
-                       memset(f, 0, sizeof(*f));
-                       f->fd = -1;
-
-                       if (fn_given)
-                               sprintf(tmp + len, "%s", td->filename);
-                       else
-                               sprintf(tmp + len, "%s.%d.%d", td->filename, td->thread_number, i);
-                       f->file_name = strdup(tmp);
+       for_each_file(td, f, i) {
+               if (td->directory && f->filetype == FIO_TYPE_FILE) {
+                       sprintf(fname, "%s/%s", td->directory, f->file_name);
+                       f->file_name = strdup(fname);
                }
-       } else {
-               td->open_files = td->nr_files = 1;
-               td->files = malloc(sizeof(struct fio_file));
-               f = &td->files[0];
-
-               memset(f, 0, sizeof(*f));
-               f->fd = -1;
-               f->file_name = strdup(td->filename);
-       }
 
-       for_each_file(td, f, i) {
                f->file_size = td->total_file_size / td->nr_files;
                f->file_offset = td->start_offset;
        }
@@ -979,8 +935,9 @@ err:
 int init_random_state(struct thread_data *td)
 {
        unsigned long seeds[5];
-       int fd, num_maps, blocks, i;
+       int fd, num_maps, blocks;
        struct fio_file *f;
+       unsigned int i;
 
        if (td->io_ops->flags & FIO_DISKLESSIO)
                return 0;
@@ -1141,6 +1098,36 @@ static int str_fst_cb(void *data, const char *str)
        return 0;
 }
 
+static int str_filename_cb(void *data, const char *input)
+{
+       struct thread_data *td = data;
+       char *fname, *str;
+
+       str = strdup(input);
+       while ((fname = strsep(&str, ":")) != NULL)
+               add_file(td, fname);
+
+       return 0;
+}
+
+static int str_directory_cb(void *data, const char fio_unused *str)
+{
+       struct thread_data *td = data;
+       struct stat sb;
+
+       if (lstat(td->directory, &sb) < 0) {
+               log_err("fio: %s is not a directory\n", td->directory);
+               td_verror(td, errno, "lstat");
+               return 1;
+       }
+       if (!S_ISDIR(sb.st_mode)) {
+               log_err("fio: %s is not a directory\n", td->directory);
+               return 1;
+       }
+
+       return 0;
+}
+
 /*
  * This is our [ini] type file parser.
  */

diff --git a/parse.c b/parse.c
index 21fb3c2eeda611fd6ab1b43010ec5c36f868fbc3..e09b1bbe42998655125d23b00c3b0b7d35214489 100644 (file)
--- a/parse.c
+++ b/parse.c
@@ -257,10 +257,20 @@ static int __handle_option(struct fio_option *o, const char *ptr, void *data,
                }
                break;
        }
-       case FIO_OPT_STR_STORE:
+       case FIO_OPT_STR_STORE: {
+               fio_opt_str_fn *fn = o->cb;
+
                cp = td_var(data, o->off1);
                *cp = strdup(ptr);
+               if (fn) {
+                       ret = fn(data, ptr);
+                       if (ret) {
+                               free(*cp);
+                               *cp = NULL;
+                       }
+               }
                break;
+       }
        case FIO_OPT_RANGE: {
                char tmp[128];
                char *p1, *p2;
@@ -579,7 +589,7 @@ void options_init(struct fio_option *options)
                }
                if (!o->cb && !o->off1)
                        fprintf(stderr, "Option %s: neither cb nor offset given\n", o->name);
-               if (o->type == FIO_OPT_STR)
+               if (o->type == FIO_OPT_STR || o->type == FIO_OPT_STR_STORE)
                        continue;
                if (o->cb && (o->off1 || o->off2 || o->off3 || o->off4))
                        fprintf(stderr, "Option %s: both cb and offset given\n", o->name);