From: Jens Axboe Date: Tue, 13 Mar 2007 09:07:47 +0000 (+0100) Subject: Allow explicit setting of a number of files X-Git-Tag: fio-1.14~27 X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=af52b3455ad892322aab2791282b6bd4efdfdbf3 Allow explicit setting of a number of files We currently only allow filename=foo for one file, add the possibility to specify any number of files by seperating with a colon. Signed-off-by: Jens Axboe --- diff --git a/HOWTO b/HOWTO index f9e3fc70..5df2607e 100644 --- a/HOWTO +++ b/HOWTO @@ -205,7 +205,11 @@ filename=str Fio normally makes up a filename based on the job name, files between threads in a job or several jobs, specify a filename for each of them to override the default. If the ioengine used is 'net', the filename is the host and - port to connect to in the format of =host:port. + port to connect to in the format of =host:port. If the + ioengine is file based, you can specify a number of files + by seperating the names with a ':' colon. So if you wanted + a job to open /dev/sda and /dev/sdb as the two working files, + you would use filename=/dev/sda:/dev/sdb rw=str Type of io pattern. Accepted values are: diff --git a/engines/cpu.c b/engines/cpu.c index 7a084e0a..b6f9075d 100644 --- a/engines/cpu.c +++ b/engines/cpu.c @@ -10,7 +10,7 @@ static int fio_cpuio_queue(struct thread_data *td, struct io_u fio_unused *io_u) static int fio_cpuio_setup(struct thread_data fio_unused *td) { struct fio_file *f; - int i; + unsigned int i; td->total_file_size = -1; td->io_size = td->total_file_size; diff --git a/engines/net.c b/engines/net.c index 2e136f3d..c2f45e5f 100644 --- a/engines/net.c +++ b/engines/net.c @@ -223,8 +223,9 @@ static int fio_netio_init(struct thread_data *td) unsigned short port; struct fio_file *f; char host[64], buf[128]; + unsigned int i; char *sep; - int ret, i; + int ret; if (!td->total_file_size) { log_err("fio: need size= set\n"); diff --git a/engines/null.c b/engines/null.c index 67ac4535..42a780aa 100644 --- a/engines/null.c +++ b/engines/null.c @@ -61,7 +61,7 @@ static int fio_null_queue(struct thread_data fio_unused *td, struct io_u *io_u) static int fio_null_setup(struct thread_data *td) { struct fio_file *f; - int i; + unsigned int i; if (!td->total_file_size) { log_err("fio: need size= set\n"); diff --git a/engines/sg.c b/engines/sg.c index 3a6a2486..98ecbd1c 100644 --- a/engines/sg.c +++ b/engines/sg.c @@ -69,8 +69,9 @@ static int fio_sgio_getevents(struct thread_data *td, int min, int max, */ struct fio_file *f = &td->files[0]; struct sgio_data *sd = td->io_ops->data; - int left = max, ret, events, i, r = 0; + int left = max, ret, r = 0; void *buf = sd->sgbuf; + unsigned int i, events; /* * Fill in the file descriptors @@ -191,7 +192,7 @@ static int fio_sgio_doio(struct thread_data *td, struct io_u *io_u, int sync) { struct fio_file *f = io_u->file; - if (td->filetype == FIO_TYPE_BD) + if (f->filetype == FIO_TYPE_BD) return fio_sgio_ioctl_doio(td, f, io_u); return fio_sgio_rw_doio(f, io_u, sync); @@ -347,12 +348,12 @@ static int fio_sgio_type_check(struct thread_data *td, struct fio_file *f) struct sgio_data *sd = td->io_ops->data; unsigned int bs; - if (td->filetype == FIO_TYPE_BD) { + if (f->filetype == FIO_TYPE_BD) { if (ioctl(f->fd, BLKSSZGET, &bs) < 0) { td_verror(td, errno, "ioctl"); return 1; } - } else if (td->filetype == FIO_TYPE_CHAR) { + } else if (f->filetype == FIO_TYPE_CHAR) { int version, ret; if (ioctl(f->fd, SG_GET_VERSION_NUM, &version) < 0) { @@ -370,7 +371,7 @@ static int fio_sgio_type_check(struct thread_data *td, struct fio_file *f) sd->bs = bs; - if (td->filetype == FIO_TYPE_BD) { + if (f->filetype == FIO_TYPE_BD) { td->io_ops->getevents = NULL; td->io_ops->event = NULL; } diff --git a/filesetup.c b/filesetup.c index 48b034a6..948cecda 100644 --- a/filesetup.c +++ b/filesetup.c @@ -15,7 +15,7 @@ static int file_ok(struct thread_data *td, struct fio_file *f) { struct stat st; - if (td->filetype != FIO_TYPE_FILE || + if (f->filetype != FIO_TYPE_FILE || (td->io_ops->flags & FIO_DISKLESSIO)) return 0; @@ -98,7 +98,8 @@ err: static int create_files(struct thread_data *td) { struct fio_file *f; - int i, err, need_create, can_extend; + int err, need_create, can_extend; + unsigned int i; for_each_file(td, f, i) f->file_size = td->total_file_size / td->nr_files; @@ -111,17 +112,20 @@ static int create_files(struct thread_data *td) return 0; need_create = 0; - if (td->filetype == FIO_TYPE_FILE) { - for_each_file(td, f, i) { - int file_there = !file_ok(td, f); + for_each_file(td, f, i) { + int file_there; + + if (f->filetype != FIO_TYPE_FILE) + continue; - if (file_there && td_write(td) && !td->overwrite) { - unlink(f->file_name); - file_there = 0; - } + file_there = !file_ok(td, f); - need_create += !file_there; + if (file_there && td_write(td) && !td->overwrite) { + unlink(f->file_name); + file_there = 0; } + + need_create += !file_there; } if (!need_create) @@ -204,9 +208,9 @@ static int get_file_size(struct thread_data *td, struct fio_file *f) { int ret = 0; - if (td->filetype == FIO_TYPE_FILE) + if (f->filetype == FIO_TYPE_FILE) ret = file_size(td, f); - else if (td->filetype == FIO_TYPE_BD) + else if (f->filetype == FIO_TYPE_BD) ret = bdev_size(td, f); else f->real_file_size = -1; @@ -234,11 +238,11 @@ int file_invalidate_cache(struct thread_data *td, struct fio_file *f) */ if (f->mmap) ret = madvise(f->mmap, f->file_size, MADV_DONTNEED); - else if (td->filetype == FIO_TYPE_FILE) { + else if (f->filetype == FIO_TYPE_FILE) { ret = fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_DONTNEED); - } else if (td->filetype == FIO_TYPE_BD) { + } else if (f->filetype == FIO_TYPE_BD) { ret = blockdev_invalidate_cache(f->fd); - } else if (td->filetype == FIO_TYPE_CHAR) + } else if (f->filetype == FIO_TYPE_CHAR) ret = 0; if (ret < 0) { @@ -267,12 +271,12 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) if (td_write(td) || td_rw(td)) { flags |= O_RDWR; - if (td->filetype == FIO_TYPE_FILE) + if (f->filetype == FIO_TYPE_FILE) flags |= O_CREAT; f->fd = open(f->file_name, flags, 0600); } else { - if (td->filetype == FIO_TYPE_CHAR) + if (f->filetype == FIO_TYPE_CHAR) flags |= O_RDWR; else flags |= O_RDONLY; @@ -316,7 +320,8 @@ err: int open_files(struct thread_data *td) { struct fio_file *f; - int i, err = 0; + unsigned int i; + int err = 0; for_each_file(td, f, i) { err = td_io_open_file(td, f); @@ -339,7 +344,8 @@ int open_files(struct thread_data *td) int setup_files(struct thread_data *td) { struct fio_file *f; - int err, i; + unsigned int i; + int err; /* * if ioengine defines a setup() method, it's responsible for @@ -385,13 +391,12 @@ int setup_files(struct thread_data *td) void close_files(struct thread_data *td) { struct fio_file *f; - int i; + unsigned int i; for_each_file(td, f, i) { if (!td->filename && f->unlink && - td->filetype == FIO_TYPE_FILE) { + f->filetype == FIO_TYPE_FILE) { unlink(f->file_name); - free(f->file_name); f->file_name = NULL; } @@ -406,3 +411,35 @@ void close_files(struct thread_data *td) td->files = NULL; td->nr_files = 0; } + +static void get_file_type(struct thread_data *td, struct fio_file *f) +{ + struct stat sb; + + f->filetype = FIO_TYPE_FILE; + + if (!lstat(td->filename, &sb)) { + if (S_ISBLK(sb.st_mode)) + f->filetype = FIO_TYPE_BD; + else if (S_ISCHR(sb.st_mode)) + f->filetype = FIO_TYPE_CHAR; + } +} + +void add_file(struct thread_data *td, const char *fname) +{ + int cur_files = td->open_files; + struct fio_file *f; + + td->files = realloc(td->files, (cur_files + 1) * sizeof(*f)); + + f = &td->files[cur_files]; + memset(f, 0, sizeof(*f)); + f->fd = -1; + f->file_name = fname; + + get_file_type(td, f); + + td->open_files++; + td->nr_uniq_files = td->open_files; +} diff --git a/fio.c b/fio.c index 6222a418..4d628b9f 100644 --- a/fio.c +++ b/fio.c @@ -255,7 +255,8 @@ static void do_verify(struct thread_data *td) { struct fio_file *f; struct io_u *io_u; - int ret, i, min_events; + int ret, min_events; + unsigned int i; /* * sync io first and invalidate cache, to make sure we really @@ -379,7 +380,8 @@ static void do_io(struct thread_data *td) { struct timeval s; unsigned long usec; - int i, ret = 0; + unsigned int i; + int ret = 0; td_set_runstate(td, TD_RUNNING); @@ -650,7 +652,8 @@ static int switch_ioscheduler(struct thread_data *td) static int clear_io_state(struct thread_data *td) { struct fio_file *f; - int i, ret; + unsigned int i; + int ret; td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0; td->this_io_bytes[0] = td->this_io_bytes[1] = 0; diff --git a/fio.h b/fio.h index 0ca23cc0..3b902a8d 100644 --- a/fio.h +++ b/fio.h @@ -223,6 +223,8 @@ enum fio_ioengine_flags { * this structure holds state information for a single file. */ struct fio_file { + enum fio_filetype filetype; + /* * A file may not be a file descriptor, let the io engine decide */ @@ -230,7 +232,7 @@ struct fio_file { unsigned long file_data; int fd; }; - char *file_name; + const char *file_name; void *mmap; unsigned long long file_size; unsigned long long real_file_size; @@ -314,7 +316,6 @@ struct thread_data { int thread_number; int groupid; struct thread_stat ts; - enum fio_filetype filetype; struct fio_file *files; unsigned int nr_files; unsigned int nr_open_files; @@ -648,6 +649,7 @@ extern int __must_check open_files(struct thread_data *); extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *); extern int __must_check generic_open_file(struct thread_data *, struct fio_file *); extern void generic_close_file(struct thread_data *, struct fio_file *); +extern void add_file(struct thread_data *, const char *); /* * ETA/status stuff @@ -764,7 +766,7 @@ extern void close_ioengine(struct thread_data *); #define for_each_td(td, i) \ for ((i) = 0, (td) = &threads[0]; (i) < (int) thread_number; (i)++, (td)++) #define for_each_file(td, f, i) \ - for ((i) = 0, (f) = &(td)->files[0]; (i) < (int) (td)->open_files; (i)++, (f)++) + for ((i) = 0, (f) = &(td)->files[0]; (i) < (td)->open_files; (i)++, (f)++) #define fio_assert(td, cond) do { \ if (!(cond)) { \ diff --git a/init.c b/init.c index 624d2842..f3c756e9 100644 --- a/init.c +++ b/init.c @@ -31,6 +31,8 @@ static int str_prioclass_cb(void *, unsigned int *); static int str_exitall_cb(void); static int str_cpumask_cb(void *, unsigned int *); static int str_fst_cb(void *, const char *); +static int str_filename_cb(void *, const char *); +static int str_directory_cb(void *, const char *); #define __stringify_1(x) #x #define __stringify(x) __stringify_1(x) @@ -55,13 +57,15 @@ static struct fio_option options[] = { .name = "directory", .type = FIO_OPT_STR_STORE, .off1 = td_var_offset(directory), + .cb = str_directory_cb, .help = "Directory to store files in", }, { .name = "filename", .type = FIO_OPT_STR_STORE, .off1 = td_var_offset(filename), - .help = "Force the use of a specific file", + .cb = str_filename_cb, + .help = "File(s) to use for the workload", }, { .name = "rw", @@ -721,12 +725,6 @@ static void fixup_options(struct thread_data *td) if (td->bs_unaligned && (td->odirect || td->io_ops->flags & FIO_RAWIO)) log_err("fio: bs_unaligned may not work with raw io\n"); - /* - * O_DIRECT and char doesn't mix, clear that flag if necessary. - */ - if (td->filetype == FIO_TYPE_CHAR && td->odirect) - td->odirect = 0; - /* * thinktime_spin must be less than thinktime */ @@ -802,11 +800,11 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num) { const char *ddir_str[] = { NULL, "read", "write", "rw", NULL, "randread", "randwrite", "randrw" }; - struct stat sb; - int numjobs, i; + unsigned int i; struct fio_file *f; const char *engine; - int fn_given; + char fname[PATH_MAX]; + int numjobs; /* * the def_thread is just for options, it's not a real job @@ -829,65 +827,23 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num) if (td->odirect) td->io_ops->flags |= FIO_RAWIO; - fn_given = (long) td->filename; - if (!td->filename) + if (!td->filename) { td->filename = strdup(jobname); - td->filetype = FIO_TYPE_FILE; - if (!lstat(td->filename, &sb)) { - if (S_ISBLK(sb.st_mode)) - td->filetype = FIO_TYPE_BD; - else if (S_ISCHR(sb.st_mode)) - td->filetype = FIO_TYPE_CHAR; + for (i = 0; i < td->nr_files; i++) { + sprintf(fname, "%s.%d.%d", td->filename, td->thread_number, i); + add_file(td, fname); + } } fixup_options(td); - if (fn_given) - td->nr_uniq_files = 1; - else - td->nr_uniq_files = td->open_files; - - if (td->filetype == FIO_TYPE_FILE) { - char tmp[PATH_MAX]; - int len = 0; - - if (td->directory && td->directory[0] != '\0') { - if (lstat(td->directory, &sb) < 0) { - log_err("fio: %s is not a directory\n", td->directory); - td_verror(td, errno, "lstat"); - return 1; - } - if (!S_ISDIR(sb.st_mode)) { - log_err("fio: %s is not a directory\n", td->directory); - return 1; - } - len = sprintf(tmp, "%s/", td->directory); - } - - td->files = malloc(sizeof(struct fio_file) * td->open_files); - - for_each_file(td, f, i) { - memset(f, 0, sizeof(*f)); - f->fd = -1; - - if (fn_given) - sprintf(tmp + len, "%s", td->filename); - else - sprintf(tmp + len, "%s.%d.%d", td->filename, td->thread_number, i); - f->file_name = strdup(tmp); + for_each_file(td, f, i) { + if (td->directory && f->filetype == FIO_TYPE_FILE) { + sprintf(fname, "%s/%s", td->directory, f->file_name); + f->file_name = strdup(fname); } - } else { - td->open_files = td->nr_files = 1; - td->files = malloc(sizeof(struct fio_file)); - f = &td->files[0]; - - memset(f, 0, sizeof(*f)); - f->fd = -1; - f->file_name = strdup(td->filename); - } - for_each_file(td, f, i) { f->file_size = td->total_file_size / td->nr_files; f->file_offset = td->start_offset; } @@ -979,8 +935,9 @@ err: int init_random_state(struct thread_data *td) { unsigned long seeds[5]; - int fd, num_maps, blocks, i; + int fd, num_maps, blocks; struct fio_file *f; + unsigned int i; if (td->io_ops->flags & FIO_DISKLESSIO) return 0; @@ -1141,6 +1098,36 @@ static int str_fst_cb(void *data, const char *str) return 0; } +static int str_filename_cb(void *data, const char *input) +{ + struct thread_data *td = data; + char *fname, *str; + + str = strdup(input); + while ((fname = strsep(&str, ":")) != NULL) + add_file(td, fname); + + return 0; +} + +static int str_directory_cb(void *data, const char fio_unused *str) +{ + struct thread_data *td = data; + struct stat sb; + + if (lstat(td->directory, &sb) < 0) { + log_err("fio: %s is not a directory\n", td->directory); + td_verror(td, errno, "lstat"); + return 1; + } + if (!S_ISDIR(sb.st_mode)) { + log_err("fio: %s is not a directory\n", td->directory); + return 1; + } + + return 0; +} + /* * This is our [ini] type file parser. */ diff --git a/parse.c b/parse.c index 21fb3c2e..e09b1bbe 100644 --- a/parse.c +++ b/parse.c @@ -257,10 +257,20 @@ static int __handle_option(struct fio_option *o, const char *ptr, void *data, } break; } - case FIO_OPT_STR_STORE: + case FIO_OPT_STR_STORE: { + fio_opt_str_fn *fn = o->cb; + cp = td_var(data, o->off1); *cp = strdup(ptr); + if (fn) { + ret = fn(data, ptr); + if (ret) { + free(*cp); + *cp = NULL; + } + } break; + } case FIO_OPT_RANGE: { char tmp[128]; char *p1, *p2; @@ -579,7 +589,7 @@ void options_init(struct fio_option *options) } if (!o->cb && !o->off1) fprintf(stderr, "Option %s: neither cb nor offset given\n", o->name); - if (o->type == FIO_OPT_STR) + if (o->type == FIO_OPT_STR || o->type == FIO_OPT_STR_STORE) continue; if (o->cb && (o->off1 || o->off2 || o->off3 || o->off4)) fprintf(stderr, "Option %s: both cb and offset given\n", o->name);