From 9c60ce649d3f976d196709d4399bb7c540ce97b5 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 15 Mar 2007 09:14:47 +0100 Subject: [PATCH] Add 'filesize' option Allows the user to define the range of file sizes generated. Signed-off-by: Jens Axboe --- HOWTO | 5 ++ filesetup.c | 49 +++++++++++++++++-- fio.c | 3 -- fio.h | 10 +++- init.c | 133 ++++++++++++++++++++++++++++------------------------ 5 files changed, 132 insertions(+), 68 deletions(-) diff --git a/HOWTO b/HOWTO index e54b37b1..a2070d66 100644 --- a/HOWTO +++ b/HOWTO @@ -237,6 +237,11 @@ size=siint The total size of file io for this job. This may describe size if larger than the current file size. If this parameter is not given and the file exists, the file size will be used. +filesize=siint Individual file sizes. May be a range, in which case fio + will select sizes for files at random within the given range + and limited to 'size' in total (if that is given). If not + given, each created file is the same size. + bs=siint The block size used for the io units. Defaults to 4k. Values can be given for both read and writes. If a single siint is given, it will apply to both. If a second siint is specified diff --git a/filesetup.c b/filesetup.c index f43efe0d..053bda9d 100644 --- a/filesetup.c +++ b/filesetup.c @@ -97,11 +97,33 @@ err: return 1; } +static unsigned long long set_rand_file_size(struct thread_data *td, + unsigned long long total_size) +{ + unsigned long long upper = total_size; + unsigned long long ret; + long r; + + if (upper > td->file_size_high) + upper = td->file_size_high; + else if (upper < td->file_size_low) + return 0; + else if (!upper) + return 0; + + r = os_random_long(&td->file_size_state); + ret = td->file_size_low + (unsigned long long) ((double) upper * (r / (RAND_MAX + 1.0))); + ret -= (ret % td->rw_min_bs); + if (ret > upper) + ret = upper; + return ret; +} + static int create_files(struct thread_data *td) { struct fio_file *f; int err, need_create, can_extend; - unsigned long long total_file_size; + unsigned long long total_file_size, local_file_size; unsigned int i, new_files; new_files = 0; @@ -132,6 +154,10 @@ static int create_files(struct thread_data *td) return 0; need_create = 0; + local_file_size = total_file_size; + if (!local_file_size) + local_file_size = -1; + for_each_file(td, f, i) { int file_there; @@ -140,7 +166,23 @@ static int create_files(struct thread_data *td) if (f->flags & FIO_FILE_EXISTS) continue; - f->file_size = total_file_size / new_files; + if (!td->file_size_low) + f->file_size = total_file_size / new_files; + else { + /* + * If we don't have enough space left for a file + * of the minimum size, bail. + */ + if (local_file_size < td->file_size_low) { + log_info("fio: limited to %d files\n", i); + new_files -= (td->nr_files - i); + td->nr_files = i; + break; + } + + f->file_size = set_rand_file_size(td, local_file_size); + local_file_size -= f->file_size; + } file_there = !file_ok(td, f); @@ -162,9 +204,8 @@ static int create_files(struct thread_data *td) } temp_stall_ts = 1; - fprintf(f_out, "%s: Laying out IO file(s) (%u x %LuMiB == %LuMiB)\n", + fprintf(f_out, "%s: Laying out IO file(s) (%u files / %LuMiB)\n", td->name, new_files, - (total_file_size >> 20) / new_files, total_file_size >> 20); err = 0; diff --git a/fio.c b/fio.c index e6bd18a0..5f06940f 100644 --- a/fio.c +++ b/fio.c @@ -726,9 +726,6 @@ static void *thread_main(void *data) goto err_sem; } - if (init_random_state(td)) - goto err_sem; - if (td->ioscheduler && switch_ioscheduler(td)) goto err_sem; diff --git a/fio.h b/fio.h index 72104226..3f0af656 100644 --- a/fio.h +++ b/fio.h @@ -492,6 +492,13 @@ struct thread_data { unsigned int file_service_nr; unsigned int file_service_left; struct fio_file *file_service_file; + + /* + * For generating file sizes + */ + os_random_state_t file_size_state; + unsigned long long file_size_low; + unsigned long long file_size_high; }; /* @@ -649,7 +656,6 @@ extern void set_genesis_time(void); * Init functions */ extern int __must_check parse_options(int, char **); -extern int __must_check init_random_state(struct thread_data *); /* * File setup/shutdown @@ -740,6 +746,8 @@ extern void td_io_close_file(struct thread_data *, struct fio_file *); fprintf(stderr, ##args); \ } while (0) +#define log_info(args...) fprintf(f_out, ##args) + FILE *get_f_out(void); FILE *get_f_err(void); diff --git a/init.c b/init.c index cadc0d66..e794b372 100644 --- a/init.c +++ b/init.c @@ -183,7 +183,14 @@ static struct fio_option options[] = { .name = "size", .type = FIO_OPT_STR_VAL, .off1 = td_var_offset(total_file_size), - .help = "Size of device or file", + .help = "Total size of device or files", + }, + { + .name = "filesize", + .type = FIO_OPT_STR_VAL, + .off1 = td_var_offset(file_size_low), + .off2 = td_var_offset(file_size_high), + .help = "Size of individual files", }, { .name = "bs", @@ -801,6 +808,9 @@ static void fixup_options(struct thread_data *td) td->rw_min_bs = min(td->min_bs[DDIR_READ], td->min_bs[DDIR_WRITE]); + if (!td->file_size_high) + td->file_size_high = td->file_size_low; + if (td_read(td) && !td_rw(td)) td->verify = 0; @@ -893,6 +903,65 @@ static int exists_and_not_file(const char *filename) return 1; } +/* + * Initialize the various random states we need (random io, block size ranges, + * read/write mix, etc). + */ +static int init_random_state(struct thread_data *td) +{ + unsigned long seeds[6]; + int fd, num_maps, blocks; + struct fio_file *f; + unsigned int i; + + fd = open("/dev/urandom", O_RDONLY); + if (fd == -1) { + td_verror(td, errno, "open"); + return 1; + } + + if (read(fd, seeds, sizeof(seeds)) < (int) sizeof(seeds)) { + td_verror(td, EIO, "read"); + close(fd); + return 1; + } + + close(fd); + + os_random_seed(seeds[0], &td->bsrange_state); + os_random_seed(seeds[1], &td->verify_state); + os_random_seed(seeds[2], &td->rwmix_state); + + if (td->file_service_type == FIO_FSERVICE_RANDOM) + os_random_seed(seeds[3], &td->next_file_state); + + os_random_seed(seeds[5], &td->file_size_state); + + if (!td_random(td)) + return 0; + + if (td->rand_repeatable) + seeds[4] = FIO_RANDSEED * td->thread_number; + + if (!td->norandommap) { + for_each_file(td, f, i) { + blocks = (f->real_file_size + td->rw_min_bs - 1) / td->rw_min_bs; + num_maps = (blocks + BLOCKS_PER_MAP-1)/ BLOCKS_PER_MAP; + f->file_map = malloc(num_maps * sizeof(long)); + if (!f->file_map) { + log_err("fio: failed allocating random map. If running a large number of jobs, try the 'norandommap' option\n"); + return 1; + } + f->num_maps = num_maps; + memset(f->file_map, 0, num_maps * sizeof(long)); + } + } + + os_random_seed(seeds[4], &td->random_state); + return 0; +} + + /* * Adds a job to the list of things todo. Sanitizes the various options * to make sure we don't have conflicts, and initializes various @@ -966,6 +1035,9 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num) td->groupid = groupid; prev_group_jobs++; + if (init_random_state(td)) + goto err; + if (setup_rate(td)) goto err; @@ -1039,65 +1111,6 @@ err: return -1; } -/* - * Initialize the various random states we need (random io, block size ranges, - * read/write mix, etc). - */ -int init_random_state(struct thread_data *td) -{ - unsigned long seeds[5]; - int fd, num_maps, blocks; - struct fio_file *f; - unsigned int i; - - if (td->io_ops->flags & FIO_DISKLESSIO) - return 0; - - fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) { - td_verror(td, errno, "open"); - return 1; - } - - if (read(fd, seeds, sizeof(seeds)) < (int) sizeof(seeds)) { - td_verror(td, EIO, "read"); - close(fd); - return 1; - } - - close(fd); - - os_random_seed(seeds[0], &td->bsrange_state); - os_random_seed(seeds[1], &td->verify_state); - os_random_seed(seeds[2], &td->rwmix_state); - - if (td->file_service_type == FIO_FSERVICE_RANDOM) - os_random_seed(seeds[3], &td->next_file_state); - - if (!td_random(td)) - return 0; - - if (td->rand_repeatable) - seeds[4] = FIO_RANDSEED * td->thread_number; - - if (!td->norandommap) { - for_each_file(td, f, i) { - blocks = (f->real_file_size + td->rw_min_bs - 1) / td->rw_min_bs; - num_maps = (blocks + BLOCKS_PER_MAP-1)/ BLOCKS_PER_MAP; - f->file_map = malloc(num_maps * sizeof(long)); - if (!f->file_map) { - log_err("fio: failed allocating random map. If running a large number of jobs, try the 'norandommap' option\n"); - return 1; - } - f->num_maps = num_maps; - memset(f->file_map, 0, num_maps * sizeof(long)); - } - } - - os_random_seed(seeds[4], &td->random_state); - return 0; -} - static void fill_cpu_mask(os_cpu_mask_t cpumask, int cpu) { #ifdef FIO_HAVE_CPU_AFFINITY -- 2.25.1