From 002fe73409d1e3d5e7dfe2885f75885bfaf506bc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Feb 2014 08:31:13 -0700 Subject: [PATCH] Fix issue with openfiles= and file sizing Fill the given size as well as we can, given the size and nr of files. Fix an issue where we don't properly honor how many files to keep open at any point in time. Signed-off-by: Jens Axboe --- backend.c | 20 +++++++++++++++++++- err.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ file.h | 1 + filesetup.c | 43 +++++++++++++++++++++++++++++++++++++++---- io_u.c | 25 +++++++++++++++++++++---- 5 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 err.h diff --git a/backend.c b/backend.c index a607134b..32bc2652 100644 --- a/backend.c +++ b/backend.c @@ -52,6 +52,7 @@ #include "server.h" #include "lib/getrusage.h" #include "idletime.h" +#include "err.h" static pthread_t disk_util_thread; static struct fio_mutex *disk_thread_mutex; @@ -478,6 +479,12 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes) break; while ((io_u = get_io_u(td)) != NULL) { + if (IS_ERR(io_u)) { + io_u = NULL; + ret = FIO_Q_BUSY; + goto reap; + } + /* * We are only interested in the places where * we wrote or trimmed IOs. Turn those into @@ -574,6 +581,7 @@ sync_done: * completed io_u's first. Note that we can get BUSY even * without IO queued, if the system is resource starved. */ +reap: full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth); if (full || !td->o.iodepth_batch_complete) { min_events = min(td->o.iodepth_batch_complete, @@ -692,7 +700,14 @@ static uint64_t do_io(struct thread_data *td) break; io_u = get_io_u(td); - if (!io_u) { + if (IS_ERR_OR_NULL(io_u)) { + int err = PTR_ERR(io_u); + + io_u = NULL; + if (err == -EBUSY) { + ret = FIO_Q_BUSY; + goto reap; + } if (td->o.latency_target) goto reap; break; @@ -1124,6 +1139,9 @@ static int keep_running(struct thread_data *td) if (diff < td_max_bs(td)) return 0; + if (fio_files_done(td)) + return 0; + return 1; } diff --git a/err.h b/err.h new file mode 100644 index 00000000..5c024ee8 --- /dev/null +++ b/err.h @@ -0,0 +1,44 @@ +#ifndef FIO_ERR_H +#define FIO_ERR_H + +/* + * Kernel pointers have redundant information, so we can use a + * scheme where we can return either an error code or a dentry + * pointer with the same return value. + * + * This should be a per-architecture thing, to allow different + * error and pointer decisions. + */ +#define MAX_ERRNO 4095 + +#define IS_ERR_VALUE(x) ((x) >= (unsigned long)-MAX_ERRNO) + +static inline void *ERR_PTR(long error) +{ + return (void *) error; +} + +static inline long PTR_ERR(const void *ptr) +{ + return (long) ptr; +} + +static inline long IS_ERR(const void *ptr) +{ + return IS_ERR_VALUE((unsigned long)ptr); +} + +static inline long IS_ERR_OR_NULL(const void *ptr) +{ + return !ptr || IS_ERR_VALUE((unsigned long)ptr); +} + +static inline int PTR_ERR_OR_ZERO(const void *ptr) +{ + if (IS_ERR(ptr)) + return PTR_ERR(ptr); + else + return 0; +} + +#endif diff --git a/file.h b/file.h index d7e05f4f..19413fc8 100644 --- a/file.h +++ b/file.h @@ -176,5 +176,6 @@ extern void dup_files(struct thread_data *, struct thread_data *); extern int get_fileno(struct thread_data *, const char *); extern void free_release_files(struct thread_data *); void fio_file_reset(struct thread_data *, struct fio_file *); +int fio_files_done(struct thread_data *); #endif diff --git a/filesetup.c b/filesetup.c index d1702e29..bae60929 100644 --- a/filesetup.c +++ b/filesetup.c @@ -734,9 +734,11 @@ int setup_files(struct thread_data *td) unsigned long long total_size, extend_size; struct thread_options *o = &td->o; struct fio_file *f; - unsigned int i; + unsigned int i, nr_fs_extra = 0; int err = 0, need_extend; int old_state; + const unsigned int bs = td_min_bs(td); + uint64_t fs = 0; dprint(FD_FILE, "setup files\n"); @@ -785,6 +787,20 @@ int setup_files(struct thread_data *td) goto err_out; } + /* + * Calculate per-file size and potential extra size for the + * first files, if needed. + */ + if (!o->file_size_low) { + uint64_t all_fs; + + fs = o->size / o->nr_files; + all_fs = fs * o->nr_files; + + if (all_fs < o->size) + nr_fs_extra = (o->size - all_fs) / bs; + } + /* * now file sizes are known, so we can set ->io_size. if size= is * not given, ->io_size is just equal to ->real_file_size. if size @@ -798,10 +814,17 @@ int setup_files(struct thread_data *td) if (!o->file_size_low) { /* * no file size range given, file size is equal to - * total size divided by number of files. if that is - * zero, set it to the real file size. + * total size divided by number of files. If that is + * zero, set it to the real file size. If the size + * doesn't divide nicely with the min blocksize, + * make the first files bigger. */ - f->io_size = o->size / o->nr_files; + f->io_size = fs; + if (nr_fs_extra) { + nr_fs_extra--; + f->io_size += bs; + } + if (!f->io_size) f->io_size = f->real_file_size - f->file_offset; } else if (f->real_file_size < o->file_size_low || @@ -1386,3 +1409,15 @@ void fio_file_reset(struct thread_data *td, struct fio_file *f) if (td->o.random_generator == FIO_RAND_GEN_LFSR) lfsr_reset(&f->lfsr, td->rand_seeds[FIO_RAND_BLOCK_OFF]); } + +int fio_files_done(struct thread_data *td) +{ + struct fio_file *f; + unsigned int i; + + for_each_file(td, f, i) + if (!fio_file_done(f)) + return 0; + + return 1; +} diff --git a/io_u.c b/io_u.c index 64ff73cd..acc1a7b1 100644 --- a/io_u.c +++ b/io_u.c @@ -11,6 +11,7 @@ #include "trim.h" #include "lib/rand.h" #include "lib/axmap.h" +#include "err.h" struct io_completion_data { int nr; /* input */ @@ -985,6 +986,9 @@ static struct fio_file *get_next_file_rand(struct thread_data *td, if (!fio_file_open(f)) { int err; + if (td->nr_open_files >= td->o.open_files) + return ERR_PTR(-EBUSY); + err = td_io_open_file(td, f); if (err) continue; @@ -1027,6 +1031,9 @@ static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf, if (!fio_file_open(f)) { int err; + if (td->nr_open_files >= td->o.open_files) + return ERR_PTR(-EBUSY); + err = td_io_open_file(td, f); if (err) { dprint(FD_FILE, "error %d on open of %s\n", @@ -1080,6 +1087,9 @@ static struct fio_file *__get_next_file(struct thread_data *td) else f = get_next_file_rand(td, FIO_FILE_open, FIO_FILE_closing); + if (IS_ERR(f)) + return f; + td->file_service_file = f; td->file_service_left = td->file_service_nr - 1; out: @@ -1099,14 +1109,14 @@ static struct fio_file *get_next_file(struct thread_data *td) return __get_next_file(td); } -static int set_io_u_file(struct thread_data *td, struct io_u *io_u) +static long set_io_u_file(struct thread_data *td, struct io_u *io_u) { struct fio_file *f; do { f = get_next_file(td); - if (!f) - return 1; + if (IS_ERR_OR_NULL(f)) + return PTR_ERR(f); io_u->file = f; get_file(f); @@ -1400,6 +1410,7 @@ struct io_u *get_io_u(struct thread_data *td) struct fio_file *f; struct io_u *io_u; int do_scramble = 0; + long ret = 0; io_u = __get_io_u(td); if (!io_u) { @@ -1425,11 +1436,17 @@ struct io_u *get_io_u(struct thread_data *td) if (read_iolog_get(td, io_u)) goto err_put; } else if (set_io_u_file(td, io_u)) { + ret = -EBUSY; dprint(FD_IO, "io_u %p, setting file failed\n", io_u); goto err_put; } f = io_u->file; + if (!f) { + dprint(FD_IO, "io_u %p, setting file failed\n", io_u); + goto err_put; + } + assert(fio_file_open(f)); if (ddir_rw(io_u->ddir)) { @@ -1478,7 +1495,7 @@ out: err_put: dprint(FD_IO, "get_io_u failed\n"); put_io_u(td, io_u); - return NULL; + return ERR_PTR(ret); } void io_u_log_error(struct thread_data *td, struct io_u *io_u) -- 2.25.1