Fix issue with openfiles= and file sizing

Fill the given size as well as we can, given the size and nr
of files.

Fix an issue where we don't properly honor how many files to
keep open at any point in time.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/backend.c b/backend.c
index a607134b878d4603fcb0d3e32e6273847fa2dc7a..32bc2652bd0ba93793c9b12cc27ca8b4cf81aef0 100644 (file)
--- a/backend.c
+++ b/backend.c
@@ -52,6 +52,7 @@
 #include "server.h"
 #include "lib/getrusage.h"
 #include "idletime.h"
+#include "err.h"
 
 static pthread_t disk_util_thread;
 static struct fio_mutex *disk_thread_mutex;
@@ -478,6 +479,12 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
                                break;
 
                        while ((io_u = get_io_u(td)) != NULL) {
+                               if (IS_ERR(io_u)) {
+                                       io_u = NULL;
+                                       ret = FIO_Q_BUSY;
+                                       goto reap;
+                               }
+
                                /*
                                 * We are only interested in the places where
                                 * we wrote or trimmed IOs. Turn those into
@@ -574,6 +581,7 @@ sync_done:
                 * completed io_u's first. Note that we can get BUSY even
                 * without IO queued, if the system is resource starved.
                 */
+reap:
                full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
                if (full || !td->o.iodepth_batch_complete) {
                        min_events = min(td->o.iodepth_batch_complete,
@@ -692,7 +700,14 @@ static uint64_t do_io(struct thread_data *td)
                        break;
 
                io_u = get_io_u(td);
-               if (!io_u) {
+               if (IS_ERR_OR_NULL(io_u)) {
+                       int err = PTR_ERR(io_u);
+
+                       io_u = NULL;
+                       if (err == -EBUSY) {
+                               ret = FIO_Q_BUSY;
+                               goto reap;
+                       }
                        if (td->o.latency_target)
                                goto reap;
                        break;
@@ -1124,6 +1139,9 @@ static int keep_running(struct thread_data *td)
                if (diff < td_max_bs(td))
                        return 0;
 
+               if (fio_files_done(td))
+                       return 0;
+
                return 1;
        }
 

diff --git a/err.h b/err.h
new file mode 100644 (file)
index 0000000..5c024ee
--- /dev/null
+++ b/err.h
@@ -0,0 +1,44 @@
+#ifndef FIO_ERR_H
+#define FIO_ERR_H
+
+/*
+ * Kernel pointers have redundant information, so we can use a
+ * scheme where we can return either an error code or a dentry
+ * pointer with the same return value.
+ *
+ * This should be a per-architecture thing, to allow different
+ * error and pointer decisions.
+ */
+#define MAX_ERRNO      4095
+
+#define IS_ERR_VALUE(x) ((x) >= (unsigned long)-MAX_ERRNO)
+
+static inline void *ERR_PTR(long error)
+{
+       return (void *) error;
+}
+
+static inline long PTR_ERR(const void *ptr)
+{
+       return (long) ptr;
+}
+
+static inline long IS_ERR(const void *ptr)
+{
+       return IS_ERR_VALUE((unsigned long)ptr);
+}
+
+static inline long IS_ERR_OR_NULL(const void *ptr)
+{
+       return !ptr || IS_ERR_VALUE((unsigned long)ptr);
+}
+
+static inline int PTR_ERR_OR_ZERO(const void *ptr)
+{
+       if (IS_ERR(ptr))
+               return PTR_ERR(ptr);
+       else
+               return 0;
+}
+
+#endif

diff --git a/file.h b/file.h
index d7e05f4fb22e0aaf97b5cb0c7453f1ce4cdc1051..19413fc8fa777d96b419e62c2c9d0c8433b4c2b2 100644 (file)
--- a/file.h
+++ b/file.h
@@ -176,5 +176,6 @@ extern void dup_files(struct thread_data *, struct thread_data *);
 extern int get_fileno(struct thread_data *, const char *);
 extern void free_release_files(struct thread_data *);
 void fio_file_reset(struct thread_data *, struct fio_file *);
+int fio_files_done(struct thread_data *);
 
 #endif

diff --git a/filesetup.c b/filesetup.c
index d1702e29c75bf1b1f73dd0d2dc3ade1ac35f9e12..bae60929df32f7f80f1dfa4ba00ab3ffbb9514b0 100644 (file)
--- a/filesetup.c
+++ b/filesetup.c
@@ -734,9 +734,11 @@ int setup_files(struct thread_data *td)
        unsigned long long total_size, extend_size;
        struct thread_options *o = &td->o;
        struct fio_file *f;
-       unsigned int i;
+       unsigned int i, nr_fs_extra = 0;
        int err = 0, need_extend;
        int old_state;
+       const unsigned int bs = td_min_bs(td);
+       uint64_t fs = 0;
 
        dprint(FD_FILE, "setup files\n");
 
@@ -785,6 +787,20 @@ int setup_files(struct thread_data *td)
                goto err_out;
        }
 
+       /*
+        * Calculate per-file size and potential extra size for the
+        * first files, if needed.
+        */
+       if (!o->file_size_low) {
+               uint64_t all_fs;
+
+               fs = o->size / o->nr_files;
+               all_fs = fs * o->nr_files;
+
+               if (all_fs < o->size)
+                       nr_fs_extra = (o->size - all_fs) / bs;
+       }
+
        /*
         * now file sizes are known, so we can set ->io_size. if size= is
         * not given, ->io_size is just equal to ->real_file_size. if size
@@ -798,10 +814,17 @@ int setup_files(struct thread_data *td)
                if (!o->file_size_low) {
                        /*
                         * no file size range given, file size is equal to
-                        * total size divided by number of files. if that is
-                        * zero, set it to the real file size.
+                        * total size divided by number of files. If that is
+                        * zero, set it to the real file size. If the size
+                        * doesn't divide nicely with the min blocksize,
+                        * make the first files bigger.
                         */
-                       f->io_size = o->size / o->nr_files;
+                       f->io_size = fs;
+                       if (nr_fs_extra) {
+                               nr_fs_extra--;
+                               f->io_size += bs;
+                       }
+
                        if (!f->io_size)
                                f->io_size = f->real_file_size - f->file_offset;
                } else if (f->real_file_size < o->file_size_low ||
@@ -1386,3 +1409,15 @@ void fio_file_reset(struct thread_data *td, struct fio_file *f)
        if (td->o.random_generator == FIO_RAND_GEN_LFSR)
                lfsr_reset(&f->lfsr, td->rand_seeds[FIO_RAND_BLOCK_OFF]);
 }
+
+int fio_files_done(struct thread_data *td)
+{
+       struct fio_file *f;
+       unsigned int i;
+
+       for_each_file(td, f, i)
+               if (!fio_file_done(f))
+                       return 0;
+
+       return 1;
+}

diff --git a/io_u.c b/io_u.c
index 64ff73cd5555b8d4efa0303085f7944cae07544c..acc1a7b1dcb9f4ab6ac2c56a1552247c905ab41e 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -11,6 +11,7 @@
 #include "trim.h"
 #include "lib/rand.h"
 #include "lib/axmap.h"
+#include "err.h"
 
 struct io_completion_data {
        int nr;                         /* input */
@@ -985,6 +986,9 @@ static struct fio_file *get_next_file_rand(struct thread_data *td,
                if (!fio_file_open(f)) {
                        int err;
 
+                       if (td->nr_open_files >= td->o.open_files)
+                               return ERR_PTR(-EBUSY);
+
                        err = td_io_open_file(td, f);
                        if (err)
                                continue;
@@ -1027,6 +1031,9 @@ static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
                if (!fio_file_open(f)) {
                        int err;
 
+                       if (td->nr_open_files >= td->o.open_files)
+                               return ERR_PTR(-EBUSY);
+
                        err = td_io_open_file(td, f);
                        if (err) {
                                dprint(FD_FILE, "error %d on open of %s\n",
@@ -1080,6 +1087,9 @@ static struct fio_file *__get_next_file(struct thread_data *td)
        else
                f = get_next_file_rand(td, FIO_FILE_open, FIO_FILE_closing);
 
+       if (IS_ERR(f))
+               return f;
+
        td->file_service_file = f;
        td->file_service_left = td->file_service_nr - 1;
 out:
@@ -1099,14 +1109,14 @@ static struct fio_file *get_next_file(struct thread_data *td)
        return __get_next_file(td);
 }
 
-static int set_io_u_file(struct thread_data *td, struct io_u *io_u)
+static long set_io_u_file(struct thread_data *td, struct io_u *io_u)
 {
        struct fio_file *f;
 
        do {
                f = get_next_file(td);
-               if (!f)
-                       return 1;
+               if (IS_ERR_OR_NULL(f))
+                       return PTR_ERR(f);
 
                io_u->file = f;
                get_file(f);
@@ -1400,6 +1410,7 @@ struct io_u *get_io_u(struct thread_data *td)
        struct fio_file *f;
        struct io_u *io_u;
        int do_scramble = 0;
+       long ret = 0;
 
        io_u = __get_io_u(td);
        if (!io_u) {
@@ -1425,11 +1436,17 @@ struct io_u *get_io_u(struct thread_data *td)
                if (read_iolog_get(td, io_u))
                        goto err_put;
        } else if (set_io_u_file(td, io_u)) {
+               ret = -EBUSY;
                dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
                goto err_put;
        }
 
        f = io_u->file;
+       if (!f) {
+               dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
+               goto err_put;
+       }
+
        assert(fio_file_open(f));
 
        if (ddir_rw(io_u->ddir)) {
@@ -1478,7 +1495,7 @@ out:
 err_put:
        dprint(FD_IO, "get_io_u failed\n");
        put_io_u(td, io_u);
-       return NULL;
+       return ERR_PTR(ret);
 }
 
 void io_u_log_error(struct thread_data *td, struct io_u *io_u)