Fix multithread issues when operating on a single shared file

When nrfiles=1, numjobs>1 and create_serialize=0, multiple threads
try to create the single shared file in parallel. If the file was
pre-existing, but an incorrect size, then multiple threads are
deleting and creating at the same time. When all of this happens
in parallel, there is a chance that the file can end up the
incorrect size (the chance increases as numjobs increases).
These changes handle the corner case described above by having
a single thread create/extend the file prior to running all of
the threads in parallel. By doing this step early, when
setup_files() is called later, it should no longer need to
create or extend the file, avoiding the race condition. The user
still needs to set a fallocate option other than 'none' or the
file will end up 0 bytes in size and the race condition will
still occur. It would be simple to add a ftruncate() to the code
to force this, but that would override the user's choice of
fallocate options.

Signed-off-by: Chris Weber <weberc@netapp.com>

diff --git a/backend.c b/backend.c
index e5bb4e259034b29af86fbaed018ae64c7bb288db..3a99850db1f5659b78bf8616521c6f84af1a38dd 100644 (file)
--- a/backend.c
+++ b/backend.c
@@ -2314,8 +2314,25 @@ static void run_threads(struct sk_out *sk_out)
        for_each_td(td, i) {
                print_status_init(td->thread_number - 1);
 
-               if (!td->o.create_serialize)
+               if (!td->o.create_serialize) {
+                       /*
+                        *  When operating on a single rile in parallel,
+                        *  perform single-threaded early setup so that
+                        *  when setup_files() does not run into issues
+                        *  later.
+                       */
+                       if (!i && td->o.nr_files==1) {
+                               if (setup_shared_file(td)) {
+                                       exit_value++;
+                                       if (td->error)
+                                               log_err("fio: pid=%d, err=%d/%s\n",
+                                                       (int) td->pid, td->error, td->verror);
+                                       td_set_runstate(td, TD_REAPED);
+                                       todo--;
+                               }
+                       }
                        continue;
+               }
 
                if (fio_verify_load_state(td))
                        goto reap;

diff --git a/file.h b/file.h
index da1b894706a64227af9a484d760fadc79edcd0d5..e646cf22f6b7ec184fe4fb6ab94f8fb5918cb76e 100644 (file)
--- a/file.h
+++ b/file.h
@@ -201,6 +201,7 @@ struct thread_data;
 extern void close_files(struct thread_data *);
 extern void close_and_free_files(struct thread_data *);
 extern uint64_t get_start_offset(struct thread_data *, struct fio_file *);
+extern int __must_check setup_shared_file(struct thread_data *);
 extern int __must_check setup_files(struct thread_data *);
 extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *);
 #ifdef __cplusplus

diff --git a/filesetup.c b/filesetup.c
index ab6c488bb750ed7c53a56214a69a9f8b100a7057..b30a2932db1ae309f5aa06a8b1f0e53b759ef0b6 100644 (file)
--- a/filesetup.c
+++ b/filesetup.c
@@ -143,7 +143,7 @@ static int extend_file(struct thread_data *td, struct fio_file *f)
        if (unlink_file || new_layout) {
                int ret;
 
-               dprint(FD_FILE, "layout unlink %s\n", f->file_name);
+               dprint(FD_FILE, "layout %d unlink %d %s\n", new_layout, unlink_file, f->file_name);
 
                ret = td_io_unlink_file(td, f);
                if (ret != 0 && ret != ENOENT) {
@@ -198,6 +198,9 @@ static int extend_file(struct thread_data *td, struct fio_file *f)
                }
        }
 
+
+       dprint(FD_FILE, "fill file %s, size %llu\n", f->file_name, (unsigned long long) f->real_file_size);
+
        left = f->real_file_size;
        bs = td->o.max_bs[DDIR_WRITE];
        if (bs > left)
@@ -1078,6 +1081,45 @@ static bool create_work_dirs(struct thread_data *td, const char *fname)
        return true;
 }
 
+int setup_shared_file(struct thread_data *td)
+{
+       struct fio_file *f;
+       uint64_t file_size;
+       int err = 0;
+
+       if (td->o.nr_files > 1) {
+               log_err("fio: shared file setup called for multiple files\n");
+               return -1;
+       }
+
+       get_file_sizes(td);
+
+       f = td->files[0];
+
+       if (f == NULL) {
+               log_err("fio: NULL shared file\n");
+               return -1;
+       }
+
+       file_size = thread_number * td->o.size;
+       dprint(FD_FILE, "shared setup %s real_file_size=%llu, desired=%llu\n", 
+                       f->file_name, (unsigned long long)f->real_file_size, (unsigned long long)file_size);
+
+       if (f->real_file_size < file_size) {
+               dprint(FD_FILE, "fio: extending shared file\n");
+               f->real_file_size = file_size;
+               err = extend_file(td, f);
+               if (!err) {
+                       err = __file_invalidate_cache(td, f, 0, f->real_file_size);
+               }
+               get_file_sizes(td);
+               dprint(FD_FILE, "shared setup new real_file_size=%llu\n", 
+                               (unsigned long long)f->real_file_size);
+       }
+
+       return err;
+}
+
 /*
  * Open the files and setup files sizes, creating files if necessary.
  */
@@ -1092,7 +1134,7 @@ int setup_files(struct thread_data *td)
        const unsigned long long bs = td_min_bs(td);
        uint64_t fs = 0;
 
-       dprint(FD_FILE, "setup files\n");
+       dprint(FD_FILE, "setup files (thread_number=%d, subjob_number=%d)\n", td->thread_number, td->subjob_number);
 
        old_state = td_bump_runstate(td, TD_SETTING_UP);