filesetup: add engine's io_ops to prepopulate file with data

In some cases (e.g. engine marked as diskless) files are not laid out.
If the first job is a read job, results are higher than expected
(because reading zero page). Each engine should deliver func to
prepopulate file with data to avoid this situation.

Signed-off-by: Łukasz Stolarczuk <lukasz.stolarczuk@intel.com>

diff --git a/engines/libpmem.c b/engines/libpmem.c
index eefb7767f3eab953f202236a395b096ef8f67636..2338f0fa24f42b4f8318ca9a4bc299283e28c8bf 100644 (file)
--- a/engines/libpmem.c
+++ b/engines/libpmem.c
@@ -255,6 +255,7 @@ FIO_STATIC struct ioengine_ops ioengine = {
        .open_file      = fio_libpmem_open_file,
        .close_file     = fio_libpmem_close_file,
        .get_file_size  = generic_get_file_size,
+       .prepopulate_file = generic_prepopulate_file,
        .flags          = FIO_SYNCIO | FIO_RAWIO | FIO_DISKLESSIO | FIO_NOEXTEND |
                                FIO_NODISKUTIL | FIO_BARRIER | FIO_MEMALIGN,
 };

diff --git a/file.h b/file.h
index 493ec04a6677e90f6c0d0182e0b082b31302451f..faf65a2a013846fb716f0b80403d6549b90120d2 100644 (file)
--- a/file.h
+++ b/file.h
@@ -207,6 +207,7 @@ extern "C" {
 extern int __must_check generic_open_file(struct thread_data *, struct fio_file *);
 extern int __must_check generic_close_file(struct thread_data *, struct fio_file *);
 extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *);
+extern int __must_check generic_prepopulate_file(struct thread_data *, struct fio_file *);
 #ifdef __cplusplus
 }
 #endif

diff --git a/filesetup.c b/filesetup.c
index 9d0337578d2266355b8ff478c5153cff88c26d77..661d4c2fa083e2c4b948b628a344c6f294b28ebe 100644 (file)
--- a/filesetup.c
+++ b/filesetup.c
@@ -338,6 +338,95 @@ error:
        return ret;
 }
 
+/*
+ * Generic function to prepopulate regular file with data.
+ * Useful if you want to make sure I/O engine has data to read.
+ * Leaves f->fd open on success, caller must close.
+ */
+int generic_prepopulate_file(struct thread_data *td, struct fio_file *f)
+{
+       int flags;
+       unsigned long long left, bs;
+       char *b = NULL;
+
+       /* generic function for regular files only */
+       assert(f->filetype == FIO_TYPE_FILE);
+
+       if (read_only) {
+               log_err("fio: refusing to write a file due to read-only\n");
+               return 0;
+       }
+
+       flags = O_WRONLY;
+       if (td->o.allow_create)
+               flags |= O_CREAT;
+
+#ifdef WIN32
+       flags |= _O_BINARY;
+#endif
+
+       dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags);
+       f->fd = open(f->file_name, flags, 0644);
+       if (f->fd < 0) {
+               int err = errno;
+
+               if (err == ENOENT && !td->o.allow_create)
+                       log_err("fio: file creation disallowed by "
+                                       "allow_file_create=0\n");
+               else
+                       td_verror(td, err, "open");
+               return 1;
+       }
+
+       left = f->real_file_size;
+       bs = td->o.max_bs[DDIR_WRITE];
+       if (bs > left)
+               bs = left;
+
+       b = malloc(bs);
+       if (!b) {
+               td_verror(td, errno, "malloc");
+               goto err;
+       }
+
+       while (left && !td->terminate) {
+               ssize_t r;
+
+               if (bs > left)
+                       bs = left;
+
+               fill_io_buffer(td, b, bs, bs);
+
+               r = write(f->fd, b, bs);
+
+               if (r > 0) {
+                       left -= r;
+               } else {
+                       td_verror(td, errno, "write");
+                       goto err;
+               }
+       }
+
+       if (td->terminate) {
+               dprint(FD_FILE, "terminate unlink %s\n", f->file_name);
+               td_io_unlink_file(td, f);
+       } else if (td->o.create_fsync) {
+               if (fsync(f->fd) < 0) {
+                       td_verror(td, errno, "fsync");
+                       goto err;
+               }
+       }
+
+       free(b);
+       return 0;
+err:
+       close(f->fd);
+       f->fd = -1;
+       if (b)
+               free(b);
+       return 1;
+}
+
 unsigned long long get_rand_file_size(struct thread_data *td)
 {
        unsigned long long ret, sized;
@@ -1254,6 +1343,43 @@ int setup_files(struct thread_data *td)
                temp_stall_ts = 0;
        }
 
+       if (err)
+               goto err_out;
+
+       /*
+        * Prepopulate files with data. It might be expected to read some
+        * "real" data instead of zero'ed files (if no writes to file occurred
+        * prior to a read job). Engine has to provide a way to do that.
+        */
+       if (td->io_ops->prepopulate_file) {
+               temp_stall_ts = 1;
+
+               for_each_file(td, f, i) {
+                       if (output_format & FIO_OUTPUT_NORMAL) {
+                               log_info("%s: Prepopulating IO file (%s)\n",
+                                                       o->name, f->file_name);
+                       }
+
+                       err = td->io_ops->prepopulate_file(td, f);
+                       if (err)
+                               break;
+
+                       err = __file_invalidate_cache(td, f, f->file_offset,
+                                                               f->io_size);
+
+                       /*
+                        * Shut up static checker
+                        */
+                       if (f->fd != -1)
+                               close(f->fd);
+
+                       f->fd = -1;
+                       if (err)
+                               break;
+               }
+               temp_stall_ts = 0;
+       }
+
        if (err)
                goto err_out;
 

diff --git a/ioengines.h b/ioengines.h
index 839b318da60464eb5d78d503ca930559c2548c41..1d01ab0a6d1345a71b04b0576f40fefca20062b9 100644 (file)
--- a/ioengines.h
+++ b/ioengines.h
@@ -8,7 +8,7 @@
 #include "io_u.h"
 #include "zbd_types.h"
 
-#define FIO_IOOPS_VERSION      28
+#define FIO_IOOPS_VERSION      29
 
 #ifndef CONFIG_DYNAMIC_ENGINES
 #define FIO_STATIC     static
@@ -47,6 +47,7 @@ struct ioengine_ops {
        int (*invalidate)(struct thread_data *, struct fio_file *);
        int (*unlink_file)(struct thread_data *, struct fio_file *);
        int (*get_file_size)(struct thread_data *, struct fio_file *);
+       int (*prepopulate_file)(struct thread_data *, struct fio_file *);
        void (*terminate)(struct thread_data *);
        int (*iomem_alloc)(struct thread_data *, size_t);
        void (*iomem_free)(struct thread_data *);