filesetup: add fallocate=truncate option.

Fixes #833. Provides the ability to initially "layout" the file
by ftruncating it to the desired size before performing IO. This
is mainly useful on Windows, which serializes all writes that
extend the size of a file. Using this option with a suitable
iodepth allows fio to emulate the behavior of Windows Explorer
file copy, which always truncates to the expected size before
issuing writes for performance reasons.

diff --git a/HOWTO b/HOWTO
index 64c6a033982b41c01b94cfe8b5088a64b4f36e93..88dbb03fc23859ad39ba351b3fa06de81c96512b 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -1173,6 +1173,10 @@ I/O type
                        Pre-allocate via :manpage:`fallocate(2)` with
                        FALLOC_FL_KEEP_SIZE set.
 
                        Pre-allocate via :manpage:`fallocate(2)` with
                        FALLOC_FL_KEEP_SIZE set.
 

+               **truncate**
+                       Extend file to final size via :manpage:`ftruncate(2)`
+                       instead of allocating.
+

                **0**
                        Backward-compatible alias for **none**.
 
                **0**
                        Backward-compatible alias for **none**.
 


@@ -1182,7 +1186,15 @@ I/O type
        May not be available on all supported platforms. **keep** is only available
        on Linux. If using ZFS on Solaris this cannot be set to **posix**
        because ZFS doesn't support pre-allocation. Default: **native** if any
        May not be available on all supported platforms. **keep** is only available
        on Linux. If using ZFS on Solaris this cannot be set to **posix**
        because ZFS doesn't support pre-allocation. Default: **native** if any

-       pre-allocation methods are available, **none** if not.
+       pre-allocation methods except **truncate** are available, **none** if not.
+
+       Note that using **truncate** on Windows will interact surprisingly
+       with non-sequential write patterns. When writing to a file that has
+       been extended by setting the end-of-file information, Windows will
+       backfill the unwritten portion of the file up to that offset with
+       zeroes before issuing the new write. This means that a single small
+       write to the end of an extended file will stall until the entire
+       file has been filled with zeroes.

 
 .. option:: fadvise_hint=str
 
 
 .. option:: fadvise_hint=str
 

diff --git a/file.h b/file.h
index e50c0f9c3fac752455bfbe74701fd86b04b844ad..ae0e6fc8996d02f8ca2e5a76629a3dd3cf94b00d 100644 (file)
--- a/file.h
+++ b/file.h
@@ -67,6 +67,7 @@ enum fio_fallocate_mode {
        FIO_FALLOCATE_POSIX     = 2,
        FIO_FALLOCATE_KEEP_SIZE = 3,
        FIO_FALLOCATE_NATIVE    = 4,
        FIO_FALLOCATE_POSIX     = 2,
        FIO_FALLOCATE_KEEP_SIZE = 3,
        FIO_FALLOCATE_NATIVE    = 4,

+       FIO_FALLOCATE_TRUNCATE  = 5,

 };
 
 /*
 };
 
 /*

diff --git a/filesetup.c b/filesetup.c
index 1d3094c14796509c4f26a5b928bba3975dd8dc74..b52a347e7ec3b755bd476b0a0f22fc04c809c701 100644 (file)
--- a/filesetup.c
+++ b/filesetup.c
@@ -95,6 +95,18 @@ static void fallocate_file(struct thread_data *td, struct fio_file *f)
                break;
                }
 #endif /* CONFIG_LINUX_FALLOCATE */
                break;
                }
 #endif /* CONFIG_LINUX_FALLOCATE */

+       case FIO_FALLOCATE_TRUNCATE: {
+               int r;
+
+               dprint(FD_FILE, "ftruncate file %s size %llu\n",
+                               f->file_name,
+                               (unsigned long long) f->real_file_size);
+               r = ftruncate(f->fd, f->real_file_size);
+               if (r != 0)
+                       td_verror(td, errno, "ftruncate");
+
+               break;
+       }

        default:
                log_err("fio: unknown fallocate mode: %d\n", td->o.fallocate_mode);
                assert(0);
        default:
                log_err("fio: unknown fallocate mode: %d\n", td->o.fallocate_mode);
                assert(0);

diff --git a/fio.1 b/fio.1
index 087d37786925515f997cb28c57875585d571898a..14569e9fb3dfc4d2fee552549da470768f4accfe 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -943,6 +943,10 @@ Pre-allocate via \fBposix_fallocate\fR\|(3).
 Pre-allocate via \fBfallocate\fR\|(2) with
 FALLOC_FL_KEEP_SIZE set.
 .TP
 Pre-allocate via \fBfallocate\fR\|(2) with
 FALLOC_FL_KEEP_SIZE set.
 .TP

+.B truncate
+Extend file to final size using \fBftruncate\fR|(2)
+instead of allocating.
+.TP

 .B 0
 Backward-compatible alias for \fBnone\fR.
 .TP
 .B 0
 Backward-compatible alias for \fBnone\fR.
 .TP


@@ -953,7 +957,15 @@ Backward-compatible alias for \fBposix\fR.
 May not be available on all supported platforms. \fBkeep\fR is only available
 on Linux. If using ZFS on Solaris this cannot be set to \fBposix\fR
 because ZFS doesn't support pre-allocation. Default: \fBnative\fR if any
 May not be available on all supported platforms. \fBkeep\fR is only available
 on Linux. If using ZFS on Solaris this cannot be set to \fBposix\fR
 because ZFS doesn't support pre-allocation. Default: \fBnative\fR if any

-pre-allocation methods are available, \fBnone\fR if not.
+pre-allocation methods except \fBtruncate\fR are available, \fBnone\fR if not.
+.P
+Note that using \fBtruncate\fR on Windows will interact surprisingly
+with non-sequential write patterns. When writing to a file that has
+been extended by setting the end-of-file information, Windows will
+backfill the unwritten portion of the file up to that offset with
+zeroes before issuing the new write. This means that a single small
+write to the end of an extended file will stall until the entire
+file has been filled with zeroes.

 .RE
 .TP
 .BI fadvise_hint \fR=\fPstr
 .RE
 .TP
 .BI fadvise_hint \fR=\fPstr

diff --git a/init.c b/init.c
index 63f2168eabcbbebb8cf6c2ae96967b46f2114713..60c857611d3cc26c316c545f0fafcc0b851a2693 100644 (file)
--- a/init.c
+++ b/init.c
@@ -852,11 +852,6 @@ static int fixup_options(struct thread_data *td)
                        o->unit_base = N2S_BYTEPERSEC;
        }
 
                        o->unit_base = N2S_BYTEPERSEC;
        }
 

-#ifndef FIO_HAVE_ANY_FALLOCATE
-       /* Platform doesn't support any fallocate so force it to none */
-       o->fallocate_mode = FIO_FALLOCATE_NONE;
-#endif
-

 #ifndef CONFIG_FDATASYNC
        if (o->fdatasync_blocks) {
                log_info("fio: this platform does not support fdatasync()"
 #ifndef CONFIG_FDATASYNC
        if (o->fdatasync_blocks) {
                log_info("fio: this platform does not support fdatasync()"

diff --git a/options.c b/options.c
index 2c5bf5e06a44a384464543ff5c2f91a865fe8653..fad1857e20fbfb34eb1c804981b17d86bf4b43fc 100644 (file)
--- a/options.c
+++ b/options.c
@@ -2412,14 +2412,17 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .parent = "nrfiles",
                .hide   = 1,
        },
                .parent = "nrfiles",
                .hide   = 1,
        },

-#ifdef FIO_HAVE_ANY_FALLOCATE

        {
                .name   = "fallocate",
                .lname  = "Fallocate",
                .type   = FIO_OPT_STR,
                .off1   = offsetof(struct thread_options, fallocate_mode),
                .help   = "Whether pre-allocation is performed when laying out files",
        {
                .name   = "fallocate",
                .lname  = "Fallocate",
                .type   = FIO_OPT_STR,
                .off1   = offsetof(struct thread_options, fallocate_mode),
                .help   = "Whether pre-allocation is performed when laying out files",

+#ifdef FIO_HAVE_DEFAULT_FALLOCATE

                .def    = "native",
                .def    = "native",

+#else
+               .def    = "none",
+#endif

                .category = FIO_OPT_C_FILE,
                .group  = FIO_OPT_G_INVALID,
                .posval = {
                .category = FIO_OPT_C_FILE,
                .group  = FIO_OPT_G_INVALID,
                .posval = {


@@ -2443,6 +2446,10 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                            .help = "Use fallocate(..., FALLOC_FL_KEEP_SIZE, ...)",
                          },
 #endif
                            .help = "Use fallocate(..., FALLOC_FL_KEEP_SIZE, ...)",
                          },
 #endif

+                         { .ival = "truncate",
+                           .oval = FIO_FALLOCATE_TRUNCATE,
+                           .help = "Truncate file to final size instead of allocating"
+                         },

                          /* Compatibility with former boolean values */
                          { .ival = "0",
                            .oval = FIO_FALLOCATE_NONE,
                          /* Compatibility with former boolean values */
                          { .ival = "0",
                            .oval = FIO_FALLOCATE_NONE,


@@ -2456,14 +2463,6 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 #endif
                },
        },
 #endif
                },
        },

-#else  /* FIO_HAVE_ANY_FALLOCATE */
-       {
-               .name   = "fallocate",
-               .lname  = "Fallocate",
-               .type   = FIO_OPT_UNSUPPORTED,
-               .help   = "Your platform does not support fallocate",
-       },
-#endif /* FIO_HAVE_ANY_FALLOCATE */

        {
                .name   = "fadvise_hint",
                .lname  = "Fadvise hint",
        {
                .name   = "fadvise_hint",
                .lname  = "Fadvise hint",

diff --git a/os/os.h b/os/os.h
index e47296801345d312249a277445e32bb42a6477a1..dadcd87bcd9625daad0728ffd2d8ce4bbd473e47 100644 (file)
--- a/os/os.h
+++ b/os/os.h
@@ -397,7 +397,7 @@ static inline bool fio_fallocate(struct fio_file *f, uint64_t offset, uint64_t l
 #endif
 
 #if defined(CONFIG_POSIX_FALLOCATE) || defined(FIO_HAVE_NATIVE_FALLOCATE)
 #endif
 
 #if defined(CONFIG_POSIX_FALLOCATE) || defined(FIO_HAVE_NATIVE_FALLOCATE)

-# define FIO_HAVE_ANY_FALLOCATE
+# define FIO_HAVE_DEFAULT_FALLOCATE

 #endif
 
 #ifndef FIO_HAVE_CPU_HAS
 #endif
 
 #ifndef FIO_HAVE_CPU_HAS