windowsaio: add best effort cache invalidation

https://stackoverflow.com/questions/478340/clear-file-cache-to-repeat-performance-testing/7113153#7113153
mentions that opening a file without buffering on Windows has the side
effect of flushing cached parts of the file out of RAM. Use this side
effect to do cache invalidation on non-direct jobs that are using the
windowsaio engine.

This change may make default bandwidth speeds on Windows look lower
compared to older versions of fio but this matches the behaviour of fio
on other platforms with invalidation (such as Linux) because we are
trying to avoid measuring cache reuse (unless invalidate=0 is set).

Note that cache invalidation will fail to happen if the file is also
open elsewhere. Unfortunately, we can't check whether we have exclusive
access without introducing a race where other jobs/programs fail during
open because we temporarily have exclusive access to the file.

Invalidation is done as a call from within fio_windowsaio_open_file
because invalidation fails with multiple file handles (see above) and
it's simpler than trying to use ReOpenFile (which is only available from
Windows 2003/Windows Vista onwards).

The impact of invalidation is demonstrated by the bandwidths achieved by
the following jobs running on a hard disk of an otherwise idle system
with 4GBytes of RAM:
./fio --stonewall --thread --size=128M --filename=fio.tmp --iodepth=64 \
  --bs=128k --invalidate=0 --direct=0 \
  --name=create --rw=write \
  --name=cached --rw=read --loops=2 \
  --name=invalidated --rw=read --loops=2 --invalidate=1

[...]
cached: (groupid=1, jobs=1): err= 0: pid=3888: Sat Oct 7 08:05:39 2017
   read: IOPS=11.7k, BW=1463MiB/s (1534MB/s)(256MiB/175msec)
[...]
invalidated: (groupid=2, jobs=1): err= 0: pid=828: Sat Oct 7 08:05:39 2017
   read: IOPS=2089, BW=261MiB/s (274MB/s)(256MiB/980msec)

v2:
- Move invalidation into a helper function.
- Add more detail and rationale to the commit message.

Signed-off-by: Sitsofe Wheeler <sitsofe@yahoo.com>

diff --git a/engines/windowsaio.c b/engines/windowsaio.c
index 314eaadf480c485a00f4753b2cd53206b5964211..2b12ddd01f69c85258ee7739ff90ade6c038be91 100644 (file)
--- a/engines/windowsaio.c
+++ b/engines/windowsaio.c
@@ -142,6 +142,44 @@ static void fio_windowsaio_cleanup(struct thread_data *td)
        }
 }
 
+static int windowsaio_invalidate_cache(struct fio_file *f)
+{
+       DWORD error;
+       DWORD isharemode = (FILE_SHARE_DELETE | FILE_SHARE_READ |
+                       FILE_SHARE_WRITE);
+       HANDLE ihFile;
+       int rc = 0;
+
+       /*
+        * Encourage Windows to drop cached parts of a file by temporarily
+        * opening it for non-buffered access. Note: this will only work when
+        * the following is the only thing with the file open on the whole
+        * system.
+        */
+       dprint(FD_IO, "windowaio: attempt invalidate cache for %s\n",
+                       f->file_name);
+       ihFile = CreateFile(f->file_name, 0, isharemode, NULL, OPEN_EXISTING,
+                       FILE_FLAG_NO_BUFFERING, NULL);
+
+       if (ihFile != INVALID_HANDLE_VALUE) {
+               if (!CloseHandle(ihFile)) {
+                       error = GetLastError();
+                       log_info("windowsaio: invalidation fd close %s "
+                                "failed: error %d\n", f->file_name, error);
+                       rc = 1;
+               }
+       } else {
+               error = GetLastError();
+               if (error != ERROR_FILE_NOT_FOUND) {
+                       log_info("windowsaio: cache invalidation of %s failed: "
+                                       "error %d\n", f->file_name, error);
+                       rc = 1;
+               }
+       }
+
+       return rc;
+}
+
 static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
 {
        int rc = 0;
@@ -200,6 +238,12 @@ static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
        else
                openmode = OPEN_EXISTING;
 
+       /* If we're going to use direct I/O, Windows will try and invalidate
+        * its cache at that point so there's no need to do it here */
+       if (td->o.invalidate_cache && !td->o.odirect) {
+               windowsaio_invalidate_cache(f);
+       }
+
        f->hFile = CreateFile(f->file_name, access, sharemode,
                NULL, openmode, flags, NULL);