X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=engines%2Fwindowsaio.c;h=13d7f19402a1759fa8c28f7e11b6e13686098e93;hp=c0d1bd7d44ea5d9433a9e0878a488e686f034e95;hb=ce4d13ca162df4127ec3b5911553802c53396705;hpb=c874d188c898b625bf4181f6cd2ab99c7e4aca63 diff --git a/engines/windowsaio.c b/engines/windowsaio.c index c0d1bd7d..13d7f194 100644 --- a/engines/windowsaio.c +++ b/engines/windowsaio.c @@ -1,6 +1,7 @@ /* - * Native Windows async IO engine - * Copyright (C) 2012-2013 Bruce Cran + * windowsaio engine + * + * IO engine using Windows IO Completion Ports. */ #include @@ -8,7 +9,6 @@ #include #include #include -#include #include "../fio.h" @@ -27,7 +27,6 @@ struct windowsaio_data { HANDLE iocp; HANDLE iothread; HANDLE iocomplete_event; - CANCELIOEX pCancelIoEx; BOOL iothread_running; }; @@ -36,119 +35,34 @@ struct thread_ctx { struct windowsaio_data *wd; }; -static int fio_windowsaio_cancel(struct thread_data *td, - struct io_u *io_u); -static BOOL timeout_expired(DWORD start_count, DWORD end_count); -static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min, - unsigned int max, struct timespec *t); -static struct io_u *fio_windowsaio_event(struct thread_data *td, int event); -static int fio_windowsaio_queue(struct thread_data *td, - struct io_u *io_u); -static void fio_windowsaio_cleanup(struct thread_data *td); static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter); -static int fio_windowsaio_init(struct thread_data *td); -static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f); -static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct fio_file *f); -static int win_to_posix_error(DWORD winerr); - -static int win_to_posix_error(DWORD winerr) -{ - switch (winerr) - { - case ERROR_FILE_NOT_FOUND: return ENOENT; - case ERROR_PATH_NOT_FOUND: return ENOENT; - case ERROR_ACCESS_DENIED: return EACCES; - case ERROR_INVALID_HANDLE: return EBADF; - case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; - case ERROR_INVALID_DATA: return EINVAL; - case ERROR_OUTOFMEMORY: return ENOMEM; - case ERROR_INVALID_DRIVE: return ENODEV; - case ERROR_NOT_SAME_DEVICE: return EXDEV; - case ERROR_WRITE_PROTECT: return EROFS; - case ERROR_BAD_UNIT: return ENODEV; - case ERROR_SHARING_VIOLATION: return EACCES; - case ERROR_LOCK_VIOLATION: return EACCES; - case ERROR_SHARING_BUFFER_EXCEEDED: return ENOLCK; - case ERROR_HANDLE_DISK_FULL: return ENOSPC; - case ERROR_NOT_SUPPORTED: return ENOSYS; - case ERROR_FILE_EXISTS: return EEXIST; - case ERROR_CANNOT_MAKE: return EPERM; - case ERROR_INVALID_PARAMETER: return EINVAL; - case ERROR_NO_PROC_SLOTS: return EAGAIN; - case ERROR_BROKEN_PIPE: return EPIPE; - case ERROR_OPEN_FAILED: return EIO; - case ERROR_NO_MORE_SEARCH_HANDLES: return ENFILE; - case ERROR_CALL_NOT_IMPLEMENTED: return ENOSYS; - case ERROR_INVALID_NAME: return ENOENT; - case ERROR_WAIT_NO_CHILDREN: return ECHILD; - case ERROR_CHILD_NOT_COMPLETE: return EBUSY; - case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY; - case ERROR_SIGNAL_REFUSED: return EIO; - case ERROR_BAD_PATHNAME: return ENOENT; - case ERROR_SIGNAL_PENDING: return EBUSY; - case ERROR_MAX_THRDS_REACHED: return EAGAIN; - case ERROR_BUSY: return EBUSY; - case ERROR_ALREADY_EXISTS: return EEXIST; - case ERROR_NO_SIGNAL_SENT: return EIO; - case ERROR_FILENAME_EXCED_RANGE: return EINVAL; - case ERROR_META_EXPANSION_TOO_LONG: return EINVAL; - case ERROR_INVALID_SIGNAL_NUMBER: return EINVAL; - case ERROR_THREAD_1_INACTIVE: return EINVAL; - case ERROR_BAD_PIPE: return EINVAL; - case ERROR_PIPE_BUSY: return EBUSY; - case ERROR_NO_DATA: return EPIPE; - case ERROR_MORE_DATA: return EAGAIN; - case ERROR_DIRECTORY: return ENOTDIR; - case ERROR_PIPE_CONNECTED: return EBUSY; - case ERROR_NO_TOKEN: return EINVAL; - case ERROR_PROCESS_ABORTED: return EFAULT; - case ERROR_BAD_DEVICE: return ENODEV; - case ERROR_BAD_USERNAME: return EINVAL; - case ERROR_OPEN_FILES: return EAGAIN; - case ERROR_ACTIVE_CONNECTIONS: return EAGAIN; - case ERROR_DEVICE_IN_USE: return EAGAIN; - case ERROR_INVALID_AT_INTERRUPT_TIME: return EINTR; - case ERROR_IO_DEVICE: return EIO; - case ERROR_NOT_OWNER: return EPERM; - case ERROR_END_OF_MEDIA: return ENOSPC; - case ERROR_EOM_OVERFLOW: return ENOSPC; - case ERROR_BEGINNING_OF_MEDIA: return ESPIPE; - case ERROR_SETMARK_DETECTED: return ESPIPE; - case ERROR_NO_DATA_DETECTED: return ENOSPC; - case ERROR_POSSIBLE_DEADLOCK: return EDEADLOCK; - case ERROR_CRC: return EIO; - case ERROR_NEGATIVE_SEEK: return EINVAL; - case ERROR_DISK_FULL: return ENOSPC; - case ERROR_NOACCESS: return EFAULT; - case ERROR_FILE_INVALID: return ENXIO; - } - - return winerr; -} static int fio_windowsaio_init(struct thread_data *td) { struct windowsaio_data *wd; - HANDLE hKernel32Dll; int rc = 0; - wd = malloc(sizeof(struct windowsaio_data)); - if (wd != NULL) - ZeroMemory(wd, sizeof(struct windowsaio_data)); - else + wd = calloc(1, sizeof(struct windowsaio_data)); + if (wd == NULL) { + log_err("windowsaio: failed to allocate memory for engine data\n"); rc = 1; + } if (!rc) { wd->aio_events = malloc(td->o.iodepth * sizeof(struct io_u*)); - if (wd->aio_events == NULL) + if (wd->aio_events == NULL) { + log_err("windowsaio: failed to allocate memory for aio events list\n"); rc = 1; + } } if (!rc) { /* Create an auto-reset event */ wd->iocomplete_event = CreateEvent(NULL, FALSE, FALSE, NULL); - if (wd->iocomplete_event == NULL) + if (wd->iocomplete_event == NULL) { + log_err("windowsaio: failed to create io complete event handle\n"); rc = 1; + } } if (rc) { @@ -160,10 +74,7 @@ static int fio_windowsaio_init(struct thread_data *td) } } - hKernel32Dll = GetModuleHandle("kernel32.dll"); - wd->pCancelIoEx = (CANCELIOEX)GetProcAddress(hKernel32Dll, "CancelIoEx"); - td->io_ops->data = wd; - + td->io_ops_data = wd; if (!rc) { struct thread_ctx *ctx; @@ -171,28 +82,35 @@ static int fio_windowsaio_init(struct thread_data *td) HANDLE hFile; hFile = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - if (hFile == INVALID_HANDLE_VALUE) + if (hFile == INVALID_HANDLE_VALUE) { + log_err("windowsaio: failed to create io completion port\n"); rc = 1; + } - wd = td->io_ops->data; + wd = td->io_ops_data; wd->iothread_running = TRUE; wd->iocp = hFile; if (!rc) ctx = malloc(sizeof(struct thread_ctx)); - if (!rc && ctx == NULL) - { - log_err("fio: out of memory in windowsaio\n"); + if (!rc && ctx == NULL) { + log_err("windowsaio: failed to allocate memory for thread context structure\n"); CloseHandle(hFile); rc = 1; } - if (!rc) - { + if (!rc) { + DWORD threadid; + ctx->iocp = hFile; ctx->wd = wd; - wd->iothread = CreateThread(NULL, 0, IoCompletionRoutine, ctx, 0, NULL); + wd->iothread = CreateThread(NULL, 0, IoCompletionRoutine, ctx, 0, &threadid); + + if (wd->iothread != NULL) + fio_setaffinity(threadid, td->o.cpumask); + else + log_err("windowsaio: failed to create io completion thread\n"); } if (rc || wd->iothread == NULL) @@ -206,7 +124,7 @@ static void fio_windowsaio_cleanup(struct thread_data *td) { struct windowsaio_data *wd; - wd = td->io_ops->data; + wd = td->io_ops_data; if (wd != NULL) { wd->iothread_running = FALSE; @@ -218,10 +136,47 @@ static void fio_windowsaio_cleanup(struct thread_data *td) free(wd->aio_events); free(wd); - td->io_ops->data = NULL; + td->io_ops_data = NULL; } } +static int windowsaio_invalidate_cache(struct fio_file *f) +{ + DWORD error; + DWORD isharemode = (FILE_SHARE_DELETE | FILE_SHARE_READ | + FILE_SHARE_WRITE); + HANDLE ihFile; + int rc = 0; + + /* + * Encourage Windows to drop cached parts of a file by temporarily + * opening it for non-buffered access. Note: this will only work when + * the following is the only thing with the file open on the whole + * system. + */ + dprint(FD_IO, "windowaio: attempt invalidate cache for %s\n", + f->file_name); + ihFile = CreateFile(f->file_name, 0, isharemode, NULL, OPEN_EXISTING, + FILE_FLAG_NO_BUFFERING, NULL); + + if (ihFile != INVALID_HANDLE_VALUE) { + if (!CloseHandle(ihFile)) { + error = GetLastError(); + log_info("windowsaio: invalidation fd close %s " + "failed: error %d\n", f->file_name, error); + rc = 1; + } + } else { + error = GetLastError(); + if (error != ERROR_FILE_NOT_FOUND) { + log_info("windowsaio: cache invalidation of %s failed: " + "error %d\n", f->file_name, error); + rc = 1; + } + } + + return rc; +} static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f) { @@ -234,12 +189,12 @@ static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f) dprint(FD_FILE, "fd open %s\n", f->file_name); if (f->filetype == FIO_TYPE_PIPE) { - log_err("fio: windowsaio doesn't support pipes\n"); + log_err("windowsaio: pipes are not supported\n"); return 1; } if (!strcmp(f->file_name, "-")) { - log_err("fio: can't read/write to stdin/out\n"); + log_err("windowsaio: can't read/write to stdin/out\n"); return 1; } @@ -250,13 +205,26 @@ static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f) /* * Inform Windows whether we're going to be doing sequential or - * random io so it can tune the Cache Manager + * random IO so it can tune the Cache Manager */ - if (td->o.td_ddir == TD_DDIR_READ || - td->o.td_ddir == TD_DDIR_WRITE) - flags |= FILE_FLAG_SEQUENTIAL_SCAN; - else + switch (td->o.fadvise_hint) { + case F_ADV_TYPE: + if (td_random(td)) + flags |= FILE_FLAG_RANDOM_ACCESS; + else + flags |= FILE_FLAG_SEQUENTIAL_SCAN; + break; + case F_ADV_RANDOM: flags |= FILE_FLAG_RANDOM_ACCESS; + break; + case F_ADV_SEQUENTIAL: + flags |= FILE_FLAG_SEQUENTIAL_SCAN; + break; + case F_ADV_NONE: + break; + default: + log_err("fio: unknown fadvise type %d\n", td->o.fadvise_hint); + } if (!td_write(td) || read_only) access = GENERIC_READ; @@ -268,21 +236,30 @@ static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f) else openmode = OPEN_EXISTING; + /* If we're going to use direct I/O, Windows will try and invalidate + * its cache at that point so there's no need to do it here */ + if (td->o.invalidate_cache && !td->o.odirect) + windowsaio_invalidate_cache(f); + f->hFile = CreateFile(f->file_name, access, sharemode, NULL, openmode, flags, NULL); - if (f->hFile == INVALID_HANDLE_VALUE) + if (f->hFile == INVALID_HANDLE_VALUE) { + log_err("windowsaio: failed to open file \"%s\"\n", f->file_name); rc = 1; + } /* Only set up the completion port and thread if we're not just * querying the device size */ - if (!rc && td->io_ops->data != NULL) { + if (!rc && td->io_ops_data != NULL) { struct windowsaio_data *wd; - wd = td->io_ops->data; + wd = td->io_ops_data; - if (CreateIoCompletionPort(f->hFile, wd->iocp, 0, 0) == NULL) + if (CreateIoCompletionPort(f->hFile, wd->iocp, 0, 0) == NULL) { + log_err("windowsaio: failed to create io completion port\n"); rc = 1; + } } return rc; @@ -295,8 +272,10 @@ static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct f dprint(FD_FILE, "fd close %s\n", f->file_name); if (f->hFile != INVALID_HANDLE_VALUE) { - if (!CloseHandle(f->hFile)) + if (!CloseHandle(f->hFile)) { + log_info("windowsaio: failed to close file handle for \"%s\"\n", f->file_name); rc = 1; + } } f->hFile = INVALID_HANDLE_VALUE; @@ -320,17 +299,18 @@ static BOOL timeout_expired(DWORD start_count, DWORD end_count) static struct io_u* fio_windowsaio_event(struct thread_data *td, int event) { - struct windowsaio_data *wd = td->io_ops->data; + struct windowsaio_data *wd = td->io_ops_data; return wd->aio_events[event]; } static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min, - unsigned int max, struct timespec *t) + unsigned int max, + const struct timespec *t) { - struct windowsaio_data *wd = td->io_ops->data; - struct flist_head *entry; + struct windowsaio_data *wd = td->io_ops_data; unsigned int dequeued = 0; struct io_u *io_u; + int i; struct fio_overlapped *fov; DWORD start_count = 0; DWORD end_count = 0; @@ -344,71 +324,75 @@ static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min, } do { - flist_for_each(entry, &td->io_u_busylist) { - io_u = flist_entry(entry, struct io_u, list); + io_u_qiter(&td->io_u_all, io_u, i) { + if (!(io_u->flags & IO_U_F_FLIGHT)) + continue; + fov = (struct fio_overlapped*)io_u->engine_data; if (fov->io_complete) { fov->io_complete = FALSE; - ResetEvent(fov->o.hEvent); wd->aio_events[dequeued] = io_u; dequeued++; } - if (dequeued >= min) - break; } + if (dequeued >= min) + break; if (dequeued < min) { status = WaitForSingleObject(wd->iocomplete_event, mswait); if (status != WAIT_OBJECT_0 && dequeued >= min) - break; + break; } - if (dequeued >= min || (t != NULL && timeout_expired(start_count, end_count))) + if (dequeued >= min || + (t != NULL && timeout_expired(start_count, end_count))) break; } while (1); return dequeued; } -static int fio_windowsaio_queue(struct thread_data *td, struct io_u *io_u) +static enum fio_q_status fio_windowsaio_queue(struct thread_data *td, + struct io_u *io_u) { struct fio_overlapped *o = io_u->engine_data; LPOVERLAPPED lpOvl = &o->o; - DWORD iobytes; BOOL success = FALSE; int rc = FIO_Q_COMPLETED; fio_ro_check(td, io_u); - lpOvl->Internal = STATUS_PENDING; + lpOvl->Internal = 0; lpOvl->InternalHigh = 0; lpOvl->Offset = io_u->offset & 0xFFFFFFFF; lpOvl->OffsetHigh = io_u->offset >> 32; switch (io_u->ddir) { case DDIR_WRITE: - success = WriteFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, &iobytes, lpOvl); + success = WriteFile(io_u->file->hFile, io_u->xfer_buf, + io_u->xfer_buflen, NULL, lpOvl); break; case DDIR_READ: - success = ReadFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, &iobytes, lpOvl); + success = ReadFile(io_u->file->hFile, io_u->xfer_buf, + io_u->xfer_buflen, NULL, lpOvl); break; case DDIR_SYNC: case DDIR_DATASYNC: case DDIR_SYNC_FILE_RANGE: success = FlushFileBuffers(io_u->file->hFile); - if (!success) - io_u->error = win_to_posix_error(GetLastError()); + if (!success) { + log_err("windowsaio: failed to flush file buffers\n"); + io_u->error = win_to_posix_error(GetLastError()); + } return FIO_Q_COMPLETED; - break; case DDIR_TRIM: - log_err("manual TRIM isn't supported on Windows"); + log_err("windowsaio: manual TRIM isn't supported on Windows\n"); io_u->error = 1; io_u->resid = io_u->xfer_buflen; return FIO_Q_COMPLETED; - break; default: assert(0); break; @@ -439,7 +423,11 @@ static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter) wd = ctx->wd; do { - if (!GetQueuedCompletionStatus(ctx->iocp, &bytes, &ulKey, &ovl, 250) && ovl == NULL) + BOOL ret; + + ret = GetQueuedCompletionStatus(ctx->iocp, &bytes, &ulKey, + &ovl, 250); + if (!ret && ovl == NULL) continue; fov = CONTAINING_RECORD(ovl, struct fio_overlapped, o); @@ -462,31 +450,11 @@ static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter) return 0; } -static int fio_windowsaio_cancel(struct thread_data *td, - struct io_u *io_u) -{ - int rc = 0; - - struct windowsaio_data *wd = td->io_ops->data; - - /* If we're running on Vista or newer, we can cancel individual IO requests */ - if (wd->pCancelIoEx != NULL) { - struct fio_overlapped *ovl = io_u->engine_data; - - if (!wd->pCancelIoEx(io_u->file->hFile, &ovl->o)) - rc = 1; - } else - rc = 1; - - return rc; -} - static void fio_windowsaio_io_u_free(struct thread_data *td, struct io_u *io_u) { struct fio_overlapped *o = io_u->engine_data; if (o) { - CloseHandle(o->o.hEvent); io_u->engine_data = NULL; free(o); } @@ -499,12 +467,7 @@ static int fio_windowsaio_io_u_init(struct thread_data *td, struct io_u *io_u) o = malloc(sizeof(*o)); o->io_complete = FALSE; o->io_u = io_u; - o->o.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if (!o->o.hEvent) { - free(o); - return 1; - } - + o->o.hEvent = NULL; io_u->engine_data = o; return 0; } @@ -514,7 +477,6 @@ static struct ioengine_ops ioengine = { .version = FIO_IOOPS_VERSION, .init = fio_windowsaio_init, .queue = fio_windowsaio_queue, - .cancel = fio_windowsaio_cancel, .getevents = fio_windowsaio_getevents, .event = fio_windowsaio_event, .cleanup = fio_windowsaio_cleanup,