2 * Native Windows async IO engine
3 * Copyright (C) 2011 Bruce Cran <bruce@cran.org.uk>
15 typedef BOOL (WINAPI *CANCELIOEX)(HANDLE hFile, LPOVERLAPPED lpOverlapped);
22 struct windowsaio_data {
24 unsigned int io_index;
28 HANDLE iothread_stopped;
29 BOOL iothread_running;
31 struct io_u **aio_events;
32 HANDLE iocomplete_event;
33 CANCELIOEX pCancelIoEx;
39 struct windowsaio_data *wd;
42 static void PrintError(LPCSTR lpszFunction);
43 static int fio_windowsaio_cancel(struct thread_data *td,
45 static BOOL TimedOut(DWORD start_count, DWORD end_count);
46 static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min,
47 unsigned int max, struct timespec *t);
48 static struct io_u *fio_windowsaio_event(struct thread_data *td, int event);
49 static int fio_windowsaio_queue(struct thread_data *td,
51 static void fio_windowsaio_cleanup(struct thread_data *td);
52 static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter);
53 static int fio_windowsaio_init(struct thread_data *td);
54 static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f);
55 static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct fio_file *f);
57 int sync_file_range(int fd, off64_t offset, off64_t nbytes,
64 static void PrintError(LPCSTR lpszFunction)
66 // Retrieve the system error message for the last-error code
69 DWORD dw = GetLastError();
72 FORMAT_MESSAGE_ALLOCATE_BUFFER |
73 FORMAT_MESSAGE_FROM_SYSTEM |
74 FORMAT_MESSAGE_IGNORE_INSERTS,
77 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
81 log_err("%s - %s", lpszFunction, lpMsgBuf);
85 static int fio_windowsaio_cancel(struct thread_data *td,
90 struct windowsaio_data *wd = td->io_ops->data;
92 /* If we're running on Vista or newer, we can cancel individual IO requests */
93 if (wd->pCancelIoEx != NULL) {
94 FIO_OVERLAPPED *ovl = io_u->engine_data;
95 if (!wd->pCancelIoEx(io_u->file->hFile, &ovl->o))
103 static BOOL TimedOut(DWORD start_count, DWORD end_count)
105 BOOL expired = FALSE;
108 current_time = GetTickCount();
110 if ((end_count > start_count) && current_time >= end_count)
112 else if (current_time < start_count && current_time > end_count)
118 static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min,
119 unsigned int max, struct timespec *t)
121 struct windowsaio_data *wd = td->io_ops->data;
122 struct flist_head *entry;
123 unsigned int dequeued = 0;
125 DWORD start_count = 0, end_count = 0;
126 BOOL timedout = FALSE;
127 unsigned int mswait = 100;
130 mswait = (t->tv_sec * 1000) + (t->tv_nsec / 1000000);
131 start_count = GetTickCount();
132 end_count = start_count + (t->tv_sec * 1000) + (t->tv_nsec / 1000000);
135 while (dequeued < min && !timedout) {
136 flist_for_each(entry, &td->io_u_busylist) {
137 io_u = flist_entry(entry, struct io_u, list);
139 if (io_u->seen == 0) {
141 wd->aio_events[dequeued] = io_u;
150 WaitForSingleObject(wd->iocomplete_event, mswait);
152 if (t != NULL && TimedOut(start_count, end_count))
159 static struct io_u *fio_windowsaio_event(struct thread_data *td, int event)
161 struct windowsaio_data *wd = td->io_ops->data;
162 return wd->aio_events[event];
165 static int fio_windowsaio_queue(struct thread_data *td,
168 struct windowsaio_data *wd;
175 fio_ro_check(td, io_u);
177 wd = td->io_ops->data;
180 ResetEvent(wd->io_handles[ind]);
183 lpOvl = &wd->ovls[ind].o;
185 lpOvl->Internal = STATUS_PENDING;
186 lpOvl->InternalHigh = 0;
187 lpOvl->Offset = io_u->offset & 0xFFFFFFFF;
188 lpOvl->OffsetHigh = io_u->offset >> 32;
189 lpOvl->hEvent = wd->io_handles[ind];
190 lpOvl->Pointer = NULL;
191 wd->ovls[ind].io_u = io_u;
196 io_u->engine_data = &wd->ovls[ind];
199 if (io_u->ddir == DDIR_WRITE) {
200 success = WriteFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, &iobytes, lpOvl);
201 } else if (io_u->ddir == DDIR_READ) {
202 success = ReadFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, &iobytes, lpOvl);
203 } else if (io_u->ddir == DDIR_SYNC ||
204 io_u->ddir == DDIR_DATASYNC ||
205 io_u->ddir == DDIR_SYNC_FILE_RANGE)
207 FlushFileBuffers(io_u->file->hFile);
208 return FIO_Q_COMPLETED;
209 } else if (io_u->ddir == DDIR_TRIM) {
210 log_info("Manual TRIM isn't supported on Windows");
211 return FIO_Q_COMPLETED;
215 if (wd->useIOCP && (success || GetLastError() == ERROR_IO_PENDING)) {
216 wd->io_index = (wd->io_index + 1) % td->o.iodepth;
218 } else if (success && !wd->useIOCP) {
219 io_u->resid = io_u->xfer_buflen - iobytes;
221 rc = FIO_Q_COMPLETED;
223 PrintError(__func__);
224 io_u->error = GetLastError();
225 io_u->resid = io_u->xfer_buflen;
226 rc = FIO_Q_COMPLETED;
232 static void fio_windowsaio_cleanup(struct thread_data *td)
235 struct windowsaio_data *wd;
237 wd = td->io_ops->data;
239 WaitForSingleObject(wd->iothread_stopped, INFINITE);
242 CloseHandle(wd->iothread);
243 CloseHandle(wd->iothread_stopped);
244 CloseHandle(wd->iocomplete_event);
246 for (i = 0; i < td->o.iodepth; i++) {
247 CloseHandle(wd->io_handles[i]);
250 free(wd->aio_events);
251 free(wd->io_handles);
255 td->io_ops->data = NULL;
259 /* Runs as a thread and waits for queued IO to complete */
260 static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter)
265 struct windowsaio_data *wd;
266 struct thread_ctx *ctx;
270 ctx = (struct thread_ctx*)lpParameter;
274 if (!GetQueuedCompletionStatus(ctx->iocp, &bytes, &ulKey, &ovl, 250))
277 fov = CONTAINING_RECORD(ovl, FIO_OVERLAPPED, o);
280 if (ovl->Internal == ERROR_SUCCESS) {
281 io_u->resid = io_u->xfer_buflen - ovl->InternalHigh;
284 io_u->resid = io_u->xfer_buflen;
285 io_u->error = ovl->Internal;
288 SetEvent(wd->iocomplete_event);
289 } while (ctx->wd->iothread_running);
291 CloseHandle(ctx->iocp);
292 SetEvent(ctx->wd->iothread_stopped);
298 static int fio_windowsaio_init(struct thread_data *td)
300 struct windowsaio_data *wd;
304 wd = malloc(sizeof(struct windowsaio_data));
306 ZeroMemory(wd, sizeof(struct windowsaio_data));
311 wd->aio_events = malloc(td->o.iodepth * sizeof(struct io_u*));
312 if (wd->aio_events == NULL)
317 wd->io_handles = malloc(td->o.iodepth * sizeof(HANDLE));
318 if (wd->io_handles == NULL)
323 wd->ovls = malloc(td->o.iodepth * sizeof(FIO_OVERLAPPED));
324 if (wd->ovls == NULL)
329 /* Create an auto-reset event */
330 wd->iocomplete_event = CreateEvent(NULL, FALSE, FALSE, NULL);
331 if (wd->iocomplete_event == NULL)
336 PrintError(__func__);
338 if (wd->ovls != NULL)
340 if (wd->io_handles != NULL)
341 free(wd->io_handles);
342 if (wd->aio_events != NULL)
343 free(wd->aio_events);
349 hKernel32Dll = GetModuleHandle("kernel32.dll");
350 wd->pCancelIoEx = GetProcAddress(hKernel32Dll, "CancelIoEx");
352 td->io_ops->data = wd;
356 static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
360 DWORD flags = FILE_FLAG_POSIX_SEMANTICS;
361 DWORD sharemode = FILE_SHARE_READ | FILE_SHARE_WRITE;
362 DWORD openmode = OPEN_ALWAYS;
366 dprint(FD_FILE, "fd open %s\n", f->file_name);
368 if (f->filetype == FIO_TYPE_PIPE) {
369 log_err("fio: windowsaio doesn't support pipes\n");
373 if (!strcmp(f->file_name, "-")) {
374 log_err("fio: can't read/write to stdin/out\n");
378 if (!td->o.odirect && !td->o.sync_io && td->io_ops->data != NULL)
379 flags |= FILE_FLAG_OVERLAPPED;
382 flags |= FILE_FLAG_NO_BUFFERING;
384 flags |= FILE_FLAG_WRITE_THROUGH;
387 if (td->o.td_ddir == TD_DDIR_READ ||
388 td->o.td_ddir == TD_DDIR_WRITE)
389 flags |= FILE_FLAG_SEQUENTIAL_SCAN;
391 flags |= FILE_FLAG_RANDOM_ACCESS;
393 if (!td_write(td) || read_only)
394 access = GENERIC_READ;
396 access = (GENERIC_READ | GENERIC_WRITE);
398 if (td->o.create_on_open > 0)
399 openmode = OPEN_ALWAYS;
401 openmode = OPEN_EXISTING;
403 f->hFile = CreateFile(f->file_name, access, sharemode,
404 NULL, openmode, flags, NULL);
406 if (f->hFile == INVALID_HANDLE_VALUE) {
407 PrintError(__func__);
411 /* Only set up the competion port and thread if we're not just
412 * querying the device size */
413 if (!rc && td->io_ops->data != NULL && !td->o.odirect && !td->o.sync_io) {
414 struct thread_ctx *ctx;
415 struct windowsaio_data *wd;
416 hFile = CreateIoCompletionPort(f->hFile, NULL, 0, 0);
419 wd = td->io_ops->data;
421 if (!td->o.odirect && !td->o.sync_io)
427 wd->iothread_running = TRUE;
428 /* Create a manual-reset event */
429 wd->iothread_stopped = CreateEvent(NULL, TRUE, FALSE, NULL);
431 if (wd->iothread_stopped == NULL)
435 for (i = 0; i < td->o.iodepth; i++) {
436 /* Create a manual-reset event for putting in OVERLAPPED */
437 wd->io_handles[i] = CreateEvent(NULL, TRUE, FALSE, NULL);
438 if (wd->io_handles[i] == NULL) {
439 PrintError(__func__);
447 ctx = malloc(sizeof(struct thread_ctx));
451 wd->iothread = CreateThread(NULL, 0, IoCompletionRoutine, ctx, 0, NULL);
454 if (rc || wd->iothread == NULL) {
455 PrintError(__func__);
463 static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct fio_file *f)
465 struct windowsaio_data *wd;
467 dprint(FD_FILE, "fd close %s\n", f->file_name);
469 if (td->io_ops->data != NULL) {
470 wd = td->io_ops->data;
471 wd->iothread_running = FALSE;
472 WaitForSingleObject(wd->iothread_stopped, INFINITE);
475 if (f->hFile != INVALID_HANDLE_VALUE) {
476 if (!CloseHandle(f->hFile))
477 PrintError(__func__);
480 f->hFile = INVALID_HANDLE_VALUE;
484 static struct ioengine_ops ioengine = {
485 .name = "windowsaio",
486 .version = FIO_IOOPS_VERSION,
487 .init = fio_windowsaio_init,
488 .queue = fio_windowsaio_queue,
489 .cancel = fio_windowsaio_cancel,
490 .getevents = fio_windowsaio_getevents,
491 .event = fio_windowsaio_event,
492 .cleanup = fio_windowsaio_cleanup,
493 .open_file = fio_windowsaio_open_file,
494 .close_file = fio_windowsaio_close_file,
495 .get_file_size = generic_get_file_size
498 static void fio_init fio_posixaio_register(void)
500 register_ioengine(&ioengine);
503 static void fio_exit fio_posixaio_unregister(void)
505 unregister_ioengine(&ioengine);