2 * Native Windows async IO engine
3 * Copyright (C) 2010 Bruce Cran <bruce@cran.org.uk>
20 struct windowsaio_data {
22 unsigned int io_index;
26 HANDLE iothread_stopped;
27 BOOL iothread_running;
29 struct io_u **aio_events;
30 HANDLE iocomplete_event;
36 struct windowsaio_data *wd;
39 static void PrintError(LPCSTR lpszFunction);
40 static int fio_windowsaio_cancel(struct thread_data *td,
42 static BOOL TimedOut(DWORD start_count, DWORD end_count);
43 static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min,
44 unsigned int max, struct timespec *t);
45 static struct io_u *fio_windowsaio_event(struct thread_data *td, int event);
46 static int fio_windowsaio_queue(struct thread_data *td,
48 static void fio_windowsaio_cleanup(struct thread_data *td);
49 static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter);
50 static int fio_windowsaio_init(struct thread_data *td);
51 static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f);
52 static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct fio_file *f);
54 /* CancelIoEx isn't in Cygwin's w32api */
55 BOOL WINAPI CancelIoEx(
57 LPOVERLAPPED lpOverlapped
62 int sync_file_range(int fd, off64_t offset, off64_t nbytes,
69 static void PrintError(LPCSTR lpszFunction)
71 // Retrieve the system error message for the last-error code
74 DWORD dw = GetLastError();
77 FORMAT_MESSAGE_ALLOCATE_BUFFER |
78 FORMAT_MESSAGE_FROM_SYSTEM |
79 FORMAT_MESSAGE_IGNORE_INSERTS,
82 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
86 log_err("%s - %s", lpszFunction, lpMsgBuf);
90 static int fio_windowsaio_cancel(struct thread_data *td,
95 struct windowsaio_data *wd = td->io_ops->data;
97 /* If we're running on Vista, we can cancel individual IO requests */
98 if (wd->have_cancelioex) {
99 FIO_OVERLAPPED *ovl = io_u->engine_data;
100 if (!CancelIoEx(io_u->file->hFile, &ovl->o))
108 static BOOL TimedOut(DWORD start_count, DWORD end_count)
110 BOOL expired = FALSE;
113 current_time = GetTickCount();
115 if ((end_count > start_count) && current_time >= end_count)
117 else if (current_time < start_count && current_time > end_count)
123 static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min,
124 unsigned int max, struct timespec *t)
126 struct windowsaio_data *wd = td->io_ops->data;
127 struct flist_head *entry;
128 unsigned int dequeued = 0;
130 DWORD start_count = 0, end_count = 0;
131 BOOL timedout = FALSE;
132 unsigned int mswait = 100;
135 mswait = (t->tv_sec * 1000) + (t->tv_nsec / 1000000);
136 start_count = GetTickCount();
137 end_count = start_count + (t->tv_sec * 1000) + (t->tv_nsec / 1000000);
140 while (dequeued < min && !timedout) {
141 flist_for_each(entry, &td->io_u_busylist) {
142 io_u = flist_entry(entry, struct io_u, list);
144 if (io_u->seen == 1) {
146 wd->aio_events[dequeued] = io_u;
154 if (dequeued < min) {
155 WaitForSingleObject(wd->iocomplete_event, mswait);
158 if (t != NULL && TimedOut(start_count, end_count))
165 static struct io_u *fio_windowsaio_event(struct thread_data *td, int event)
167 struct windowsaio_data *wd = td->io_ops->data;
168 return wd->aio_events[event];
171 static int fio_windowsaio_queue(struct thread_data *td,
174 struct windowsaio_data *wd;
180 fio_ro_check(td, io_u);
182 wd = td->io_ops->data;
185 ResetEvent(wd->io_handles[ind]);
186 wd->ovls[ind].o.Internal = 0;
187 wd->ovls[ind].o.InternalHigh = 0;
188 wd->ovls[ind].o.Offset = io_u->offset & 0xFFFFFFFF;
189 wd->ovls[ind].o.OffsetHigh = io_u->offset >> 32;
190 wd->ovls[ind].o.hEvent = wd->io_handles[ind];
191 wd->ovls[ind].io_u = io_u;
193 io_u->engine_data = &wd->ovls[ind];
196 if (io_u->ddir == DDIR_WRITE) {
197 success = WriteFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, &iobytes, &wd->ovls[ind].o);
198 } else if (io_u->ddir == DDIR_READ) {
199 success = ReadFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, &iobytes, &wd->ovls[ind].o);
200 } else if (io_u->ddir == DDIR_SYNC ||
201 io_u->ddir == DDIR_DATASYNC ||
202 io_u->ddir == DDIR_SYNC_FILE_RANGE)
204 FlushFileBuffers(io_u->file->hFile);
205 return FIO_Q_COMPLETED;
206 } else if (io_u->ddir == DDIR_TRIM) {
207 log_info("explicit TRIM isn't supported on Windows");
208 return FIO_Q_COMPLETED;
214 io_u->resid = io_u->xfer_buflen - iobytes;
216 rc = FIO_Q_COMPLETED;
217 } else if (!success && GetLastError() == ERROR_IO_PENDING) {
218 wd->io_index = (wd->io_index + 1) % (2 * td->o.iodepth);
221 PrintError(__func__);
222 io_u->error = GetLastError();
223 io_u->resid = io_u->xfer_buflen;
224 rc = FIO_Q_COMPLETED;
230 static void fio_windowsaio_cleanup(struct thread_data *td)
233 struct windowsaio_data *wd;
235 wd = td->io_ops->data;
237 WaitForSingleObject(wd->iothread_stopped, INFINITE);
240 CloseHandle(wd->iothread);
241 CloseHandle(wd->iothread_stopped);
242 CloseHandle(wd->iocomplete_event);
244 for (i = 0; i < 2 * td->o.iodepth; i++) {
245 CloseHandle(wd->io_handles[i]);
248 free(wd->aio_events);
249 free(wd->io_handles);
253 td->io_ops->data = NULL;
257 /* Runs as a thread and waits for queued IO to complete */
258 static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter)
263 struct windowsaio_data *wd;
264 struct thread_ctx *ctx;
268 ctx = (struct thread_ctx*)lpParameter;
271 while (ctx->wd->iothread_running) {
272 if (!GetQueuedCompletionStatus(ctx->iocp, &bytes, &ulKey, &ovl, 250))
275 fov = CONTAINING_RECORD(ovl, FIO_OVERLAPPED, o);
281 if (ovl->Internal == ERROR_SUCCESS) {
282 io_u->resid = io_u->xfer_buflen - ovl->InternalHigh;
285 io_u->resid = io_u->xfer_buflen;
286 io_u->error = ovl->Internal;
290 SetEvent(wd->iocomplete_event);
293 CloseHandle(ctx->iocp);
294 SetEvent(ctx->wd->iothread_stopped);
300 static int fio_windowsaio_init(struct thread_data *td)
302 struct windowsaio_data *wd;
303 OSVERSIONINFO osInfo;
306 wd = malloc(sizeof(struct windowsaio_data));
308 ZeroMemory(wd, sizeof(struct windowsaio_data));
313 wd->aio_events = malloc(td->o.iodepth * sizeof(struct io_u*));
314 if (wd->aio_events == NULL)
319 wd->io_handles = malloc(2 * td->o.iodepth * sizeof(HANDLE));
320 if (wd->io_handles == NULL)
325 wd->ovls = malloc(2 * td->o.iodepth * sizeof(FIO_OVERLAPPED));
326 if (wd->ovls == NULL)
331 /* Create an auto-reset event */
332 wd->iocomplete_event = CreateEvent(NULL, FALSE, FALSE, NULL);
333 if (wd->iocomplete_event == NULL)
338 osInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
339 GetVersionEx(&osInfo);
341 if (osInfo.dwMajorVersion >= 6)
342 wd->have_cancelioex = TRUE;
344 wd->have_cancelioex = FALSE;
348 PrintError(__func__);
350 if (wd->ovls != NULL)
352 if (wd->io_handles != NULL)
353 free(wd->io_handles);
354 if (wd->aio_events != NULL)
355 free(wd->aio_events);
361 td->io_ops->data = wd;
365 static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
369 DWORD flags = FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_POSIX_SEMANTICS | FILE_FLAG_OVERLAPPED;
370 DWORD sharemode = FILE_SHARE_READ | FILE_SHARE_WRITE;
371 DWORD openmode = OPEN_ALWAYS;
375 dprint(FD_FILE, "fd open %s\n", f->file_name);
377 if (f->filetype == FIO_TYPE_PIPE) {
378 log_err("fio: windowsaio doesn't support pipes\n");
382 if (!strcmp(f->file_name, "-")) {
383 log_err("fio: can't read/write to stdin/out\n");
388 flags |= FILE_FLAG_NO_BUFFERING;
390 flags |= FILE_FLAG_WRITE_THROUGH;
393 if (td->o.td_ddir == TD_DDIR_READ ||
394 td->o.td_ddir == TD_DDIR_WRITE ||
395 td->o.td_ddir == TD_DDIR_RANDRW)
397 flags |= FILE_FLAG_SEQUENTIAL_SCAN;
401 flags |= FILE_FLAG_RANDOM_ACCESS;
404 if (td_read(td) || read_only)
405 access = GENERIC_READ;
407 access = (GENERIC_READ | GENERIC_WRITE);
409 if (td->o.create_on_open > 0)
410 openmode = OPEN_ALWAYS;
412 openmode = OPEN_EXISTING;
414 f->hFile = CreateFile(f->file_name, access, sharemode,
415 NULL, openmode, flags, NULL);
417 if (f->hFile == INVALID_HANDLE_VALUE) {
418 PrintError(__func__);
422 /* Only set up the competion port and thread if we're not just
423 * querying the device size */
424 if (!rc && td->io_ops->data != NULL) {
425 struct windowsaio_data *wd;
426 struct thread_ctx *ctx;
427 hFile = CreateIoCompletionPort(f->hFile, NULL, 0, 0);
429 wd = td->io_ops->data;
432 wd->iothread_running = TRUE;
433 /* Create a manual-reset event */
434 wd->iothread_stopped = CreateEvent(NULL, TRUE, FALSE, NULL);
436 if (wd->iothread_stopped == NULL)
440 for (i = 0; i < 2 * td->o.iodepth; i++) {
441 /* Create a manual-reset event for putting in OVERLAPPED */
442 wd->io_handles[i] = CreateEvent(NULL, TRUE, FALSE, NULL);
443 if (wd->io_handles[i] == NULL) {
444 PrintError(__func__);
452 ctx = malloc(sizeof(struct thread_ctx));
456 wd->iothread = CreateThread(NULL, 0, IoCompletionRoutine, ctx, 0, NULL);
459 if (rc || wd->iothread == NULL) {
460 PrintError(__func__);
468 static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct fio_file *f)
470 struct windowsaio_data *wd;
472 dprint(FD_FILE, "fd close %s\n", f->file_name);
474 if (td->io_ops->data != NULL) {
475 wd = td->io_ops->data;
476 wd->iothread_running = FALSE;
477 WaitForSingleObject(wd->iothread_stopped, INFINITE);
480 if (f->hFile != INVALID_HANDLE_VALUE) {
481 if (!CloseHandle(f->hFile))
482 PrintError(__func__);
485 f->hFile = INVALID_HANDLE_VALUE;
489 static struct ioengine_ops ioengine = {
490 .name = "windowsaio",
491 .version = FIO_IOOPS_VERSION,
492 .init = fio_windowsaio_init,
493 .queue = fio_windowsaio_queue,
494 .cancel = fio_windowsaio_cancel,
495 .getevents = fio_windowsaio_getevents,
496 .event = fio_windowsaio_event,
497 .cleanup = fio_windowsaio_cleanup,
498 .open_file = fio_windowsaio_open_file,
499 .close_file = fio_windowsaio_close_file,
500 .get_file_size = generic_get_file_size
503 static void fio_init fio_posixaio_register(void)
505 register_ioengine(&ioengine);
508 static void fio_exit fio_posixaio_unregister(void)
510 unregister_ioengine(&ioengine);