Commit | Line | Data |
---|---|---|
ebac4655 JA |
1 | /* |
2 | * The io parts of the fio tool, includes workers for sync and mmap'ed | |
3 | * io, as well as both posix and linux libaio support. | |
4 | * | |
5 | * sync io is implemented on top of aio. | |
6 | * | |
7 | * This is not really specific to fio, if the get_io_u/put_io_u and | |
8 | * structures was pulled into this as well it would be a perfectly | |
9 | * generic io engine that could be used for other projects. | |
10 | * | |
11 | */ | |
12 | #include <stdio.h> | |
13 | #include <stdlib.h> | |
14 | #include <unistd.h> | |
5c4e1dbc | 15 | #include <string.h> |
2866c82d | 16 | #include <dlfcn.h> |
0c6e7517 | 17 | #include <assert.h> |
8c16d840 | 18 | |
ebac4655 | 19 | #include "fio.h" |
7c9b1bce | 20 | #include "diskutil.h" |
ebac4655 | 21 | |
01743ee1 | 22 | static FLIST_HEAD(engine_list); |
5f350952 | 23 | |
8c16d840 JA |
24 | static int check_engine_ops(struct ioengine_ops *ops) |
25 | { | |
5f350952 | 26 | if (ops->version != FIO_IOOPS_VERSION) { |
5ec10eaa JA |
27 | log_err("bad ioops version %d (want %d)\n", ops->version, |
28 | FIO_IOOPS_VERSION); | |
5f350952 JA |
29 | return 1; |
30 | } | |
31 | ||
36167d82 JA |
32 | if (!ops->queue) { |
33 | log_err("%s: no queue handler\n", ops->name); | |
34 | return 1; | |
35 | } | |
36 | ||
37 | /* | |
38 | * sync engines only need a ->queue() | |
39 | */ | |
40 | if (ops->flags & FIO_SYNCIO) | |
41 | return 0; | |
5ec10eaa | 42 | |
8c16d840 | 43 | if (!ops->event) { |
36167d82 | 44 | log_err("%s: no event handler\n", ops->name); |
8c16d840 JA |
45 | return 1; |
46 | } | |
47 | if (!ops->getevents) { | |
36167d82 | 48 | log_err("%s: no getevents handler\n", ops->name); |
8c16d840 JA |
49 | return 1; |
50 | } | |
51 | if (!ops->queue) { | |
36167d82 | 52 | log_err("%s: no queue handler\n", ops->name); |
8c16d840 JA |
53 | return 1; |
54 | } | |
5ec10eaa | 55 | |
8c16d840 JA |
56 | return 0; |
57 | } | |
58 | ||
5f350952 | 59 | void unregister_ioengine(struct ioengine_ops *ops) |
ebac4655 | 60 | { |
ee56ad50 | 61 | dprint(FD_IO, "ioengine %s unregistered\n", ops->name); |
01743ee1 JA |
62 | flist_del(&ops->list); |
63 | INIT_FLIST_HEAD(&ops->list); | |
5f350952 JA |
64 | } |
65 | ||
b2fdda43 | 66 | void register_ioengine(struct ioengine_ops *ops) |
5f350952 | 67 | { |
ee56ad50 | 68 | dprint(FD_IO, "ioengine %s registered\n", ops->name); |
01743ee1 JA |
69 | INIT_FLIST_HEAD(&ops->list); |
70 | flist_add_tail(&ops->list, &engine_list); | |
5f350952 JA |
71 | } |
72 | ||
73 | static struct ioengine_ops *find_ioengine(const char *name) | |
74 | { | |
75 | struct ioengine_ops *ops; | |
01743ee1 | 76 | struct flist_head *entry; |
ebac4655 | 77 | |
01743ee1 JA |
78 | flist_for_each(entry, &engine_list) { |
79 | ops = flist_entry(entry, struct ioengine_ops, list); | |
bc5b77a8 | 80 | if (!strcmp(name, ops->name)) |
5f350952 JA |
81 | return ops; |
82 | } | |
83 | ||
84 | return NULL; | |
85 | } | |
86 | ||
87 | static struct ioengine_ops *dlopen_ioengine(struct thread_data *td, | |
88 | const char *engine_lib) | |
89 | { | |
90 | struct ioengine_ops *ops; | |
91 | void *dlhandle; | |
92 | ||
ee56ad50 JA |
93 | dprint(FD_IO, "dload engine %s\n", engine_lib); |
94 | ||
2866c82d JA |
95 | dlerror(); |
96 | dlhandle = dlopen(engine_lib, RTLD_LAZY); | |
d4dbaaa8 | 97 | if (!dlhandle) { |
e1161c32 | 98 | td_vmsg(td, -1, dlerror(), "dlopen"); |
d4dbaaa8 JA |
99 | return NULL; |
100 | } | |
8756e4d4 | 101 | |
da51c050 JA |
102 | /* |
103 | * Unlike the included modules, external engines should have a | |
104 | * non-static ioengine structure that we can reference. | |
105 | */ | |
2866c82d | 106 | ops = dlsym(dlhandle, "ioengine"); |
d4dbaaa8 | 107 | if (!ops) { |
e1161c32 | 108 | td_vmsg(td, -1, dlerror(), "dlsym"); |
d4dbaaa8 JA |
109 | dlclose(dlhandle); |
110 | return NULL; | |
111 | } | |
8756e4d4 | 112 | |
5f350952 JA |
113 | ops->dlhandle = dlhandle; |
114 | return ops; | |
115 | } | |
116 | ||
117 | struct ioengine_ops *load_ioengine(struct thread_data *td, const char *name) | |
118 | { | |
119 | struct ioengine_ops *ops, *ret; | |
120 | char engine[16]; | |
121 | ||
ee56ad50 JA |
122 | dprint(FD_IO, "load ioengine %s\n", name); |
123 | ||
5f350952 JA |
124 | strncpy(engine, name, sizeof(engine) - 1); |
125 | ||
126 | /* | |
127 | * linux libaio has alias names, so convert to what we want | |
128 | */ | |
129 | if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) | |
130 | strcpy(engine, "libaio"); | |
131 | ||
132 | ops = find_ioengine(engine); | |
133 | if (!ops) | |
134 | ops = dlopen_ioengine(td, name); | |
135 | ||
136 | if (!ops) { | |
137 | log_err("fio: engine %s not loadable\n", name); | |
b902ceb5 JA |
138 | return NULL; |
139 | } | |
140 | ||
8c16d840 JA |
141 | /* |
142 | * Check that the required methods are there. | |
143 | */ | |
5f350952 | 144 | if (check_engine_ops(ops)) |
8c16d840 | 145 | return NULL; |
8c16d840 | 146 | |
84585003 JA |
147 | ret = malloc(sizeof(*ret)); |
148 | memcpy(ret, ops, sizeof(*ret)); | |
149 | ret->data = NULL; | |
84585003 JA |
150 | |
151 | return ret; | |
8756e4d4 JA |
152 | } |
153 | ||
2866c82d | 154 | void close_ioengine(struct thread_data *td) |
8756e4d4 | 155 | { |
ee56ad50 JA |
156 | dprint(FD_IO, "close ioengine %s\n", td->io_ops->name); |
157 | ||
2992b059 | 158 | if (td->io_ops->cleanup) { |
2866c82d | 159 | td->io_ops->cleanup(td); |
2992b059 JA |
160 | td->io_ops->data = NULL; |
161 | } | |
b990b5c0 | 162 | |
5f350952 JA |
163 | if (td->io_ops->dlhandle) |
164 | dlclose(td->io_ops->dlhandle); | |
165 | ||
84585003 JA |
166 | free(td->io_ops); |
167 | td->io_ops = NULL; | |
b990b5c0 | 168 | } |
10ba535a JA |
169 | |
170 | int td_io_prep(struct thread_data *td, struct io_u *io_u) | |
171 | { | |
ee56ad50 | 172 | dprint_io_u(io_u, "prep"); |
7101d9c2 JA |
173 | fio_ro_check(td, io_u); |
174 | ||
4d4e80f2 | 175 | lock_file(td, io_u->file, io_u->ddir); |
b2bd2bd9 | 176 | |
2ba1c290 JA |
177 | if (td->io_ops->prep) { |
178 | int ret = td->io_ops->prep(td, io_u); | |
179 | ||
180 | dprint(FD_IO, "->prep(%p)=%d\n", io_u, ret); | |
b2bd2bd9 | 181 | if (ret) |
4d4e80f2 | 182 | unlock_file(td, io_u->file); |
2ba1c290 JA |
183 | return ret; |
184 | } | |
10ba535a JA |
185 | |
186 | return 0; | |
187 | } | |
188 | ||
e7d2e616 | 189 | int td_io_getevents(struct thread_data *td, unsigned int min, unsigned int max, |
10ba535a JA |
190 | struct timespec *t) |
191 | { | |
ee56ad50 | 192 | int r = 0; |
face81b2 | 193 | |
ee56ad50 JA |
194 | if (min > 0 && td->io_ops->commit) { |
195 | r = td->io_ops->commit(td); | |
face81b2 | 196 | if (r < 0) |
ee56ad50 | 197 | goto out; |
face81b2 | 198 | } |
4950421a JA |
199 | if (max > td->cur_depth) |
200 | max = td->cur_depth; | |
201 | if (min > max) | |
202 | max = min; | |
36167d82 | 203 | |
ee56ad50 | 204 | r = 0; |
4950421a | 205 | if (max && td->io_ops->getevents) |
ee56ad50 JA |
206 | r = td->io_ops->getevents(td, min, max, t); |
207 | out: | |
838bc709 JA |
208 | if (r >= 0) |
209 | io_u_mark_complete(td, r); | |
f3e11d05 | 210 | else |
7c639b14 | 211 | td_verror(td, r, "get_events"); |
f3e11d05 | 212 | |
ee56ad50 JA |
213 | dprint(FD_IO, "getevents: %d\n", r); |
214 | return r; | |
10ba535a JA |
215 | } |
216 | ||
217 | int td_io_queue(struct thread_data *td, struct io_u *io_u) | |
218 | { | |
7e77dd02 JA |
219 | int ret; |
220 | ||
ee56ad50 | 221 | dprint_io_u(io_u, "queue"); |
7101d9c2 JA |
222 | fio_ro_check(td, io_u); |
223 | ||
0c6e7517 JA |
224 | assert((io_u->flags & IO_U_F_FLIGHT) == 0); |
225 | io_u->flags |= IO_U_F_FLIGHT; | |
226 | ||
d6aed795 | 227 | assert(fio_file_open(io_u->file)); |
3d7b485f | 228 | |
11786802 JA |
229 | io_u->error = 0; |
230 | io_u->resid = 0; | |
231 | ||
433afcb4 | 232 | if (td->io_ops->flags & FIO_SYNCIO) { |
12d9d841 | 233 | if (fio_fill_issue_time(td)) |
9520ebb9 | 234 | fio_gettime(&io_u->issue_time, NULL); |
d0c15328 JA |
235 | |
236 | /* | |
237 | * only used for iolog | |
238 | */ | |
239 | if (td->o.read_iolog_file) | |
240 | memcpy(&td->last_issue, &io_u->issue_time, | |
5ec10eaa | 241 | sizeof(struct timeval)); |
433afcb4 JA |
242 | } |
243 | ||
ff58fced | 244 | if (ddir_rw(io_u->ddir)) |
755200a3 JA |
245 | td->io_issues[io_u->ddir]++; |
246 | ||
7e77dd02 | 247 | ret = td->io_ops->queue(td, io_u); |
5aeb77df | 248 | |
4d4e80f2 | 249 | unlock_file(td, io_u->file); |
b2bd2bd9 | 250 | |
cb211682 JA |
251 | /* |
252 | * Add warning for O_DIRECT so that users have an easier time | |
253 | * spotting potentially bad alignment. If this triggers for the first | |
254 | * IO, then it's likely an alignment problem or because the host fs | |
255 | * does not support O_DIRECT | |
256 | */ | |
ff58fced | 257 | if (io_u->error == EINVAL && td->io_issues[io_u->ddir & 1] == 1 && |
cb211682 JA |
258 | td->o.odirect) { |
259 | log_info("fio: first direct IO errored. File system may not " | |
260 | "support direct IO, or iomem_align= is bad.\n"); | |
261 | } | |
262 | ||
838bc709 JA |
263 | if (!td->io_ops->commit) { |
264 | io_u_mark_submit(td, 1); | |
265 | io_u_mark_complete(td, 1); | |
266 | } | |
267 | ||
d8005759 | 268 | if (ret == FIO_Q_COMPLETED) { |
ff58fced | 269 | if (ddir_rw(io_u->ddir)) { |
d8005759 JA |
270 | io_u_mark_depth(td, 1); |
271 | td->ts.total_io_u[io_u->ddir]++; | |
0d29de83 JA |
272 | } else if (io_u->ddir == DDIR_TRIM) |
273 | td->ts.total_io_u[2]++; | |
d8005759 | 274 | } else if (ret == FIO_Q_QUEUED) { |
eb7c8ae2 JA |
275 | int r; |
276 | ||
ff58fced | 277 | if (ddir_rw(io_u->ddir)) { |
d8005759 JA |
278 | td->io_u_queued++; |
279 | td->ts.total_io_u[io_u->ddir]++; | |
280 | } | |
281 | ||
282 | if (td->io_u_queued >= td->o.iodepth_batch) { | |
eb7c8ae2 JA |
283 | r = td_io_commit(td); |
284 | if (r < 0) | |
285 | return r; | |
286 | } | |
287 | } | |
cb5ab512 | 288 | |
433afcb4 | 289 | if ((td->io_ops->flags & FIO_SYNCIO) == 0) { |
12d9d841 | 290 | if (fio_fill_issue_time(td)) |
9520ebb9 | 291 | fio_gettime(&io_u->issue_time, NULL); |
d0c15328 JA |
292 | |
293 | /* | |
294 | * only used for iolog | |
295 | */ | |
296 | if (td->o.read_iolog_file) | |
297 | memcpy(&td->last_issue, &io_u->issue_time, | |
298 | sizeof(struct timeval)); | |
433afcb4 JA |
299 | } |
300 | ||
7e77dd02 | 301 | return ret; |
10ba535a | 302 | } |
8c16d840 JA |
303 | |
304 | int td_io_init(struct thread_data *td) | |
305 | { | |
eeb12160 | 306 | int ret = 0; |
8c16d840 | 307 | |
eeb12160 JA |
308 | if (td->io_ops->init) { |
309 | ret = td->io_ops->init(td); | |
5ec10eaa JA |
310 | if (ret && td->o.iodepth > 1) { |
311 | log_err("fio: io engine init failed. Perhaps try" | |
312 | " reducing io depth?\n"); | |
313 | } | |
eeb12160 JA |
314 | } |
315 | ||
316 | return ret; | |
8c16d840 | 317 | } |
755200a3 JA |
318 | |
319 | int td_io_commit(struct thread_data *td) | |
320 | { | |
f3e11d05 JA |
321 | int ret; |
322 | ||
ee56ad50 JA |
323 | dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth); |
324 | ||
d8005759 | 325 | if (!td->cur_depth || !td->io_u_queued) |
e1161c32 | 326 | return 0; |
cb5ab512 | 327 | |
d8005759 | 328 | io_u_mark_depth(td, td->io_u_queued); |
cb5ab512 | 329 | td->io_u_queued = 0; |
d8005759 | 330 | |
f3e11d05 JA |
331 | if (td->io_ops->commit) { |
332 | ret = td->io_ops->commit(td); | |
333 | if (ret) | |
334 | td_verror(td, -ret, "io commit"); | |
335 | } | |
755200a3 JA |
336 | |
337 | return 0; | |
338 | } | |
b5af8293 JA |
339 | |
340 | int td_io_open_file(struct thread_data *td, struct fio_file *f) | |
341 | { | |
22a57ba8 JA |
342 | assert(!fio_file_open(f)); |
343 | assert(f->fd == -1); | |
344 | ||
413d6693 JA |
345 | if (td->io_ops->open_file(td, f)) { |
346 | if (td->error == EINVAL && td->o.odirect) | |
347 | log_err("fio: destination does not support O_DIRECT\n"); | |
5ec10eaa JA |
348 | if (td->error == EMFILE) { |
349 | log_err("fio: try reducing/setting openfiles (failed" | |
350 | " at %u of %u)\n", td->nr_open_files, | |
351 | td->o.nr_files); | |
352 | } | |
413d6693 | 353 | |
22a57ba8 JA |
354 | assert(f->fd == -1); |
355 | assert(!fio_file_open(f)); | |
413d6693 JA |
356 | return 1; |
357 | } | |
358 | ||
d5707a35 | 359 | fio_file_reset(f); |
d6aed795 JA |
360 | fio_file_set_open(f); |
361 | fio_file_clear_closing(f); | |
c97bd0fa | 362 | disk_util_inc(f->du); |
d5707a35 JA |
363 | |
364 | td->nr_open_files++; | |
365 | get_file(f); | |
366 | ||
66159828 JA |
367 | if (f->filetype == FIO_TYPE_PIPE) { |
368 | if (td_random(td)) { | |
369 | log_err("fio: can't seek on pipes (no random io)\n"); | |
370 | goto err; | |
371 | } | |
372 | } | |
373 | ||
413d6693 JA |
374 | if (td->io_ops->flags & FIO_DISKLESSIO) |
375 | goto done; | |
376 | ||
377 | if (td->o.invalidate_cache && file_invalidate_cache(td, f)) | |
378 | goto err; | |
379 | ||
66159828 JA |
380 | if (td->o.fadvise_hint && |
381 | (f->filetype == FIO_TYPE_BD || f->filetype == FIO_TYPE_FILE)) { | |
413d6693 JA |
382 | int flags; |
383 | ||
384 | if (td_random(td)) | |
385 | flags = POSIX_FADV_RANDOM; | |
386 | else | |
387 | flags = POSIX_FADV_SEQUENTIAL; | |
388 | ||
389 | if (fadvise(f->fd, f->file_offset, f->io_size, flags) < 0) { | |
390 | td_verror(td, errno, "fadvise"); | |
391 | goto err; | |
392 | } | |
7bb48f84 | 393 | } |
a978ba68 | 394 | |
e116f2b9 JA |
395 | #ifdef FIO_OS_DIRECTIO |
396 | /* | |
397 | * Some OS's have a distinct call to mark the file non-buffered, | |
398 | * instead of using O_DIRECT (Solaris) | |
399 | */ | |
400 | if (td->o.odirect) { | |
401 | int ret = fio_set_odirect(f->fd); | |
402 | ||
403 | if (ret) { | |
404 | td_verror(td, ret, "fio_set_odirect"); | |
405 | goto err; | |
406 | } | |
407 | } | |
408 | #endif | |
409 | ||
413d6693 | 410 | done: |
f29b25a3 | 411 | log_file(td, f, FIO_LOG_OPEN_FILE); |
413d6693 JA |
412 | return 0; |
413 | err: | |
c97bd0fa | 414 | disk_util_dec(f->du); |
b284075a JA |
415 | if (td->io_ops->close_file) |
416 | td->io_ops->close_file(td, f); | |
7bb48f84 | 417 | return 1; |
b5af8293 JA |
418 | } |
419 | ||
6977bcd0 | 420 | int td_io_close_file(struct thread_data *td, struct fio_file *f) |
b5af8293 | 421 | { |
d6aed795 | 422 | if (!fio_file_closing(f)) |
f29b25a3 JA |
423 | log_file(td, f, FIO_LOG_CLOSE_FILE); |
424 | ||
0ad920e7 JA |
425 | /* |
426 | * mark as closing, do real close when last io on it has completed | |
427 | */ | |
d6aed795 | 428 | fio_file_set_closing(f); |
0ad920e7 | 429 | |
c97bd0fa | 430 | disk_util_dec(f->du); |
4d4e80f2 | 431 | unlock_file_all(td, f); |
29c1349f | 432 | |
6977bcd0 | 433 | return put_file(td, f); |
b5af8293 | 434 | } |
df9c26b1 JA |
435 | |
436 | int td_io_get_file_size(struct thread_data *td, struct fio_file *f) | |
437 | { | |
438 | if (!td->io_ops->get_file_size) | |
439 | return 0; | |
440 | ||
441 | return td->io_ops->get_file_size(td, f); | |
442 | } | |
44f29692 | 443 | |
0a28ecda | 444 | static int do_sync_file_range(struct thread_data *td, struct fio_file *f) |
44f29692 JA |
445 | { |
446 | off64_t offset, nbytes; | |
447 | ||
448 | offset = f->first_write; | |
449 | nbytes = f->last_write - f->first_write; | |
450 | ||
3843deb3 JA |
451 | if (!nbytes) |
452 | return 0; | |
44f29692 | 453 | |
3843deb3 | 454 | return sync_file_range(f->fd, offset, nbytes, td->o.sync_file_range); |
44f29692 | 455 | } |
0a28ecda JA |
456 | |
457 | int do_io_u_sync(struct thread_data *td, struct io_u *io_u) | |
458 | { | |
459 | int ret; | |
460 | ||
461 | if (io_u->ddir == DDIR_SYNC) { | |
462 | ret = fsync(io_u->file->fd); | |
463 | } else if (io_u->ddir == DDIR_DATASYNC) { | |
464 | #ifdef FIO_HAVE_FDATASYNC | |
465 | ret = fdatasync(io_u->file->fd); | |
466 | #else | |
467 | ret = io_u->xfer_buflen; | |
468 | io_u->error = EINVAL; | |
469 | #endif | |
470 | } else if (io_u->ddir == DDIR_SYNC_FILE_RANGE) | |
471 | ret = do_sync_file_range(td, io_u->file); | |
472 | else { | |
473 | ret = io_u->xfer_buflen; | |
474 | io_u->error = EINVAL; | |
475 | } | |
476 | ||
cb849a79 JA |
477 | if (ret < 0) |
478 | io_u->error = errno; | |
479 | ||
0a28ecda JA |
480 | return ret; |
481 | } | |
a5f3027c JA |
482 | |
483 | int do_io_u_trim(struct thread_data *td, struct io_u *io_u) | |
484 | { | |
485 | #ifndef FIO_HAVE_TRIM | |
486 | io_u->error = EINVAL; | |
ff58fced | 487 | return 0; |
a5f3027c JA |
488 | #else |
489 | struct fio_file *f = io_u->file; | |
490 | int ret; | |
491 | ||
c6404a44 | 492 | ret = os_trim(f->fd, io_u->offset, io_u->xfer_buflen); |
a5f3027c | 493 | if (!ret) |
ff58fced | 494 | return io_u->xfer_buflen;; |
a5f3027c | 495 | |
ff58fced JA |
496 | io_u->error = ret; |
497 | return 0; | |
a5f3027c JA |
498 | #endif |
499 | } |