projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
cmdprio: add mode to make the logic easier to reason about
[fio.git] / engines / io_uring.c
... / ...
CommitLineData
1/*
2 * io_uring engine
3 *
4 * IO engine using the new native Linux aio io_uring interface. See:
5 *
6 * http://git.kernel.dk/cgit/linux-block/log/?h=io_uring
7 *
8 */
9#include <stdlib.h>
10#include <unistd.h>
11#include <errno.h>
12#include <sys/time.h>
13#include <sys/resource.h>
14
15#include "../fio.h"
16#include "../lib/pow2.h"
17#include "../optgroup.h"
18#include "../lib/memalign.h"
19#include "../lib/fls.h"
20#include "../lib/roundup.h"
21
22#ifdef ARCH_HAVE_IOURING
23
24#include "../lib/types.h"
25#include "../os/linux/io_uring.h"
26#include "cmdprio.h"
27
28struct io_sq_ring {
29 unsigned *head;
30 unsigned *tail;
31 unsigned *ring_mask;
32 unsigned *ring_entries;
33 unsigned *flags;
34 unsigned *array;
35};
36
37struct io_cq_ring {
38 unsigned *head;
39 unsigned *tail;
40 unsigned *ring_mask;
41 unsigned *ring_entries;
42 struct io_uring_cqe *cqes;
43};
44
45struct ioring_mmap {
46 void *ptr;
47 size_t len;
48};
49
50struct ioring_data {
51 int ring_fd;
52
53 struct io_u **io_u_index;
54
55 int *fds;
56
57 struct io_sq_ring sq_ring;
58 struct io_uring_sqe *sqes;
59 struct iovec *iovecs;
60 unsigned sq_ring_mask;
61
62 struct io_cq_ring cq_ring;
63 unsigned cq_ring_mask;
64
65 int queued;
66 int cq_ring_off;
67 unsigned iodepth;
68 int prepped;
69
70 struct ioring_mmap mmap[3];
71};
72
73struct ioring_options {
74 struct thread_data *td;
75 unsigned int hipri;
76 struct cmdprio cmdprio;
77 unsigned int fixedbufs;
78 unsigned int registerfiles;
79 unsigned int sqpoll_thread;
80 unsigned int sqpoll_set;
81 unsigned int sqpoll_cpu;
82 unsigned int nonvectored;
83 unsigned int uncached;
84 unsigned int nowait;
85 unsigned int force_async;
86};
87
88static const int ddir_to_op[2][2] = {
89 { IORING_OP_READV, IORING_OP_READ },
90 { IORING_OP_WRITEV, IORING_OP_WRITE }
91};
92
93static const int fixed_ddir_to_op[2] = {
94 IORING_OP_READ_FIXED,
95 IORING_OP_WRITE_FIXED
96};
97
98static int fio_ioring_sqpoll_cb(void *data, unsigned long long *val)
99{
100 struct ioring_options *o = data;
101
102 o->sqpoll_cpu = *val;
103 o->sqpoll_set = 1;
104 return 0;
105}
106
107static int str_cmdprio_bssplit_cb(void *data, const char *input)
108{
109 struct ioring_options *o = data;
110 struct thread_data *td = o->td;
111 struct cmdprio *cmdprio = &o->cmdprio;
112
113 return fio_cmdprio_bssplit_parse(td, input, cmdprio);
114}
115
116static struct fio_option options[] = {
117 {
118 .name = "hipri",
119 .lname = "High Priority",
120 .type = FIO_OPT_STR_SET,
121 .off1 = offsetof(struct ioring_options, hipri),
122 .help = "Use polled IO completions",
123 .category = FIO_OPT_C_ENGINE,
124 .group = FIO_OPT_G_IOURING,
125 },
126#ifdef FIO_HAVE_IOPRIO_CLASS
127 {
128 .name = "cmdprio_percentage",
129 .lname = "high priority percentage",
130 .type = FIO_OPT_INT,
131 .off1 = offsetof(struct ioring_options,
132 cmdprio.percentage[DDIR_READ]),
133 .off2 = offsetof(struct ioring_options,
134 cmdprio.percentage[DDIR_WRITE]),
135 .minval = 0,
136 .maxval = 100,
137 .help = "Send high priority I/O this percentage of the time",
138 .category = FIO_OPT_C_ENGINE,
139 .group = FIO_OPT_G_IOURING,
140 },
141 {
142 .name = "cmdprio_class",
143 .lname = "Asynchronous I/O priority class",
144 .type = FIO_OPT_INT,
145 .off1 = offsetof(struct ioring_options,
146 cmdprio.class[DDIR_READ]),
147 .off2 = offsetof(struct ioring_options,
148 cmdprio.class[DDIR_WRITE]),
149 .help = "Set asynchronous IO priority class",
150 .minval = IOPRIO_MIN_PRIO_CLASS + 1,
151 .maxval = IOPRIO_MAX_PRIO_CLASS,
152 .interval = 1,
153 .category = FIO_OPT_C_ENGINE,
154 .group = FIO_OPT_G_IOURING,
155 },
156 {
157 .name = "cmdprio",
158 .lname = "Asynchronous I/O priority level",
159 .type = FIO_OPT_INT,
160 .off1 = offsetof(struct ioring_options,
161 cmdprio.level[DDIR_READ]),
162 .off2 = offsetof(struct ioring_options,
163 cmdprio.level[DDIR_WRITE]),
164 .help = "Set asynchronous IO priority level",
165 .minval = IOPRIO_MIN_PRIO,
166 .maxval = IOPRIO_MAX_PRIO,
167 .interval = 1,
168 .category = FIO_OPT_C_ENGINE,
169 .group = FIO_OPT_G_IOURING,
170 },
171 {
172 .name = "cmdprio_bssplit",
173 .lname = "Priority percentage block size split",
174 .type = FIO_OPT_STR_ULL,
175 .cb = str_cmdprio_bssplit_cb,
176 .off1 = offsetof(struct ioring_options, cmdprio.bssplit),
177 .help = "Set priority percentages for different block sizes",
178 .category = FIO_OPT_C_ENGINE,
179 .group = FIO_OPT_G_IOURING,
180 },
181#else
182 {
183 .name = "cmdprio_percentage",
184 .lname = "high priority percentage",
185 .type = FIO_OPT_UNSUPPORTED,
186 .help = "Your platform does not support I/O priority classes",
187 },
188 {
189 .name = "cmdprio_class",
190 .lname = "Asynchronous I/O priority class",
191 .type = FIO_OPT_UNSUPPORTED,
192 .help = "Your platform does not support I/O priority classes",
193 },
194 {
195 .name = "cmdprio",
196 .lname = "Asynchronous I/O priority level",
197 .type = FIO_OPT_UNSUPPORTED,
198 .help = "Your platform does not support I/O priority classes",
199 },
200 {
201 .name = "cmdprio_bssplit",
202 .lname = "Priority percentage block size split",
203 .type = FIO_OPT_UNSUPPORTED,
204 .help = "Your platform does not support I/O priority classes",
205 },
206#endif
207 {
208 .name = "fixedbufs",
209 .lname = "Fixed (pre-mapped) IO buffers",
210 .type = FIO_OPT_STR_SET,
211 .off1 = offsetof(struct ioring_options, fixedbufs),
212 .help = "Pre map IO buffers",
213 .category = FIO_OPT_C_ENGINE,
214 .group = FIO_OPT_G_IOURING,
215 },
216 {
217 .name = "registerfiles",
218 .lname = "Register file set",
219 .type = FIO_OPT_STR_SET,
220 .off1 = offsetof(struct ioring_options, registerfiles),
221 .help = "Pre-open/register files",
222 .category = FIO_OPT_C_ENGINE,
223 .group = FIO_OPT_G_IOURING,
224 },
225 {
226 .name = "sqthread_poll",
227 .lname = "Kernel SQ thread polling",
228 .type = FIO_OPT_INT,
229 .off1 = offsetof(struct ioring_options, sqpoll_thread),
230 .help = "Offload submission/completion to kernel thread",
231 .category = FIO_OPT_C_ENGINE,
232 .group = FIO_OPT_G_IOURING,
233 },
234 {
235 .name = "sqthread_poll_cpu",
236 .lname = "SQ Thread Poll CPU",
237 .type = FIO_OPT_INT,
238 .cb = fio_ioring_sqpoll_cb,
239 .help = "What CPU to run SQ thread polling on",
240 .category = FIO_OPT_C_ENGINE,
241 .group = FIO_OPT_G_IOURING,
242 },
243 {
244 .name = "nonvectored",
245 .lname = "Non-vectored",
246 .type = FIO_OPT_INT,
247 .off1 = offsetof(struct ioring_options, nonvectored),
248 .def = "-1",
249 .help = "Use non-vectored read/write commands",
250 .category = FIO_OPT_C_ENGINE,
251 .group = FIO_OPT_G_IOURING,
252 },
253 {
254 .name = "uncached",
255 .lname = "Uncached",
256 .type = FIO_OPT_INT,
257 .off1 = offsetof(struct ioring_options, uncached),
258 .help = "Use RWF_UNCACHED for buffered read/writes",
259 .category = FIO_OPT_C_ENGINE,
260 .group = FIO_OPT_G_IOURING,
261 },
262 {
263 .name = "nowait",
264 .lname = "RWF_NOWAIT",
265 .type = FIO_OPT_BOOL,
266 .off1 = offsetof(struct ioring_options, nowait),
267 .help = "Use RWF_NOWAIT for reads/writes",
268 .category = FIO_OPT_C_ENGINE,
269 .group = FIO_OPT_G_IOURING,
270 },
271 {
272 .name = "force_async",
273 .lname = "Force async",
274 .type = FIO_OPT_INT,
275 .off1 = offsetof(struct ioring_options, force_async),
276 .help = "Set IOSQE_ASYNC every N requests",
277 .category = FIO_OPT_C_ENGINE,
278 .group = FIO_OPT_G_IOURING,
279 },
280 {
281 .name = NULL,
282 },
283};
284
285static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit,
286 unsigned int min_complete, unsigned int flags)
287{
288 return syscall(__NR_io_uring_enter, ld->ring_fd, to_submit,
289 min_complete, flags, NULL, 0);
290}
291
292static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
293{
294 struct ioring_data *ld = td->io_ops_data;
295 struct ioring_options *o = td->eo;
296 struct fio_file *f = io_u->file;
297 struct io_uring_sqe *sqe;
298
299 sqe = &ld->sqes[io_u->index];
300
301 if (o->registerfiles) {
302 sqe->fd = f->engine_pos;
303 sqe->flags = IOSQE_FIXED_FILE;
304 } else {
305 sqe->fd = f->fd;
306 sqe->flags = 0;
307 }
308
309 if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) {
310 if (o->fixedbufs) {
311 sqe->opcode = fixed_ddir_to_op[io_u->ddir];
312 sqe->addr = (unsigned long) io_u->xfer_buf;
313 sqe->len = io_u->xfer_buflen;
314 sqe->buf_index = io_u->index;
315 } else {
316 struct iovec *iov = &ld->iovecs[io_u->index];
317
318 /*
319 * Update based on actual io_u, requeue could have
320 * adjusted these
321 */
322 iov->iov_base = io_u->xfer_buf;
323 iov->iov_len = io_u->xfer_buflen;
324
325 sqe->opcode = ddir_to_op[io_u->ddir][!!o->nonvectored];
326 if (o->nonvectored) {
327 sqe->addr = (unsigned long) iov->iov_base;
328 sqe->len = iov->iov_len;
329 } else {
330 sqe->addr = (unsigned long) iov;
331 sqe->len = 1;
332 }
333 }
334 sqe->rw_flags = 0;
335 if (!td->o.odirect && o->uncached)
336 sqe->rw_flags |= RWF_UNCACHED;
337 if (o->nowait)
338 sqe->rw_flags |= RWF_NOWAIT;
339
340 /*
341 * Since io_uring can have a submission context (sqthread_poll)
342 * that is different from the process context, we cannot rely on
343 * the IO priority set by ioprio_set() (option prio/prioclass)
344 * to be inherited.
345 * td->ioprio will have the value of the "default prio", so set
346 * this unconditionally. This value might get overridden by
347 * fio_ioring_cmdprio_prep() if the option cmdprio_percentage or
348 * cmdprio_bssplit is used.
349 */
350 sqe->ioprio = td->ioprio;
351 sqe->off = io_u->offset;
352 } else if (ddir_sync(io_u->ddir)) {
353 sqe->ioprio = 0;
354 if (io_u->ddir == DDIR_SYNC_FILE_RANGE) {
355 sqe->off = f->first_write;
356 sqe->len = f->last_write - f->first_write;
357 sqe->sync_range_flags = td->o.sync_file_range;
358 sqe->opcode = IORING_OP_SYNC_FILE_RANGE;
359 } else {
360 sqe->off = 0;
361 sqe->addr = 0;
362 sqe->len = 0;
363 if (io_u->ddir == DDIR_DATASYNC)
364 sqe->fsync_flags |= IORING_FSYNC_DATASYNC;
365 sqe->opcode = IORING_OP_FSYNC;
366 }
367 }
368
369 if (o->force_async && ++ld->prepped == o->force_async) {
370 ld->prepped = 0;
371 sqe->flags |= IOSQE_ASYNC;
372 }
373
374 sqe->user_data = (unsigned long) io_u;
375 return 0;
376}
377
378static struct io_u *fio_ioring_event(struct thread_data *td, int event)
379{
380 struct ioring_data *ld = td->io_ops_data;
381 struct io_uring_cqe *cqe;
382 struct io_u *io_u;
383 unsigned index;
384
385 index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
386
387 cqe = &ld->cq_ring.cqes[index];
388 io_u = (struct io_u *) (uintptr_t) cqe->user_data;
389
390 if (cqe->res != io_u->xfer_buflen) {
391 if (cqe->res > io_u->xfer_buflen)
392 io_u->error = -cqe->res;
393 else
394 io_u->resid = io_u->xfer_buflen - cqe->res;
395 } else
396 io_u->error = 0;
397
398 return io_u;
399}
400
401static int fio_ioring_cqring_reap(struct thread_data *td, unsigned int events,
402 unsigned int max)
403{
404 struct ioring_data *ld = td->io_ops_data;
405 struct io_cq_ring *ring = &ld->cq_ring;
406 unsigned head, reaped = 0;
407
408 head = *ring->head;
409 do {
410 if (head == atomic_load_acquire(ring->tail))
411 break;
412 reaped++;
413 head++;
414 } while (reaped + events < max);
415
416 if (reaped)
417 atomic_store_release(ring->head, head);
418
419 return reaped;
420}
421
422static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
423 unsigned int max, const struct timespec *t)
424{
425 struct ioring_data *ld = td->io_ops_data;
426 unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min;
427 struct ioring_options *o = td->eo;
428 struct io_cq_ring *ring = &ld->cq_ring;
429 unsigned events = 0;
430 int r;
431
432 ld->cq_ring_off = *ring->head;
433 do {
434 r = fio_ioring_cqring_reap(td, events, max);
435 if (r) {
436 events += r;
437 if (actual_min != 0)
438 actual_min -= r;
439 continue;
440 }
441
442 if (!o->sqpoll_thread) {
443 r = io_uring_enter(ld, 0, actual_min,
444 IORING_ENTER_GETEVENTS);
445 if (r < 0) {
446 if (errno == EAGAIN || errno == EINTR)
447 continue;
448 td_verror(td, errno, "io_uring_enter");
449 break;
450 }
451 }
452 } while (events < min);
453
454 return r < 0 ? r : events;
455}
456
457static inline void fio_ioring_cmdprio_prep(struct thread_data *td,
458 struct io_u *io_u)
459{
460 struct ioring_data *ld = td->io_ops_data;
461 struct ioring_options *o = td->eo;
462 struct cmdprio *cmdprio = &o->cmdprio;
463
464 if (fio_cmdprio_set_ioprio(td, cmdprio, io_u))
465 ld->sqes[io_u->index].ioprio = io_u->ioprio;
466}
467
468static enum fio_q_status fio_ioring_queue(struct thread_data *td,
469 struct io_u *io_u)
470{
471 struct ioring_data *ld = td->io_ops_data;
472 struct io_sq_ring *ring = &ld->sq_ring;
473 struct ioring_options *o = td->eo;
474 unsigned tail, next_tail;
475
476 fio_ro_check(td, io_u);
477
478 if (ld->queued == ld->iodepth)
479 return FIO_Q_BUSY;
480
481 if (io_u->ddir == DDIR_TRIM) {
482 if (ld->queued)
483 return FIO_Q_BUSY;
484
485 do_io_u_trim(td, io_u);
486 io_u_mark_submit(td, 1);
487 io_u_mark_complete(td, 1);
488 return FIO_Q_COMPLETED;
489 }
490
491 tail = *ring->tail;
492 next_tail = tail + 1;
493 if (next_tail == atomic_load_acquire(ring->head))
494 return FIO_Q_BUSY;
495
496 if (o->cmdprio.mode != CMDPRIO_MODE_NONE)
497 fio_ioring_cmdprio_prep(td, io_u);
498
499 ring->array[tail & ld->sq_ring_mask] = io_u->index;
500 atomic_store_release(ring->tail, next_tail);
501
502 ld->queued++;
503 return FIO_Q_QUEUED;
504}
505
506static void fio_ioring_queued(struct thread_data *td, int start, int nr)
507{
508 struct ioring_data *ld = td->io_ops_data;
509 struct timespec now;
510
511 if (!fio_fill_issue_time(td))
512 return;
513
514 fio_gettime(&now, NULL);
515
516 while (nr--) {
517 struct io_sq_ring *ring = &ld->sq_ring;
518 int index = ring->array[start & ld->sq_ring_mask];
519 struct io_u *io_u = ld->io_u_index[index];
520
521 memcpy(&io_u->issue_time, &now, sizeof(now));
522 io_u_queued(td, io_u);
523
524 start++;
525 }
526}
527
528static int fio_ioring_commit(struct thread_data *td)
529{
530 struct ioring_data *ld = td->io_ops_data;
531 struct ioring_options *o = td->eo;
532 int ret;
533
534 if (!ld->queued)
535 return 0;
536
537 /*
538 * Kernel side does submission. just need to check if the ring is
539 * flagged as needing a kick, if so, call io_uring_enter(). This
540 * only happens if we've been idle too long.
541 */
542 if (o->sqpoll_thread) {
543 struct io_sq_ring *ring = &ld->sq_ring;
544 unsigned flags;
545
546 flags = atomic_load_acquire(ring->flags);
547 if (flags & IORING_SQ_NEED_WAKEUP)
548 io_uring_enter(ld, ld->queued, 0,
549 IORING_ENTER_SQ_WAKEUP);
550 ld->queued = 0;
551 return 0;
552 }
553
554 do {
555 unsigned start = *ld->sq_ring.head;
556 long nr = ld->queued;
557
558 ret = io_uring_enter(ld, nr, 0, IORING_ENTER_GETEVENTS);
559 if (ret > 0) {
560 fio_ioring_queued(td, start, ret);
561 io_u_mark_submit(td, ret);
562
563 ld->queued -= ret;
564 ret = 0;
565 } else if (!ret) {
566 io_u_mark_submit(td, ret);
567 continue;
568 } else {
569 if (errno == EAGAIN || errno == EINTR) {
570 ret = fio_ioring_cqring_reap(td, 0, ld->queued);
571 if (ret)
572 continue;
573 /* Shouldn't happen */
574 usleep(1);
575 continue;
576 }
577 td_verror(td, errno, "io_uring_enter submit");
578 break;
579 }
580 } while (ld->queued);
581
582 return ret;
583}
584
585static void fio_ioring_unmap(struct ioring_data *ld)
586{
587 int i;
588
589 for (i = 0; i < FIO_ARRAY_SIZE(ld->mmap); i++)
590 munmap(ld->mmap[i].ptr, ld->mmap[i].len);
591 close(ld->ring_fd);
592}
593
594static void fio_ioring_cleanup(struct thread_data *td)
595{
596 struct ioring_data *ld = td->io_ops_data;
597
598 if (ld) {
599 if (!(td->flags & TD_F_CHILD))
600 fio_ioring_unmap(ld);
601
602 free(ld->io_u_index);
603 free(ld->iovecs);
604 free(ld->fds);
605 free(ld);
606 }
607}
608
609static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p)
610{
611 struct io_sq_ring *sring = &ld->sq_ring;
612 struct io_cq_ring *cring = &ld->cq_ring;
613 void *ptr;
614
615 ld->mmap[0].len = p->sq_off.array + p->sq_entries * sizeof(__u32);
616 ptr = mmap(0, ld->mmap[0].len, PROT_READ | PROT_WRITE,
617 MAP_SHARED | MAP_POPULATE, ld->ring_fd,
618 IORING_OFF_SQ_RING);
619 ld->mmap[0].ptr = ptr;
620 sring->head = ptr + p->sq_off.head;
621 sring->tail = ptr + p->sq_off.tail;
622 sring->ring_mask = ptr + p->sq_off.ring_mask;
623 sring->ring_entries = ptr + p->sq_off.ring_entries;
624 sring->flags = ptr + p->sq_off.flags;
625 sring->array = ptr + p->sq_off.array;
626 ld->sq_ring_mask = *sring->ring_mask;
627
628 ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_sqe);
629 ld->sqes = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE,
630 MAP_SHARED | MAP_POPULATE, ld->ring_fd,
631 IORING_OFF_SQES);
632 ld->mmap[1].ptr = ld->sqes;
633
634 ld->mmap[2].len = p->cq_off.cqes +
635 p->cq_entries * sizeof(struct io_uring_cqe);
636 ptr = mmap(0, ld->mmap[2].len, PROT_READ | PROT_WRITE,
637 MAP_SHARED | MAP_POPULATE, ld->ring_fd,
638 IORING_OFF_CQ_RING);
639 ld->mmap[2].ptr = ptr;
640 cring->head = ptr + p->cq_off.head;
641 cring->tail = ptr + p->cq_off.tail;
642 cring->ring_mask = ptr + p->cq_off.ring_mask;
643 cring->ring_entries = ptr + p->cq_off.ring_entries;
644 cring->cqes = ptr + p->cq_off.cqes;
645 ld->cq_ring_mask = *cring->ring_mask;
646 return 0;
647}
648
649static void fio_ioring_probe(struct thread_data *td)
650{
651 struct ioring_data *ld = td->io_ops_data;
652 struct ioring_options *o = td->eo;
653 struct io_uring_probe *p;
654 int ret;
655
656 /* already set by user, don't touch */
657 if (o->nonvectored != -1)
658 return;
659
660 /* default to off, as that's always safe */
661 o->nonvectored = 0;
662
663 p = malloc(sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
664 if (!p)
665 return;
666
667 memset(p, 0, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
668 ret = syscall(__NR_io_uring_register, ld->ring_fd,
669 IORING_REGISTER_PROBE, p, 256);
670 if (ret < 0)
671 goto out;
672
673 if (IORING_OP_WRITE > p->ops_len)
674 goto out;
675
676 if ((p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED) &&
677 (p->ops[IORING_OP_WRITE].flags & IO_URING_OP_SUPPORTED))
678 o->nonvectored = 1;
679out:
680 free(p);
681}
682
683static int fio_ioring_queue_init(struct thread_data *td)
684{
685 struct ioring_data *ld = td->io_ops_data;
686 struct ioring_options *o = td->eo;
687 int depth = td->o.iodepth;
688 struct io_uring_params p;
689 int ret;
690
691 memset(&p, 0, sizeof(p));
692
693 if (o->hipri)
694 p.flags |= IORING_SETUP_IOPOLL;
695 if (o->sqpoll_thread) {
696 p.flags |= IORING_SETUP_SQPOLL;
697 if (o->sqpoll_set) {
698 p.flags |= IORING_SETUP_SQ_AFF;
699 p.sq_thread_cpu = o->sqpoll_cpu;
700 }
701 }
702
703 ret = syscall(__NR_io_uring_setup, depth, &p);
704 if (ret < 0)
705 return ret;
706
707 ld->ring_fd = ret;
708
709 fio_ioring_probe(td);
710
711 if (o->fixedbufs) {
712 ret = syscall(__NR_io_uring_register, ld->ring_fd,
713 IORING_REGISTER_BUFFERS, ld->iovecs, depth);
714 if (ret < 0)
715 return ret;
716 }
717
718 return fio_ioring_mmap(ld, &p);
719}
720
721static int fio_ioring_register_files(struct thread_data *td)
722{
723 struct ioring_data *ld = td->io_ops_data;
724 struct fio_file *f;
725 unsigned int i;
726 int ret;
727
728 ld->fds = calloc(td->o.nr_files, sizeof(int));
729
730 for_each_file(td, f, i) {
731 ret = generic_open_file(td, f);
732 if (ret)
733 goto err;
734 ld->fds[i] = f->fd;
735 f->engine_pos = i;
736 }
737
738 ret = syscall(__NR_io_uring_register, ld->ring_fd,
739 IORING_REGISTER_FILES, ld->fds, td->o.nr_files);
740 if (ret) {
741err:
742 free(ld->fds);
743 ld->fds = NULL;
744 }
745
746 /*
747 * Pretend the file is closed again, and really close it if we hit
748 * an error.
749 */
750 for_each_file(td, f, i) {
751 if (ret) {
752 int fio_unused ret2;
753 ret2 = generic_close_file(td, f);
754 } else
755 f->fd = -1;
756 }
757
758 return ret;
759}
760
761static int fio_ioring_post_init(struct thread_data *td)
762{
763 struct ioring_data *ld = td->io_ops_data;
764 struct ioring_options *o = td->eo;
765 struct io_u *io_u;
766 int err, i;
767
768 for (i = 0; i < td->o.iodepth; i++) {
769 struct iovec *iov = &ld->iovecs[i];
770
771 io_u = ld->io_u_index[i];
772 iov->iov_base = io_u->buf;
773 iov->iov_len = td_max_bs(td);
774 }
775
776 err = fio_ioring_queue_init(td);
777 if (err) {
778 int init_err = errno;
779
780 if (init_err == ENOSYS)
781 log_err("fio: your kernel doesn't support io_uring\n");
782 td_verror(td, init_err, "io_queue_init");
783 return 1;
784 }
785
786 for (i = 0; i < td->o.iodepth; i++) {
787 struct io_uring_sqe *sqe;
788
789 sqe = &ld->sqes[i];
790 memset(sqe, 0, sizeof(*sqe));
791 }
792
793 if (o->registerfiles) {
794 err = fio_ioring_register_files(td);
795 if (err) {
796 td_verror(td, errno, "ioring_register_files");
797 return 1;
798 }
799 }
800
801 return 0;
802}
803
804static int fio_ioring_init(struct thread_data *td)
805{
806 struct ioring_options *o = td->eo;
807 struct ioring_data *ld;
808 struct cmdprio *cmdprio = &o->cmdprio;
809 int ret;
810
811 /* sqthread submission requires registered files */
812 if (o->sqpoll_thread)
813 o->registerfiles = 1;
814
815 if (o->registerfiles && td->o.nr_files != td->o.open_files) {
816 log_err("fio: io_uring registered files require nr_files to "
817 "be identical to open_files\n");
818 return 1;
819 }
820
821 ld = calloc(1, sizeof(*ld));
822
823 /* ring depth must be a power-of-2 */
824 ld->iodepth = td->o.iodepth;
825 td->o.iodepth = roundup_pow2(td->o.iodepth);
826
827 /* io_u index */
828 ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
829 ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
830
831 td->io_ops_data = ld;
832
833 ret = fio_cmdprio_init(td, cmdprio);
834 if (ret) {
835 td_verror(td, EINVAL, "fio_ioring_init");
836 return 1;
837 }
838
839 return 0;
840}
841
842static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
843{
844 struct ioring_data *ld = td->io_ops_data;
845
846 ld->io_u_index[io_u->index] = io_u;
847 return 0;
848}
849
850static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f)
851{
852 struct ioring_data *ld = td->io_ops_data;
853 struct ioring_options *o = td->eo;
854
855 if (!ld || !o->registerfiles)
856 return generic_open_file(td, f);
857
858 f->fd = ld->fds[f->engine_pos];
859 return 0;
860}
861
862static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f)
863{
864 struct ioring_data *ld = td->io_ops_data;
865 struct ioring_options *o = td->eo;
866
867 if (!ld || !o->registerfiles)
868 return generic_close_file(td, f);
869
870 f->fd = -1;
871 return 0;
872}
873
874static struct ioengine_ops ioengine = {
875 .name = "io_uring",
876 .version = FIO_IOOPS_VERSION,
877 .flags = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD,
878 .init = fio_ioring_init,
879 .post_init = fio_ioring_post_init,
880 .io_u_init = fio_ioring_io_u_init,
881 .prep = fio_ioring_prep,
882 .queue = fio_ioring_queue,
883 .commit = fio_ioring_commit,
884 .getevents = fio_ioring_getevents,
885 .event = fio_ioring_event,
886 .cleanup = fio_ioring_cleanup,
887 .open_file = fio_ioring_open_file,
888 .close_file = fio_ioring_close_file,
889 .get_file_size = generic_get_file_size,
890 .options = options,
891 .option_struct_size = sizeof(struct ioring_options),
892};
893
894static void fio_init fio_ioring_register(void)
895{
896 register_ioengine(&ioengine);
897}
898
899static void fio_exit fio_ioring_unregister(void)
900{
901 unregister_ioengine(&ioengine);
902}
903#endif
fio - Flexible IO Tester