X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=t%2Fio_uring.c;h=55b75f6ed9664858a7cc6155ce0c159a146a4479;hb=HEAD;hp=0a90f85c55ad73a6ea5117dffec1607c979fdbd2;hpb=7d04588a766308d5903f6cfe34ed72f6c7612d19;p=fio.git diff --git a/t/io_uring.c b/t/io_uring.c index 0a90f85c..aa6e09e9 100644 --- a/t/io_uring.c +++ b/t/io_uring.c @@ -28,8 +28,10 @@ #include #include #include +#include #include "../arch/arch.h" +#include "../os/os.h" #include "../lib/types.h" #include "../lib/roundup.h" #include "../lib/rand.h" @@ -93,6 +95,7 @@ struct submitter { unsigned long reaps; unsigned long done; unsigned long calls; + unsigned long io_errors; volatile int finish; __s32 *fds; @@ -108,6 +111,7 @@ struct submitter { #endif int numa_node; + int per_file_depth; const char *filename; struct file files[MAX_FDS]; @@ -128,7 +132,6 @@ static int batch_complete = BATCH_COMPLETE; static int bs = BS; static int polled = 1; /* use IO polling */ static int fixedbufs = 1; /* use fixed user buffers */ -static int dma_map; /* pre-map DMA buffers */ static int register_files = 1; /* use fixed files */ static int buffered = 0; /* use buffered IO, not O_DIRECT */ static int sq_thread_poll = 0; /* use kernel submission/poller thread */ @@ -154,17 +157,6 @@ static float plist[] = { 1.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 95.0, 99.0, 99.5, 99.9, 99.95, 99.99 }; static int plist_len = 17; -#ifndef IORING_REGISTER_MAP_BUFFERS -#define IORING_REGISTER_MAP_BUFFERS 22 -struct io_uring_map_buffers { - __s32 fd; - __u32 buf_start; - __u32 buf_end; - __u32 flags; - __u64 rsvd[2]; -}; -#endif - static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns, enum nvme_csi csi, void *data) { @@ -404,22 +396,6 @@ static void add_stat(struct submitter *s, int clock_index, int nr) #endif } -static int io_uring_map_buffers(struct submitter *s) -{ - struct io_uring_map_buffers map = { - .fd = s->files[0].real_fd, - .buf_end = depth, - }; - - if (do_nop) - return 0; - if (s->nr_files > 1) - fprintf(stdout, "Mapping buffers may not work with multiple files\n"); - - return syscall(__NR_io_uring_register, s->ring_fd, - IORING_REGISTER_MAP_BUFFERS, &map, 1); -} - static int io_uring_register_buffers(struct submitter *s) { if (do_nop) @@ -448,6 +424,8 @@ static int io_uring_register_files(struct submitter *s) static int io_uring_setup(unsigned entries, struct io_uring_params *p) { + int ret; + /* * Clamp CQ ring size at our SQ ring size, we don't need more entries * than that. @@ -455,7 +433,28 @@ static int io_uring_setup(unsigned entries, struct io_uring_params *p) p->flags |= IORING_SETUP_CQSIZE; p->cq_entries = entries; - return syscall(__NR_io_uring_setup, entries, p); + p->flags |= IORING_SETUP_COOP_TASKRUN; + p->flags |= IORING_SETUP_SINGLE_ISSUER; + p->flags |= IORING_SETUP_DEFER_TASKRUN; +retry: + ret = syscall(__NR_io_uring_setup, entries, p); + if (!ret) + return 0; + + if (errno == EINVAL && p->flags & IORING_SETUP_COOP_TASKRUN) { + p->flags &= ~IORING_SETUP_COOP_TASKRUN; + goto retry; + } + if (errno == EINVAL && p->flags & IORING_SETUP_SINGLE_ISSUER) { + p->flags &= ~IORING_SETUP_SINGLE_ISSUER; + goto retry; + } + if (errno == EINVAL && p->flags & IORING_SETUP_DEFER_TASKRUN) { + p->flags &= ~IORING_SETUP_DEFER_TASKRUN; + goto retry; + } + + return ret; } static void io_uring_probe(int fd) @@ -463,11 +462,10 @@ static void io_uring_probe(int fd) struct io_uring_probe *p; int ret; - p = malloc(sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + p = calloc(1, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); if (!p) return; - memset(p, 0, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); ret = syscall(__NR_io_uring_register, fd, IORING_REGISTER_PROBE, p, 256); if (ret < 0) goto out; @@ -495,53 +493,59 @@ static int io_uring_enter(struct submitter *s, unsigned int to_submit, #endif } -#ifndef CONFIG_HAVE_GETTID -static int gettid(void) +static unsigned long long get_offset(struct submitter *s, struct file *f) { - return syscall(__NR_gettid); -} -#endif + unsigned long long offset; + long r; -static unsigned file_depth(struct submitter *s) -{ - return (depth + s->nr_files - 1) / s->nr_files; + if (random_io) { + unsigned long long block; + + r = __rand64(&s->rand_state); + block = r % f->max_blocks; + offset = block * (unsigned long long) bs; + } else { + offset = f->cur_off; + f->cur_off += bs; + if (f->cur_off + bs > f->max_size) + f->cur_off = 0; + } + + return offset; } -static void init_io(struct submitter *s, unsigned index) +static struct file *get_next_file(struct submitter *s) { - struct io_uring_sqe *sqe = &s->sqes[index]; - unsigned long offset; struct file *f; - long r; - - if (do_nop) { - sqe->opcode = IORING_OP_NOP; - return; - } if (s->nr_files == 1) { f = &s->files[0]; } else { f = &s->files[s->cur_file]; - if (f->pending_ios >= file_depth(s)) { + if (f->pending_ios >= s->per_file_depth) { s->cur_file++; if (s->cur_file == s->nr_files) s->cur_file = 0; f = &s->files[s->cur_file]; } } + f->pending_ios++; + return f; +} - if (random_io) { - r = __rand64(&s->rand_state); - offset = (r % (f->max_blocks - 1)) * bs; - } else { - offset = f->cur_off; - f->cur_off += bs; - if (f->cur_off + bs > f->max_size) - f->cur_off = 0; +static void init_io(struct submitter *s, unsigned index) +{ + struct io_uring_sqe *sqe = &s->sqes[index]; + struct file *f; + + if (do_nop) { + sqe->opcode = IORING_OP_NOP; + return; } + f = get_next_file(s); + if (register_files) { sqe->flags = IOSQE_FIXED_FILE; sqe->fd = f->fixed_fd; @@ -566,7 +570,7 @@ static void init_io(struct submitter *s, unsigned index) sqe->buf_index = 0; } sqe->ioprio = 0; - sqe->off = offset; + sqe->off = get_offset(s, f); sqe->user_data = (unsigned long) f->fileno; if (stats && stats_running) sqe->user_data |= ((uint64_t)s->clock_index << 32); @@ -580,30 +584,10 @@ static void init_io_pt(struct submitter *s, unsigned index) struct nvme_uring_cmd *cmd; unsigned long long slba; unsigned long long nlb; - long r; - if (s->nr_files == 1) { - f = &s->files[0]; - } else { - f = &s->files[s->cur_file]; - if (f->pending_ios >= file_depth(s)) { - s->cur_file++; - if (s->cur_file == s->nr_files) - s->cur_file = 0; - f = &s->files[s->cur_file]; - } - } - f->pending_ios++; + f = get_next_file(s); - if (random_io) { - r = __rand64(&s->rand_state); - offset = (r % (f->max_blocks - 1)) * bs; - } else { - offset = f->cur_off; - f->cur_off += bs; - if (f->cur_off + bs > f->max_size) - f->cur_off = 0; - } + offset = get_offset(s, f); if (register_files) { sqe->fd = f->fixed_fd; @@ -615,7 +599,7 @@ static void init_io_pt(struct submitter *s, unsigned index) sqe->opcode = IORING_OP_URING_CMD; sqe->user_data = (unsigned long) f->fileno; if (stats) - sqe->user_data |= ((unsigned long)s->clock_index << 32); + sqe->user_data |= ((__u64) s->clock_index << 32ULL); sqe->cmd_op = NVME_URING_CMD_IO; slba = offset >> f->lba_shift; nlb = (bs >> f->lba_shift) - 1; @@ -627,6 +611,10 @@ static void init_io_pt(struct submitter *s, unsigned index) cmd->cdw12 = nlb; cmd->addr = (unsigned long) s->iovecs[index].iov_base; cmd->data_len = bs; + if (fixedbufs) { + sqe->uring_cmd_flags = IORING_URING_CMD_FIXED; + sqe->buf_index = index; + } cmd->nsid = f->nsid; cmd->opcode = 2; } @@ -634,12 +622,17 @@ static void init_io_pt(struct submitter *s, unsigned index) static int prep_more_ios_uring(struct submitter *s, int max_ios) { struct io_sq_ring *ring = &s->sq_ring; - unsigned index, tail, next_tail, prepped = 0; + unsigned head, index, tail, next_tail, prepped = 0; + + if (sq_thread_poll) + head = atomic_load_acquire(ring->head); + else + head = *ring->head; next_tail = tail = *ring->tail; do { next_tail++; - if (next_tail == atomic_load_acquire(ring->head)) + if (next_tail == head) break; index = tail & sq_ring_mask; @@ -647,7 +640,6 @@ static int prep_more_ios_uring(struct submitter *s, int max_ios) init_io_pt(s, index); else init_io(s, index); - ring->array[index] = index; prepped++; tail = next_tail; } while (prepped < max_ios); @@ -681,7 +673,7 @@ static int get_file_size(struct file *f) bs, lbs); return -1; } - f->max_blocks = nlba / bs; + f->max_blocks = nlba; f->max_size = nlba; f->lba_shift = ilog2(lbs); return 0; @@ -714,7 +706,6 @@ static int reap_events_uring(struct submitter *s) do { struct file *f; - read_barrier(); if (head == atomic_load_acquire(ring->tail)) break; cqe = &ring->cqes[head & cq_ring_mask]; @@ -724,10 +715,14 @@ static int reap_events_uring(struct submitter *s) f = &s->files[fileno]; f->pending_ios--; if (cqe->res != bs) { - printf("io: unexpected ret=%d\n", cqe->res); - if (polled && cqe->res == -EOPNOTSUPP) - printf("Your filesystem/driver/kernel doesn't support polled IO\n"); - return -1; + if (cqe->res == -ENODATA || cqe->res == -EIO) { + s->io_errors++; + } else { + printf("io: unexpected ret=%d\n", cqe->res); + if (polled && cqe->res == -EOPNOTSUPP) + printf("Your filesystem/driver/kernel doesn't support polled IO\n"); + return -1; + } } } if (stats) { @@ -769,7 +764,6 @@ static int reap_events_uring_pt(struct submitter *s) do { struct file *f; - read_barrier(); if (head == atomic_load_acquire(ring->tail)) break; index = head & cq_ring_mask; @@ -826,14 +820,17 @@ static void set_affinity(struct submitter *s) #endif } -static int detect_node(struct submitter *s, const char *name) +static int detect_node(struct submitter *s, char *name) { #ifdef CONFIG_LIBNUMA const char *base = basename(name); char str[128]; int ret, fd, node; - sprintf(str, "/sys/block/%s/device/numa_node", base); + if (pt) + sprintf(str, "/sys/class/nvme-generic/%s/device/numa_node", base); + else + sprintf(str, "/sys/block/%s/device/numa_node", base); fd = open(str, O_RDONLY); if (fd < 0) return -1; @@ -872,6 +869,7 @@ static int setup_aio(struct submitter *s) fixedbufs = register_files = 0; } + s->per_file_depth = (depth + s->nr_files - 1) / s->nr_files; return io_queue_init(roundup_pow2(depth), &s->aio_ctx); #else fprintf(stderr, "Legacy AIO not available on this system/build\n"); @@ -885,7 +883,7 @@ static int setup_ring(struct submitter *s) struct io_sq_ring *sring = &s->sq_ring; struct io_cq_ring *cring = &s->cq_ring; struct io_uring_params p; - int ret, fd; + int ret, fd, i; void *ptr; size_t len; @@ -927,14 +925,6 @@ static int setup_ring(struct submitter *s) perror("io_uring_register_buffers"); return 1; } - - if (dma_map) { - ret = io_uring_map_buffers(s); - if (ret < 0) { - perror("io_uring_map_buffers"); - return 1; - } - } } if (register_files) { @@ -980,6 +970,13 @@ static int setup_ring(struct submitter *s) cring->ring_entries = ptr + p.cq_off.ring_entries; cring->cqes = ptr + p.cq_off.cqes; cq_ring_mask = *cring->ring_mask; + + for (i = 0; i < p.sq_entries; i++) + sring->array[i] = i; + + s->per_file_depth = INT_MAX; + if (s->nr_files) + s->per_file_depth = (depth + s->nr_files - 1) / s->nr_files; return 0; } @@ -1006,8 +1003,8 @@ static int submitter_init(struct submitter *s) static int init_printed; char buf[80]; s->tid = gettid(); - printf("submitter=%d, tid=%d, file=%s, node=%d\n", s->index, s->tid, - s->filename, s->numa_node); + printf("submitter=%d, tid=%d, file=%s, nfiles=%d, node=%d\n", s->index, s->tid, + s->filename, s->nr_files, s->numa_node); set_affinity(s); @@ -1022,7 +1019,7 @@ static int submitter_init(struct submitter *s) buf = allocate_mem(s, bs); if (!buf) - return 1; + return -1; s->iovecs[i].iov_base = buf; s->iovecs[i].iov_len = bs; } @@ -1032,18 +1029,19 @@ static int submitter_init(struct submitter *s) err = 0; } else if (!aio) { err = setup_ring(s); - sprintf(buf, "Engine=io_uring, sq_ring=%d, cq_ring=%d\n", *s->sq_ring.ring_entries, *s->cq_ring.ring_entries); + if (!err) + sprintf(buf, "Engine=io_uring, sq_ring=%d, cq_ring=%d\n", *s->sq_ring.ring_entries, *s->cq_ring.ring_entries); } else { sprintf(buf, "Engine=aio\n"); err = setup_aio(s); } if (err) { printf("queue setup failed: %s, %d\n", strerror(errno), err); - return 1; + return -1; } if (!init_printed) { - printf("polled=%d, fixedbufs=%d/%d, register_files=%d, buffered=%d, QD=%d\n", polled, fixedbufs, dma_map, register_files, buffered, depth); + printf("polled=%d, fixedbufs=%d, register_files=%d, buffered=%d, QD=%d\n", polled, fixedbufs, register_files, buffered, depth); printf("%s", buf); init_printed = 1; } @@ -1078,32 +1076,17 @@ static int submitter_init(struct submitter *s) static int prep_more_ios_aio(struct submitter *s, int max_ios, struct iocb *iocbs) { uint64_t data; - long long offset; struct file *f; unsigned index; - long r; index = 0; while (index < max_ios) { struct iocb *iocb = &iocbs[index]; - if (s->nr_files == 1) { - f = &s->files[0]; - } else { - f = &s->files[s->cur_file]; - if (f->pending_ios >= file_depth(s)) { - s->cur_file++; - if (s->cur_file == s->nr_files) - s->cur_file = 0; - f = &s->files[s->cur_file]; - } - } - f->pending_ios++; + f = get_next_file(s); - r = lrand48(); - offset = (r % (f->max_blocks - 1)) * bs; io_prep_pread(iocb, f->real_fd, s->iovecs[index].iov_base, - s->iovecs[index].iov_len, offset); + s->iovecs[index].iov_len, get_offset(s, f)); data = f->fileno; if (stats && stats_running) @@ -1125,10 +1108,14 @@ static int reap_events_aio(struct submitter *s, struct io_event *events, int evs f->pending_ios--; if (events[reaped].res != bs) { - printf("io: unexpected ret=%ld\n", events[reaped].res); - return -1; - } - if (stats) { + if (events[reaped].res == -ENODATA || + events[reaped].res == -EIO) { + s->io_errors++; + } else { + printf("io: unexpected ret=%ld\n", events[reaped].res); + return -1; + } + } else if (stats) { int clock_index = data >> 32; if (last_idx != clock_index) { @@ -1160,9 +1147,15 @@ static void *submitter_aio_fn(void *data) struct iocb *iocbs; struct io_event *events; #ifdef ARCH_HAVE_CPU_CLOCK - int nr_batch = submitter_init(s); -#else - submitter_init(s); + int nr_batch; +#endif + + ret = submitter_init(s); + if (ret < 0) + goto done; + +#ifdef ARCH_HAVE_CPU_CLOCK + nr_batch = ret; #endif iocbsptr = calloc(depth, sizeof(struct iocb *)); @@ -1226,6 +1219,7 @@ static void *submitter_aio_fn(void *data) free(iocbsptr); free(iocbs); free(events); +done: finish = 1; return NULL; } @@ -1265,9 +1259,15 @@ static void *submitter_uring_fn(void *data) struct io_sq_ring *ring = &s->sq_ring; int ret, prepped; #ifdef ARCH_HAVE_CPU_CLOCK - int nr_batch = submitter_init(s); -#else - submitter_init(s); + int nr_batch; +#endif + + ret = submitter_init(s); + if (ret < 0) + goto done; + +#ifdef ARCH_HAVE_CPU_CLOCK + nr_batch = ret; #endif if (register_ring) @@ -1371,6 +1371,7 @@ submit: if (register_ring) io_uring_unregister_ring(s); +done: finish = 1; return NULL; } @@ -1381,35 +1382,14 @@ static void *submitter_sync_fn(void *data) struct submitter *s = data; int ret; - submitter_init(s); + if (submitter_init(s) < 0) + goto done; do { uint64_t offset; struct file *f; - long r; - - if (s->nr_files == 1) { - f = &s->files[0]; - } else { - f = &s->files[s->cur_file]; - if (f->pending_ios >= file_depth(s)) { - s->cur_file++; - if (s->cur_file == s->nr_files) - s->cur_file = 0; - f = &s->files[s->cur_file]; - } - } - f->pending_ios++; - if (random_io) { - r = __rand64(&s->rand_state); - offset = (r % (f->max_blocks - 1)) * bs; - } else { - offset = f->cur_off; - f->cur_off += bs; - if (f->cur_off + bs > f->max_size) - f->cur_off = 0; - } + f = get_next_file(s); #ifdef ARCH_HAVE_CPU_CLOCK if (stats) @@ -1419,6 +1399,7 @@ static void *submitter_sync_fn(void *data) s->inflight++; s->calls++; + offset = get_offset(s, f); if (polled) ret = preadv2(f->real_fd, &s->iovecs[0], 1, offset, RWF_HIPRI); else @@ -1438,6 +1419,7 @@ static void *submitter_sync_fn(void *data) add_stat(s, s->clock_index, 1); } while (!s->finish); +done: finish = 1; return NULL; } @@ -1511,7 +1493,6 @@ static void usage(char *argv, int status) " -b : Block size, default %d\n" " -p : Polled IO, default %d\n" " -B : Fixed buffers, default %d\n" - " -D : DMA map fixed buffers, default %d\n" " -F : Register files, default %d\n" " -n : Number of threads, default %d\n" " -O : Use O_DIRECT, default %d\n" @@ -1526,7 +1507,7 @@ static void usage(char *argv, int status) " -P : Automatically place on device home node %d\n" " -u : Use nvme-passthrough I/O, default %d\n", argv, DEPTH, BATCH_SUBMIT, BATCH_COMPLETE, BS, polled, - fixedbufs, dma_map, register_files, nthreads, !buffered, do_nop, + fixedbufs, register_files, nthreads, !buffered, do_nop, stats, runtime == 0 ? "unlimited" : runtime_str, random_io, aio, use_sync, register_ring, numa_placement, pt); exit(status); @@ -1579,7 +1560,7 @@ static void write_tsc_rate(void) int main(int argc, char *argv[]) { struct submitter *s; - unsigned long done, calls, reap; + unsigned long done, calls, reap, io_errors; int i, j, flags, fd, opt, threads_per_f, threads_rem = 0, nfiles; struct file f; void *ret; @@ -1648,9 +1629,6 @@ int main(int argc, char *argv[]) case 'r': runtime = atoi(optarg); break; - case 'D': - dma_map = !!atoi(optarg); - break; case 'R': random_io = !!atoi(optarg); break; @@ -1686,8 +1664,6 @@ int main(int argc, char *argv[]) batch_complete = depth; if (batch_submit > depth) batch_submit = depth; - if (!fixedbufs && dma_map) - dma_map = 0; submitter = calloc(nthreads, sizeof(*submitter) + roundup_pow2(depth) * sizeof(struct iovec)); @@ -1695,7 +1671,7 @@ int main(int argc, char *argv[]) s = get_submitter(j); s->numa_node = -1; s->index = j; - s->done = s->calls = s->reaps = 0; + s->done = s->calls = s->reaps = s->io_errors = 0; } flags = O_RDONLY | O_NOATIME; @@ -1780,11 +1756,12 @@ int main(int argc, char *argv[]) #endif } - reap = calls = done = 0; + reap = calls = done = io_errors = 0; do { unsigned long this_done = 0; unsigned long this_reap = 0; unsigned long this_call = 0; + unsigned long this_io_errors = 0; unsigned long rpc = 0, ipc = 0; unsigned long iops, bw; @@ -1805,6 +1782,7 @@ int main(int argc, char *argv[]) this_done += s->done; this_call += s->calls; this_reap += s->reaps; + this_io_errors += s->io_errors; } if (this_call - calls) { rpc = (this_done - done) / (this_call - calls); @@ -1812,6 +1790,7 @@ int main(int argc, char *argv[]) } else rpc = ipc = -1; iops = this_done - done; + iops -= this_io_errors - io_errors; if (bs > 1048576) bw = iops * (bs / 1048576); else @@ -1839,6 +1818,7 @@ int main(int argc, char *argv[]) done = this_done; calls = this_call; reap = this_reap; + io_errors = this_io_errors; } while (!finish); for (j = 0; j < nthreads; j++) { @@ -1846,6 +1826,9 @@ int main(int argc, char *argv[]) pthread_join(s->thread, &ret); close(s->ring_fd); + if (s->io_errors) + printf("%d: %lu IO errors\n", s->tid, s->io_errors); + if (stats) { unsigned long nr;