t/io_uring.c

   1 #include <stdio.h>
   2 #include <errno.h>
   3 #include <assert.h>
   4 #include <stdlib.h>
   5 #include <stddef.h>
   6 #include <signal.h>
   7 #include <inttypes.h>
   8 #include <math.h>
   9
  10 #include <sys/types.h>
  11 #include <sys/stat.h>
  12 #include <sys/ioctl.h>
  13 #include <sys/syscall.h>
  14 #include <sys/resource.h>
  15 #include <sys/mman.h>
  16 #include <sys/uio.h>
  17 #include <linux/fs.h>
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20 #include <string.h>
  21 #include <pthread.h>
  22 #include <sched.h>
  23
  24 #include "../arch/arch.h"
  25 #include "../lib/types.h"
  26 #include "../lib/roundup.h"
  27 #include "../minmax.h"
  28 #include "../os/linux/io_uring.h"
  29
  30 struct io_sq_ring {
  31         unsigned *head;
  32         unsigned *tail;
  33         unsigned *ring_mask;
  34         unsigned *ring_entries;
  35         unsigned *flags;
  36         unsigned *array;
  37 };
  38
  39 struct io_cq_ring {
  40         unsigned *head;
  41         unsigned *tail;
  42         unsigned *ring_mask;
  43         unsigned *ring_entries;
  44         struct io_uring_cqe *cqes;
  45 };
  46
  47 #define DEPTH                   128
  48 #define BATCH_SUBMIT            32
  49 #define BATCH_COMPLETE          32
  50 #define BS                      4096
  51
  52 #define MAX_FDS                 16
  53
  54 static unsigned sq_ring_mask, cq_ring_mask;
  55
  56 struct file {
  57         unsigned long max_blocks;
  58         unsigned pending_ios;
  59         int real_fd;
  60         int fixed_fd;
  61         int fileno;
  62 };
  63
  64 #define PLAT_BITS               6
  65 #define PLAT_VAL                (1 << PLAT_BITS)
  66 #define PLAT_GROUP_NR           29
  67 #define PLAT_NR                 (PLAT_GROUP_NR * PLAT_VAL)
  68
  69 struct submitter {
  70         pthread_t thread;
  71         int ring_fd;
  72         int index;
  73         struct io_sq_ring sq_ring;
  74         struct io_uring_sqe *sqes;
  75         struct io_cq_ring cq_ring;
  76         int inflight;
  77         int tid;
  78         unsigned long reaps;
  79         unsigned long done;
  80         unsigned long calls;
  81         volatile int finish;
  82
  83         __s32 *fds;
  84
  85         unsigned long *clock_batch;
  86         int clock_index;
  87         unsigned long *plat;
  88
  89         struct file files[MAX_FDS];
  90         unsigned nr_files;
  91         unsigned cur_file;
  92         struct iovec iovecs[];
  93 };
  94
  95 static struct submitter *submitter;
  96 static volatile int finish;
  97
  98 static int depth = DEPTH;
  99 static int batch_submit = BATCH_SUBMIT;
 100 static int batch_complete = BATCH_COMPLETE;
 101 static int bs = BS;
 102 static int polled = 1;          /* use IO polling */
 103 static int fixedbufs = 1;       /* use fixed user buffers */
 104 static int register_files = 1;  /* use fixed files */
 105 static int buffered = 0;        /* use buffered IO, not O_DIRECT */
 106 static int sq_thread_poll = 0;  /* use kernel submission/poller thread */
 107 static int sq_thread_cpu = -1;  /* pin above thread to this CPU */
 108 static int do_nop = 0;          /* no-op SQ ring commands */
 109 static int nthreads = 1;
 110 static int stats = 0;           /* generate IO stats */
 111 static unsigned long tsc_rate;
 112
 113 static int vectored = 1;
 114
 115 static float plist[] = { 1.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0,
 116                         80.0, 90.0, 95.0, 99.9, 99.5, 99.9, 99.95, 99.99 };
 117 static int plist_len = 17;
 118
 119 static unsigned long cycles_to_nsec(unsigned long cycles)
 120 {
 121         uint64_t val;
 122
 123         if (!tsc_rate)
 124                 return cycles;
 125
 126         val = cycles * 1000000000ULL;
 127         return val / tsc_rate;
 128 }
 129
 130 static unsigned long plat_idx_to_val(unsigned int idx)
 131 {
 132         unsigned int error_bits;
 133         unsigned long k, base;
 134
 135         assert(idx < PLAT_NR);
 136
 137         /* MSB <= (PLAT_BITS-1), cannot be rounded off. Use
 138          * all bits of the sample as index */
 139         if (idx < (PLAT_VAL << 1))
 140                 return cycles_to_nsec(idx);
 141
 142         /* Find the group and compute the minimum value of that group */
 143         error_bits = (idx >> PLAT_BITS) - 1;
 144         base = ((unsigned long) 1) << (error_bits + PLAT_BITS);
 145
 146         /* Find its bucket number of the group */
 147         k = idx % PLAT_VAL;
 148
 149         /* Return the mean of the range of the bucket */
 150         return cycles_to_nsec(base + ((k + 0.5) * (1 << error_bits)));
 151 }
 152
 153 unsigned int calc_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
 154                                    unsigned long **output,
 155                                    unsigned long *maxv, unsigned long *minv)
 156 {
 157         unsigned long sum = 0;
 158         unsigned int len = plist_len, i, j = 0;
 159         unsigned long *ovals = NULL;
 160         bool is_last;
 161
 162         *minv = -1ULL;
 163         *maxv = 0;
 164
 165         ovals = malloc(len * sizeof(*ovals));
 166         if (!ovals)
 167                 return 0;
 168
 169         /*
 170          * Calculate bucket values, note down max and min values
 171          */
 172         is_last = false;
 173         for (i = 0; i < PLAT_NR && !is_last; i++) {
 174                 sum += io_u_plat[i];
 175                 while (sum >= ((long double) plist[j] / 100.0 * nr)) {
 176                         assert(plist[j] <= 100.0);
 177
 178                         ovals[j] = plat_idx_to_val(i);
 179                         if (ovals[j] < *minv)
 180                                 *minv = ovals[j];
 181                         if (ovals[j] > *maxv)
 182                                 *maxv = ovals[j];
 183
 184                         is_last = (j == len - 1) != 0;
 185                         if (is_last)
 186                                 break;
 187
 188                         j++;
 189                 }
 190         }
 191
 192         if (!is_last)
 193                 fprintf(stderr, "error calculating latency percentiles\n");
 194
 195         *output = ovals;
 196         return len;
 197 }
 198
 199 static void show_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
 200                                   unsigned int precision)
 201 {
 202         unsigned int divisor, len, i, j = 0;
 203         unsigned long minv, maxv;
 204         unsigned long *ovals;
 205         int per_line, scale_down, time_width;
 206         bool is_last;
 207         char fmt[32];
 208
 209         len = calc_clat_percentiles(io_u_plat, nr, &ovals, &maxv, &minv);
 210         if (!len || !ovals)
 211                 goto out;
 212
 213         if (!tsc_rate) {
 214                 scale_down = 0;
 215                 divisor = 1;
 216                 printf("    percentiles (tsc ticks):\n     |");
 217         } else if (minv > 2000 && maxv > 99999) {
 218                 scale_down = 1;
 219                 divisor = 1000;
 220                 printf("    percentiles (usec):\n     |");
 221         } else {
 222                 scale_down = 0;
 223                 divisor = 1;
 224                 printf("    percentiles (nsec):\n     |");
 225         }
 226
 227         time_width = max(5, (int) (log10(maxv / divisor) + 1));
 228         snprintf(fmt, sizeof(fmt), " %%%u.%ufth=[%%%dllu]%%c", precision + 3,
 229                         precision, time_width);
 230         /* fmt will be something like " %5.2fth=[%4llu]%c" */
 231         per_line = (80 - 7) / (precision + 10 + time_width);
 232
 233         for (j = 0; j < len; j++) {
 234                 /* for formatting */
 235                 if (j != 0 && (j % per_line) == 0)
 236                         printf("     |");
 237
 238                 /* end of the list */
 239                 is_last = (j == len - 1) != 0;
 240
 241                 for (i = 0; i < scale_down; i++)
 242                         ovals[j] = (ovals[j] + 999) / 1000;
 243
 244                 printf(fmt, plist[j], ovals[j], is_last ? '\n' : ',');
 245
 246                 if (is_last)
 247                         break;
 248
 249                 if ((j % per_line) == per_line - 1)     /* for formatting */
 250                         printf("\n");
 251         }
 252
 253 out:
 254         free(ovals);
 255 }
 256
 257 static unsigned int plat_val_to_idx(unsigned long val)
 258 {
 259         unsigned int msb, error_bits, base, offset, idx;
 260
 261         /* Find MSB starting from bit 0 */
 262         if (val == 0)
 263                 msb = 0;
 264         else
 265                 msb = (sizeof(val)*8) - __builtin_clzll(val) - 1;
 266
 267         /*
 268          * MSB <= (PLAT_BITS-1), cannot be rounded off. Use
 269          * all bits of the sample as index
 270          */
 271         if (msb <= PLAT_BITS)
 272                 return val;
 273
 274         /* Compute the number of error bits to discard*/
 275         error_bits = msb - PLAT_BITS;
 276
 277         /* Compute the number of buckets before the group */
 278         base = (error_bits + 1) << PLAT_BITS;
 279
 280         /*
 281          * Discard the error bits and apply the mask to find the
 282          * index for the buckets in the group
 283          */
 284         offset = (PLAT_VAL - 1) & (val >> error_bits);
 285
 286         /* Make sure the index does not exceed (array size - 1) */
 287         idx = (base + offset) < (PLAT_NR - 1) ?
 288                 (base + offset) : (PLAT_NR - 1);
 289
 290         return idx;
 291 }
 292
 293 static void add_stat(struct submitter *s, int clock_index, int nr)
 294 {
 295 #ifdef ARCH_HAVE_CPU_CLOCK
 296         unsigned long cycles;
 297         unsigned int pidx;
 298
 299         cycles = get_cpu_clock();
 300         cycles -= s->clock_batch[clock_index];
 301         pidx = plat_val_to_idx(cycles);
 302         s->plat[pidx] += nr;
 303 #endif
 304 }
 305
 306 static int io_uring_register_buffers(struct submitter *s)
 307 {
 308         if (do_nop)
 309                 return 0;
 310
 311         return syscall(__NR_io_uring_register, s->ring_fd,
 312                         IORING_REGISTER_BUFFERS, s->iovecs, depth);
 313 }
 314
 315 static int io_uring_register_files(struct submitter *s)
 316 {
 317         int i;
 318
 319         if (do_nop)
 320                 return 0;
 321
 322         s->fds = calloc(s->nr_files, sizeof(__s32));
 323         for (i = 0; i < s->nr_files; i++) {
 324                 s->fds[i] = s->files[i].real_fd;
 325                 s->files[i].fixed_fd = i;
 326         }
 327
 328         return syscall(__NR_io_uring_register, s->ring_fd,
 329                         IORING_REGISTER_FILES, s->fds, s->nr_files);
 330 }
 331
 332 static int io_uring_setup(unsigned entries, struct io_uring_params *p)
 333 {
 334         return syscall(__NR_io_uring_setup, entries, p);
 335 }
 336
 337 static void io_uring_probe(int fd)
 338 {
 339         struct io_uring_probe *p;
 340         int ret;
 341
 342         p = malloc(sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
 343         if (!p)
 344                 return;
 345
 346         memset(p, 0, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
 347         ret = syscall(__NR_io_uring_register, fd, IORING_REGISTER_PROBE, p, 256);
 348         if (ret < 0)
 349                 goto out;
 350
 351         if (IORING_OP_READ > p->ops_len)
 352                 goto out;
 353
 354         if ((p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED))
 355                 vectored = 0;
 356 out:
 357         free(p);
 358 }
 359
 360 static int io_uring_enter(struct submitter *s, unsigned int to_submit,
 361                           unsigned int min_complete, unsigned int flags)
 362 {
 363         return syscall(__NR_io_uring_enter, s->ring_fd, to_submit, min_complete,
 364                         flags, NULL, 0);
 365 }
 366
 367 #ifndef CONFIG_HAVE_GETTID
 368 static int gettid(void)
 369 {
 370         return syscall(__NR_gettid);
 371 }
 372 #endif
 373
 374 static unsigned file_depth(struct submitter *s)
 375 {
 376         return (depth + s->nr_files - 1) / s->nr_files;
 377 }
 378
 379 static void init_io(struct submitter *s, unsigned index)
 380 {
 381         struct io_uring_sqe *sqe = &s->sqes[index];
 382         unsigned long offset;
 383         struct file *f;
 384         long r;
 385
 386         if (do_nop) {
 387                 sqe->opcode = IORING_OP_NOP;
 388                 return;
 389         }
 390
 391         if (s->nr_files == 1) {
 392                 f = &s->files[0];
 393         } else {
 394                 f = &s->files[s->cur_file];
 395                 if (f->pending_ios >= file_depth(s)) {
 396                         s->cur_file++;
 397                         if (s->cur_file == s->nr_files)
 398                                 s->cur_file = 0;
 399                         f = &s->files[s->cur_file];
 400                 }
 401         }
 402         f->pending_ios++;
 403
 404         r = lrand48();
 405         offset = (r % (f->max_blocks - 1)) * bs;
 406
 407         if (register_files) {
 408                 sqe->flags = IOSQE_FIXED_FILE;
 409                 sqe->fd = f->fixed_fd;
 410         } else {
 411                 sqe->flags = 0;
 412                 sqe->fd = f->real_fd;
 413         }
 414         if (fixedbufs) {
 415                 sqe->opcode = IORING_OP_READ_FIXED;
 416                 sqe->addr = (unsigned long) s->iovecs[index].iov_base;
 417                 sqe->len = bs;
 418                 sqe->buf_index = index;
 419         } else if (!vectored) {
 420                 sqe->opcode = IORING_OP_READ;
 421                 sqe->addr = (unsigned long) s->iovecs[index].iov_base;
 422                 sqe->len = bs;
 423                 sqe->buf_index = 0;
 424         } else {
 425                 sqe->opcode = IORING_OP_READV;
 426                 sqe->addr = (unsigned long) &s->iovecs[index];
 427                 sqe->len = 1;
 428                 sqe->buf_index = 0;
 429         }
 430         sqe->ioprio = 0;
 431         sqe->off = offset;
 432         sqe->user_data = (unsigned long) f->fileno;
 433         if (stats)
 434                 sqe->user_data |= ((unsigned long)s->clock_index << 32);
 435 }
 436
 437 static int prep_more_ios(struct submitter *s, int max_ios)
 438 {
 439         struct io_sq_ring *ring = &s->sq_ring;
 440         unsigned index, tail, next_tail, prepped = 0;
 441
 442         next_tail = tail = *ring->tail;
 443         do {
 444                 next_tail++;
 445                 if (next_tail == atomic_load_acquire(ring->head))
 446                         break;
 447
 448                 index = tail & sq_ring_mask;
 449                 init_io(s, index);
 450                 ring->array[index] = index;
 451                 prepped++;
 452                 tail = next_tail;
 453         } while (prepped < max_ios);
 454
 455         if (prepped)
 456                 atomic_store_release(ring->tail, tail);
 457         return prepped;
 458 }
 459
 460 static int get_file_size(struct file *f)
 461 {
 462         struct stat st;
 463
 464         if (fstat(f->real_fd, &st) < 0)
 465                 return -1;
 466         if (S_ISBLK(st.st_mode)) {
 467                 unsigned long long bytes;
 468
 469                 if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
 470                         return -1;
 471
 472                 f->max_blocks = bytes / bs;
 473                 return 0;
 474         } else if (S_ISREG(st.st_mode)) {
 475                 f->max_blocks = st.st_size / bs;
 476                 return 0;
 477         }
 478
 479         return -1;
 480 }
 481
 482 static int reap_events(struct submitter *s)
 483 {
 484         struct io_cq_ring *ring = &s->cq_ring;
 485         struct io_uring_cqe *cqe;
 486         unsigned head, reaped = 0;
 487         int last_idx = -1, stat_nr = 0;
 488
 489         head = *ring->head;
 490         do {
 491                 struct file *f;
 492
 493                 read_barrier();
 494                 if (head == atomic_load_acquire(ring->tail))
 495                         break;
 496                 cqe = &ring->cqes[head & cq_ring_mask];
 497                 if (!do_nop) {
 498                         int fileno = cqe->user_data & 0xffffffff;
 499
 500                         f = &s->files[fileno];
 501                         f->pending_ios--;
 502                         if (cqe->res != bs) {
 503                                 printf("io: unexpected ret=%d\n", cqe->res);
 504                                 if (polled && cqe->res == -EOPNOTSUPP)
 505                                         printf("Your filesystem/driver/kernel doesn't support polled IO\n");
 506                                 return -1;
 507                         }
 508                 }
 509                 if (stats) {
 510                         int clock_index = cqe->user_data >> 32;
 511
 512                         if (last_idx != clock_index) {
 513                                 if (last_idx != -1) {
 514                                         add_stat(s, last_idx, stat_nr);
 515                                         stat_nr = 0;
 516                                 }
 517                                 last_idx = clock_index;
 518                         }
 519                         stat_nr++;
 520                         add_stat(s, clock_index, 1);
 521                 }
 522                 reaped++;
 523                 head++;
 524         } while (1);
 525
 526         if (stat_nr)
 527                 add_stat(s, last_idx, stat_nr);
 528
 529         if (reaped) {
 530                 s->inflight -= reaped;
 531                 atomic_store_release(ring->head, head);
 532         }
 533         return reaped;
 534 }
 535
 536 static void *submitter_fn(void *data)
 537 {
 538         struct submitter *s = data;
 539         struct io_sq_ring *ring = &s->sq_ring;
 540         int i, ret, prepped, nr_batch;
 541
 542         s->tid = gettid();
 543         printf("submitter=%d\n", s->tid);
 544
 545         srand48(pthread_self());
 546
 547         for (i = 0; i < MAX_FDS; i++)
 548                 s->files[i].fileno = i;
 549
 550         if (stats) {
 551                 nr_batch = roundup_pow2(depth / batch_submit);
 552                 s->clock_batch = calloc(nr_batch, sizeof(unsigned long));
 553                 s->clock_index = 0;
 554
 555                 s->plat = calloc(PLAT_NR, sizeof(unsigned long));
 556         } else {
 557                 s->clock_batch = NULL;
 558                 s->plat = NULL;
 559                 nr_batch = 0;
 560         }
 561
 562         prepped = 0;
 563         do {
 564                 int to_wait, to_submit, this_reap, to_prep;
 565                 unsigned ring_flags = 0;
 566
 567                 if (!prepped && s->inflight < depth) {
 568                         to_prep = min(depth - s->inflight, batch_submit);
 569                         prepped = prep_more_ios(s, to_prep);
 570 #ifdef ARCH_HAVE_CPU_CLOCK
 571                         if (prepped && stats) {
 572                                 s->clock_batch[s->clock_index] = get_cpu_clock();
 573                                 s->clock_index = (s->clock_index + 1) & (nr_batch - 1);
 574                         }
 575 #endif
 576                 }
 577                 s->inflight += prepped;
 578 submit_more:
 579                 to_submit = prepped;
 580 submit:
 581                 if (to_submit && (s->inflight + to_submit <= depth))
 582                         to_wait = 0;
 583                 else
 584                         to_wait = min(s->inflight + to_submit, batch_complete);
 585
 586                 /*
 587                  * Only need to call io_uring_enter if we're not using SQ thread
 588                  * poll, or if IORING_SQ_NEED_WAKEUP is set.
 589                  */
 590                 if (sq_thread_poll)
 591                         ring_flags = atomic_load_acquire(ring->flags);
 592                 if (!sq_thread_poll || ring_flags & IORING_SQ_NEED_WAKEUP) {
 593                         unsigned flags = 0;
 594
 595                         if (to_wait)
 596                                 flags = IORING_ENTER_GETEVENTS;
 597                         if (ring_flags & IORING_SQ_NEED_WAKEUP)
 598                                 flags |= IORING_ENTER_SQ_WAKEUP;
 599                         ret = io_uring_enter(s, to_submit, to_wait, flags);
 600                         s->calls++;
 601                 } else {
 602                         /* for SQPOLL, we submitted it all effectively */
 603                         ret = to_submit;
 604                 }
 605
 606                 /*
 607                  * For non SQ thread poll, we already got the events we needed
 608                  * through the io_uring_enter() above. For SQ thread poll, we
 609                  * need to loop here until we find enough events.
 610                  */
 611                 this_reap = 0;
 612                 do {
 613                         int r;
 614                         r = reap_events(s);
 615                         if (r == -1) {
 616                                 s->finish = 1;
 617                                 break;
 618                         } else if (r > 0)
 619                                 this_reap += r;
 620                 } while (sq_thread_poll && this_reap < to_wait);
 621                 s->reaps += this_reap;
 622
 623                 if (ret >= 0) {
 624                         if (!ret) {
 625                                 to_submit = 0;
 626                                 if (s->inflight)
 627                                         goto submit;
 628                                 continue;
 629                         } else if (ret < to_submit) {
 630                                 int diff = to_submit - ret;
 631
 632                                 s->done += ret;
 633                                 prepped -= diff;
 634                                 goto submit_more;
 635                         }
 636                         s->done += ret;
 637                         prepped = 0;
 638                         continue;
 639                 } else if (ret < 0) {
 640                         if (errno == EAGAIN) {
 641                                 if (s->finish)
 642                                         break;
 643                                 if (this_reap)
 644                                         goto submit;
 645                                 to_submit = 0;
 646                                 goto submit;
 647                         }
 648                         printf("io_submit: %s\n", strerror(errno));
 649                         break;
 650                 }
 651         } while (!s->finish);
 652
 653         finish = 1;
 654         return NULL;
 655 }
 656
 657 static struct submitter *get_submitter(int offset)
 658 {
 659         void *ret;
 660
 661         ret = submitter;
 662         if (offset)
 663                 ret += offset * (sizeof(*submitter) + depth * sizeof(struct iovec));
 664         return ret;
 665 }
 666
 667 static void sig_int(int sig)
 668 {
 669         int j;
 670
 671         printf("Exiting on signal %d\n", sig);
 672         for (j = 0; j < nthreads; j++) {
 673                 struct submitter *s = get_submitter(j);
 674                 s->finish = 1;
 675         }
 676         finish = 1;
 677 }
 678
 679 static void arm_sig_int(void)
 680 {
 681         struct sigaction act;
 682
 683         memset(&act, 0, sizeof(act));
 684         act.sa_handler = sig_int;
 685         act.sa_flags = SA_RESTART;
 686         sigaction(SIGINT, &act, NULL);
 687 }
 688
 689 static int setup_ring(struct submitter *s)
 690 {
 691         struct io_sq_ring *sring = &s->sq_ring;
 692         struct io_cq_ring *cring = &s->cq_ring;
 693         struct io_uring_params p;
 694         int ret, fd;
 695         void *ptr;
 696
 697         memset(&p, 0, sizeof(p));
 698
 699         if (polled && !do_nop)
 700                 p.flags |= IORING_SETUP_IOPOLL;
 701         if (sq_thread_poll) {
 702                 p.flags |= IORING_SETUP_SQPOLL;
 703                 if (sq_thread_cpu != -1) {
 704                         p.flags |= IORING_SETUP_SQ_AFF;
 705                         p.sq_thread_cpu = sq_thread_cpu;
 706                 }
 707         }
 708
 709         fd = io_uring_setup(depth, &p);
 710         if (fd < 0) {
 711                 perror("io_uring_setup");
 712                 return 1;
 713         }
 714         s->ring_fd = fd;
 715
 716         io_uring_probe(fd);
 717
 718         if (fixedbufs) {
 719                 struct rlimit rlim;
 720
 721                 rlim.rlim_cur = RLIM_INFINITY;
 722                 rlim.rlim_max = RLIM_INFINITY;
 723                 /* ignore potential error, not needed on newer kernels */
 724                 setrlimit(RLIMIT_MEMLOCK, &rlim);
 725
 726                 ret = io_uring_register_buffers(s);
 727                 if (ret < 0) {
 728                         perror("io_uring_register_buffers");
 729                         return 1;
 730                 }
 731         }
 732
 733         if (register_files) {
 734                 ret = io_uring_register_files(s);
 735                 if (ret < 0) {
 736                         perror("io_uring_register_files");
 737                         return 1;
 738                 }
 739         }
 740
 741         ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
 742                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 743                         IORING_OFF_SQ_RING);
 744         printf("sq_ring ptr = 0x%p\n", ptr);
 745         sring->head = ptr + p.sq_off.head;
 746         sring->tail = ptr + p.sq_off.tail;
 747         sring->ring_mask = ptr + p.sq_off.ring_mask;
 748         sring->ring_entries = ptr + p.sq_off.ring_entries;
 749         sring->flags = ptr + p.sq_off.flags;
 750         sring->array = ptr + p.sq_off.array;
 751         sq_ring_mask = *sring->ring_mask;
 752
 753         s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
 754                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 755                         IORING_OFF_SQES);
 756         printf("sqes ptr    = 0x%p\n", s->sqes);
 757
 758         ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
 759                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 760                         IORING_OFF_CQ_RING);
 761         printf("cq_ring ptr = 0x%p\n", ptr);
 762         cring->head = ptr + p.cq_off.head;
 763         cring->tail = ptr + p.cq_off.tail;
 764         cring->ring_mask = ptr + p.cq_off.ring_mask;
 765         cring->ring_entries = ptr + p.cq_off.ring_entries;
 766         cring->cqes = ptr + p.cq_off.cqes;
 767         cq_ring_mask = *cring->ring_mask;
 768         return 0;
 769 }
 770
 771 static void file_depths(char *buf)
 772 {
 773         bool prev = false;
 774         char *p;
 775         int i, j;
 776
 777         buf[0] = '\0';
 778         p = buf;
 779         for (j = 0; j < nthreads; j++) {
 780                 struct submitter *s = get_submitter(j);
 781
 782                 for (i = 0; i < s->nr_files; i++) {
 783                         struct file *f = &s->files[i];
 784
 785                         if (prev)
 786                                 p += sprintf(p, " %d", f->pending_ios);
 787                         else
 788                                 p += sprintf(p, "%d", f->pending_ios);
 789                         prev = true;
 790                 }
 791         }
 792 }
 793
 794 static void usage(char *argv, int status)
 795 {
 796         printf("%s [options] -- [filenames]\n"
 797                 " -d <int>  : IO Depth, default %d\n"
 798                 " -s <int>  : Batch submit, default %d\n"
 799                 " -c <int>  : Batch complete, default %d\n"
 800                 " -b <int>  : Block size, default %d\n"
 801                 " -p <bool> : Polled IO, default %d\n"
 802                 " -B <bool> : Fixed buffers, default %d\n"
 803                 " -F <bool> : Register files, default %d\n"
 804                 " -n <int>  : Number of threads, default %d\n"
 805                 " -O <bool> : Use O_DIRECT, default %d\n"
 806                 " -N <bool> : Perform just no-op requests, default %d\n"
 807                 " -t <bool> : Track IO latencies, default %d\n"
 808                 " -T <int>  : TSC rate in HZ\n",
 809                 argv, DEPTH, BATCH_SUBMIT, BATCH_COMPLETE, BS, polled,
 810                 fixedbufs, register_files, nthreads, !buffered, do_nop, stats);
 811         exit(status);
 812 }
 813
 814 int main(int argc, char *argv[])
 815 {
 816         struct submitter *s;
 817         unsigned long done, calls, reap;
 818         int err, i, j, flags, fd, opt, threads_per_f, threads_rem = 0, nfiles;
 819         struct file f;
 820         char *fdepths;
 821         void *ret;
 822
 823         if (!do_nop && argc < 2)
 824                 usage(argv[0], 1);
 825
 826         while ((opt = getopt(argc, argv, "d:s:c:b:p:B:F:n:N:O:t:T:h?")) != -1) {
 827                 switch (opt) {
 828                 case 'd':
 829                         depth = atoi(optarg);
 830                         break;
 831                 case 's':
 832                         batch_submit = atoi(optarg);
 833                         if (!batch_submit)
 834                                 batch_submit = 1;
 835                         break;
 836                 case 'c':
 837                         batch_complete = atoi(optarg);
 838                         if (!batch_complete)
 839                                 batch_complete = 1;
 840                         break;
 841                 case 'b':
 842                         bs = atoi(optarg);
 843                         break;
 844                 case 'p':
 845                         polled = !!atoi(optarg);
 846                         break;
 847                 case 'B':
 848                         fixedbufs = !!atoi(optarg);
 849                         break;
 850                 case 'F':
 851                         register_files = !!atoi(optarg);
 852                         break;
 853                 case 'n':
 854                         nthreads = atoi(optarg);
 855                         if (!nthreads) {
 856                                 printf("Threads must be non-zero\n");
 857                                 usage(argv[0], 1);
 858                         }
 859                         break;
 860                 case 'N':
 861                         do_nop = !!atoi(optarg);
 862                         break;
 863                 case 'O':
 864                         buffered = !atoi(optarg);
 865                         break;
 866                 case 't':
 867 #ifndef ARCH_HAVE_CPU_CLOCK
 868                         fprintf(stderr, "Stats not supported on this CPU\n");
 869                         return 1;
 870 #endif
 871                         stats = !!atoi(optarg);
 872                         break;
 873                 case 'T':
 874 #ifndef ARCH_HAVE_CPU_CLOCK
 875                         fprintf(stderr, "Stats not supported on this CPU\n");
 876                         return 1;
 877 #endif
 878                         tsc_rate = strtoul(optarg, NULL, 10);
 879                         break;
 880                 case 'h':
 881                 case '?':
 882                 default:
 883                         usage(argv[0], 0);
 884                         break;
 885                 }
 886         }
 887
 888         if (batch_complete > depth)
 889                 batch_complete = depth;
 890         if (batch_submit > depth)
 891                 batch_submit = depth;
 892
 893         submitter = calloc(nthreads, sizeof(*submitter) +
 894                                 depth * sizeof(struct iovec));
 895         for (j = 0; j < nthreads; j++) {
 896                 s = get_submitter(j);
 897                 s->index = j;
 898                 s->done = s->calls = s->reaps = 0;
 899         }
 900
 901         flags = O_RDONLY | O_NOATIME;
 902         if (!buffered)
 903                 flags |= O_DIRECT;
 904
 905         j = 0;
 906         i = optind;
 907         nfiles = argc - i;
 908         if (!do_nop) {
 909                 if (!nfiles) {
 910                         printf("No files specified\n");
 911                         usage(argv[0], 1);
 912                 }
 913                 threads_per_f = nthreads / nfiles;
 914                 /* make sure each thread gets assigned files */
 915                 if (threads_per_f == 0) {
 916                         threads_per_f = 1;
 917                 } else {
 918                         threads_rem = nthreads - threads_per_f * nfiles;
 919                 }
 920         }
 921         while (!do_nop && i < argc) {
 922                 int k, limit;
 923
 924                 memset(&f, 0, sizeof(f));
 925
 926                 fd = open(argv[i], flags);
 927                 if (fd < 0) {
 928                         perror("open");
 929                         return 1;
 930                 }
 931                 f.real_fd = fd;
 932                 if (get_file_size(&f)) {
 933                         printf("failed getting size of device/file\n");
 934                         return 1;
 935                 }
 936                 if (f.max_blocks <= 1) {
 937                         printf("Zero file/device size?\n");
 938                         return 1;
 939                 }
 940                 f.max_blocks--;
 941
 942                 limit = threads_per_f;
 943                 limit += threads_rem > 0 ? 1 : 0;
 944                 for (k = 0; k < limit; k++) {
 945                         s = get_submitter((j + k) % nthreads);
 946
 947                         if (s->nr_files == MAX_FDS) {
 948                                 printf("Max number of files (%d) reached\n", MAX_FDS);
 949                                 break;
 950                         }
 951
 952                         memcpy(&s->files[s->nr_files], &f, sizeof(f));
 953
 954                         printf("Added file %s (submitter %d)\n", argv[i], s->index);
 955                         s->nr_files++;
 956                 }
 957                 threads_rem--;
 958                 i++;
 959                 j += limit;
 960         }
 961
 962         arm_sig_int();
 963
 964         for (j = 0; j < nthreads; j++) {
 965                 s = get_submitter(j);
 966                 for (i = 0; i < depth; i++) {
 967                         void *buf;
 968
 969                         if (posix_memalign(&buf, bs, bs)) {
 970                                 printf("failed alloc\n");
 971                                 return 1;
 972                         }
 973                         s->iovecs[i].iov_base = buf;
 974                         s->iovecs[i].iov_len = bs;
 975                 }
 976         }
 977
 978         for (j = 0; j < nthreads; j++) {
 979                 s = get_submitter(j);
 980
 981                 err = setup_ring(s);
 982                 if (err) {
 983                         printf("ring setup failed: %s, %d\n", strerror(errno), err);
 984                         return 1;
 985                 }
 986         }
 987         s = get_submitter(0);
 988         printf("polled=%d, fixedbufs=%d, register_files=%d, buffered=%d", polled, fixedbufs, register_files, buffered);
 989         printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", depth, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
 990
 991         for (j = 0; j < nthreads; j++) {
 992                 s = get_submitter(j);
 993                 pthread_create(&s->thread, NULL, submitter_fn, s);
 994         }
 995
 996         fdepths = malloc(8 * s->nr_files * nthreads);
 997         reap = calls = done = 0;
 998         do {
 999                 unsigned long this_done = 0;
1000                 unsigned long this_reap = 0;
1001                 unsigned long this_call = 0;
1002                 unsigned long rpc = 0, ipc = 0;
1003                 unsigned long iops, bw;
1004
1005                 sleep(1);
1006                 for (j = 0; j < nthreads; j++) {
1007                         this_done += s->done;
1008                         this_call += s->calls;
1009                         this_reap += s->reaps;
1010                 }
1011                 if (this_call - calls) {
1012                         rpc = (this_done - done) / (this_call - calls);
1013                         ipc = (this_reap - reap) / (this_call - calls);
1014                 } else
1015                         rpc = ipc = -1;
1016                 file_depths(fdepths);
1017                 iops = this_done - done;
1018                 if (bs > 1048576)
1019                         bw = iops * (bs / 1048576);
1020                 else
1021                         bw = iops / (1048576 / bs);
1022                 printf("IOPS=%lu, ", iops);
1023                 if (!do_nop)
1024                         printf("BW=%luMiB/s, ", bw);
1025                 printf("IOS/call=%ld/%ld, inflight=(%s)\n", rpc, ipc, fdepths);
1026                 done = this_done;
1027                 calls = this_call;
1028                 reap = this_reap;
1029         } while (!finish);
1030
1031         for (j = 0; j < nthreads; j++) {
1032                 s = get_submitter(j);
1033                 pthread_join(s->thread, &ret);
1034                 close(s->ring_fd);
1035
1036                 if (stats) {
1037                         unsigned long nr;
1038
1039                         printf("%d: Latency percentiles:\n", s->tid);
1040                         for (i = 0, nr = 0; i < PLAT_NR; i++)
1041                                 nr += s->plat[i];
1042                         show_clat_percentiles(s->plat, nr, 4);
1043                         free(s->clock_batch);
1044                         free(s->plat);
1045                 }
1046         }
1047
1048         free(fdepths);
1049         free(submitter);
1050         return 0;
1051 }