t/io_uring.c

   1 #include <stdio.h>
   2 #include <errno.h>
   3 #include <assert.h>
   4 #include <stdlib.h>
   5 #include <stddef.h>
   6 #include <signal.h>
   7 #include <inttypes.h>
   8 #include <math.h>
   9
  10 #include <sys/types.h>
  11 #include <sys/stat.h>
  12 #include <sys/ioctl.h>
  13 #include <sys/syscall.h>
  14 #include <sys/resource.h>
  15 #include <sys/mman.h>
  16 #include <sys/uio.h>
  17 #include <linux/fs.h>
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20 #include <string.h>
  21 #include <pthread.h>
  22 #include <sched.h>
  23
  24 #include "../arch/arch.h"
  25 #include "../lib/types.h"
  26 #include "../lib/roundup.h"
  27 #include "../minmax.h"
  28 #include "../os/linux/io_uring.h"
  29
  30 struct io_sq_ring {
  31         unsigned *head;
  32         unsigned *tail;
  33         unsigned *ring_mask;
  34         unsigned *ring_entries;
  35         unsigned *flags;
  36         unsigned *array;
  37 };
  38
  39 struct io_cq_ring {
  40         unsigned *head;
  41         unsigned *tail;
  42         unsigned *ring_mask;
  43         unsigned *ring_entries;
  44         struct io_uring_cqe *cqes;
  45 };
  46
  47 #define DEPTH                   128
  48 #define BATCH_SUBMIT            32
  49 #define BATCH_COMPLETE          32
  50 #define BS                      4096
  51
  52 #define MAX_FDS                 16
  53
  54 static unsigned sq_ring_mask, cq_ring_mask;
  55
  56 struct file {
  57         unsigned long max_blocks;
  58         unsigned pending_ios;
  59         int real_fd;
  60         int fixed_fd;
  61         int fileno;
  62 };
  63
  64 #define PLAT_BITS               6
  65 #define PLAT_VAL                (1 << PLAT_BITS)
  66 #define PLAT_GROUP_NR           29
  67 #define PLAT_NR                 (PLAT_GROUP_NR * PLAT_VAL)
  68
  69 struct submitter {
  70         pthread_t thread;
  71         int ring_fd;
  72         int index;
  73         struct io_sq_ring sq_ring;
  74         struct io_uring_sqe *sqes;
  75         struct io_cq_ring cq_ring;
  76         int inflight;
  77         int tid;
  78         unsigned long reaps;
  79         unsigned long done;
  80         unsigned long calls;
  81         volatile int finish;
  82
  83         __s32 *fds;
  84
  85         unsigned long *clock_batch;
  86         int clock_index;
  87         unsigned long *plat;
  88
  89         struct file files[MAX_FDS];
  90         unsigned nr_files;
  91         unsigned cur_file;
  92         struct iovec iovecs[];
  93 };
  94
  95 static struct submitter *submitter;
  96 static volatile int finish;
  97
  98 static int depth = DEPTH;
  99 static int batch_submit = BATCH_SUBMIT;
 100 static int batch_complete = BATCH_COMPLETE;
 101 static int bs = BS;
 102 static int polled = 1;          /* use IO polling */
 103 static int fixedbufs = 1;       /* use fixed user buffers */
 104 static int register_files = 1;  /* use fixed files */
 105 static int buffered = 0;        /* use buffered IO, not O_DIRECT */
 106 static int sq_thread_poll = 0;  /* use kernel submission/poller thread */
 107 static int sq_thread_cpu = -1;  /* pin above thread to this CPU */
 108 static int do_nop = 0;          /* no-op SQ ring commands */
 109 static int nthreads = 1;
 110 static int stats = 0;           /* generate IO stats */
 111 static unsigned long tsc_rate;
 112
 113 static int vectored = 1;
 114
 115 static float plist[] = { 1.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0,
 116                         80.0, 90.0, 95.0, 99.9, 99.5, 99.9, 99.95, 99.99 };
 117 static int plist_len = 17;
 118
 119 static unsigned long cycles_to_nsec(unsigned long cycles)
 120 {
 121         uint64_t val;
 122
 123         if (!tsc_rate)
 124                 return cycles;
 125
 126         val = cycles * 1000000000ULL;
 127         return val / tsc_rate;
 128 }
 129
 130 static unsigned long plat_idx_to_val(unsigned int idx)
 131 {
 132         unsigned int error_bits;
 133         unsigned long k, base;
 134
 135         assert(idx < PLAT_NR);
 136
 137         /* MSB <= (PLAT_BITS-1), cannot be rounded off. Use
 138          * all bits of the sample as index */
 139         if (idx < (PLAT_VAL << 1))
 140                 return cycles_to_nsec(idx);
 141
 142         /* Find the group and compute the minimum value of that group */
 143         error_bits = (idx >> PLAT_BITS) - 1;
 144         base = ((unsigned long) 1) << (error_bits + PLAT_BITS);
 145
 146         /* Find its bucket number of the group */
 147         k = idx % PLAT_VAL;
 148
 149         /* Return the mean of the range of the bucket */
 150         return cycles_to_nsec(base + ((k + 0.5) * (1 << error_bits)));
 151 }
 152
 153 unsigned int calc_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
 154                                    unsigned long **output,
 155                                    unsigned long *maxv, unsigned long *minv)
 156 {
 157         unsigned long sum = 0;
 158         unsigned int len = plist_len, i, j = 0;
 159         unsigned long *ovals = NULL;
 160         bool is_last;
 161
 162         *minv = -1ULL;
 163         *maxv = 0;
 164
 165         ovals = malloc(len * sizeof(*ovals));
 166         if (!ovals)
 167                 return 0;
 168
 169         /*
 170          * Calculate bucket values, note down max and min values
 171          */
 172         is_last = false;
 173         for (i = 0; i < PLAT_NR && !is_last; i++) {
 174                 sum += io_u_plat[i];
 175                 while (sum >= ((long double) plist[j] / 100.0 * nr)) {
 176                         assert(plist[j] <= 100.0);
 177
 178                         ovals[j] = plat_idx_to_val(i);
 179                         if (ovals[j] < *minv)
 180                                 *minv = ovals[j];
 181                         if (ovals[j] > *maxv)
 182                                 *maxv = ovals[j];
 183
 184                         is_last = (j == len - 1) != 0;
 185                         if (is_last)
 186                                 break;
 187
 188                         j++;
 189                 }
 190         }
 191
 192         if (!is_last)
 193                 fprintf(stderr, "error calculating latency percentiles\n");
 194
 195         *output = ovals;
 196         return len;
 197 }
 198
 199 static void show_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
 200                                   unsigned int precision)
 201 {
 202         unsigned int divisor, len, i, j = 0;
 203         unsigned long minv, maxv;
 204         unsigned long *ovals;
 205         int per_line, scale_down, time_width;
 206         bool is_last;
 207         char fmt[32];
 208
 209         len = calc_clat_percentiles(io_u_plat, nr, &ovals, &maxv, &minv);
 210         if (!len || !ovals)
 211                 goto out;
 212
 213         if (!tsc_rate) {
 214                 scale_down = 0;
 215                 divisor = 1;
 216                 printf("    percentiles (tsc ticks):\n     |");
 217         } else if (minv > 2000 && maxv > 99999) {
 218                 scale_down = 1;
 219                 divisor = 1000;
 220                 printf("    percentiles (usec):\n     |");
 221         } else {
 222                 scale_down = 0;
 223                 divisor = 1;
 224                 printf("    percentiles (nsec):\n     |");
 225         }
 226
 227         time_width = max(5, (int) (log10(maxv / divisor) + 1));
 228         snprintf(fmt, sizeof(fmt), " %%%u.%ufth=[%%%dllu]%%c", precision + 3,
 229                         precision, time_width);
 230         /* fmt will be something like " %5.2fth=[%4llu]%c" */
 231         per_line = (80 - 7) / (precision + 10 + time_width);
 232
 233         for (j = 0; j < len; j++) {
 234                 /* for formatting */
 235                 if (j != 0 && (j % per_line) == 0)
 236                         printf("     |");
 237
 238                 /* end of the list */
 239                 is_last = (j == len - 1) != 0;
 240
 241                 for (i = 0; i < scale_down; i++)
 242                         ovals[j] = (ovals[j] + 999) / 1000;
 243
 244                 printf(fmt, plist[j], ovals[j], is_last ? '\n' : ',');
 245
 246                 if (is_last)
 247                         break;
 248
 249                 if ((j % per_line) == per_line - 1)     /* for formatting */
 250                         printf("\n");
 251         }
 252
 253 out:
 254         free(ovals);
 255 }
 256
 257 static unsigned int plat_val_to_idx(unsigned long val)
 258 {
 259         unsigned int msb, error_bits, base, offset, idx;
 260
 261         /* Find MSB starting from bit 0 */
 262         if (val == 0)
 263                 msb = 0;
 264         else
 265                 msb = (sizeof(val)*8) - __builtin_clzll(val) - 1;
 266
 267         /*
 268          * MSB <= (PLAT_BITS-1), cannot be rounded off. Use
 269          * all bits of the sample as index
 270          */
 271         if (msb <= PLAT_BITS)
 272                 return val;
 273
 274         /* Compute the number of error bits to discard*/
 275         error_bits = msb - PLAT_BITS;
 276
 277         /* Compute the number of buckets before the group */
 278         base = (error_bits + 1) << PLAT_BITS;
 279
 280         /*
 281          * Discard the error bits and apply the mask to find the
 282          * index for the buckets in the group
 283          */
 284         offset = (PLAT_VAL - 1) & (val >> error_bits);
 285
 286         /* Make sure the index does not exceed (array size - 1) */
 287         idx = (base + offset) < (PLAT_NR - 1) ?
 288                 (base + offset) : (PLAT_NR - 1);
 289
 290         return idx;
 291 }
 292
 293 static void add_stat(struct submitter *s, int clock_index, int nr)
 294 {
 295 #ifdef ARCH_HAVE_CPU_CLOCK
 296         unsigned long cycles;
 297         unsigned int pidx;
 298
 299         cycles = get_cpu_clock();
 300         cycles -= s->clock_batch[clock_index];
 301         pidx = plat_val_to_idx(cycles);
 302         s->plat[pidx] += nr;
 303 #endif
 304 }
 305
 306 static int io_uring_register_buffers(struct submitter *s)
 307 {
 308         if (do_nop)
 309                 return 0;
 310
 311         return syscall(__NR_io_uring_register, s->ring_fd,
 312                         IORING_REGISTER_BUFFERS, s->iovecs, depth);
 313 }
 314
 315 static int io_uring_register_files(struct submitter *s)
 316 {
 317         int i;
 318
 319         if (do_nop)
 320                 return 0;
 321
 322         s->fds = calloc(s->nr_files, sizeof(__s32));
 323         for (i = 0; i < s->nr_files; i++) {
 324                 s->fds[i] = s->files[i].real_fd;
 325                 s->files[i].fixed_fd = i;
 326         }
 327
 328         return syscall(__NR_io_uring_register, s->ring_fd,
 329                         IORING_REGISTER_FILES, s->fds, s->nr_files);
 330 }
 331
 332 static int io_uring_setup(unsigned entries, struct io_uring_params *p)
 333 {
 334         return syscall(__NR_io_uring_setup, entries, p);
 335 }
 336
 337 static void io_uring_probe(int fd)
 338 {
 339         struct io_uring_probe *p;
 340         int ret;
 341
 342         p = malloc(sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
 343         if (!p)
 344                 return;
 345
 346         memset(p, 0, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
 347         ret = syscall(__NR_io_uring_register, fd, IORING_REGISTER_PROBE, p, 256);
 348         if (ret < 0)
 349                 goto out;
 350
 351         if (IORING_OP_READ > p->ops_len)
 352                 goto out;
 353
 354         if ((p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED))
 355                 vectored = 0;
 356 out:
 357         free(p);
 358 }
 359
 360 static int io_uring_enter(struct submitter *s, unsigned int to_submit,
 361                           unsigned int min_complete, unsigned int flags)
 362 {
 363         return syscall(__NR_io_uring_enter, s->ring_fd, to_submit, min_complete,
 364                         flags, NULL, 0);
 365 }
 366
 367 #ifndef CONFIG_HAVE_GETTID
 368 static int gettid(void)
 369 {
 370         return syscall(__NR_gettid);
 371 }
 372 #endif
 373
 374 static unsigned file_depth(struct submitter *s)
 375 {
 376         return (depth + s->nr_files - 1) / s->nr_files;
 377 }
 378
 379 static void init_io(struct submitter *s, unsigned index)
 380 {
 381         struct io_uring_sqe *sqe = &s->sqes[index];
 382         unsigned long offset;
 383         struct file *f;
 384         long r;
 385
 386         if (do_nop) {
 387                 sqe->opcode = IORING_OP_NOP;
 388                 return;
 389         }
 390
 391         if (s->nr_files == 1) {
 392                 f = &s->files[0];
 393         } else {
 394                 f = &s->files[s->cur_file];
 395                 if (f->pending_ios >= file_depth(s)) {
 396                         s->cur_file++;
 397                         if (s->cur_file == s->nr_files)
 398                                 s->cur_file = 0;
 399                         f = &s->files[s->cur_file];
 400                 }
 401         }
 402         f->pending_ios++;
 403
 404         r = lrand48();
 405         offset = (r % (f->max_blocks - 1)) * bs;
 406
 407         if (register_files) {
 408                 sqe->flags = IOSQE_FIXED_FILE;
 409                 sqe->fd = f->fixed_fd;
 410         } else {
 411                 sqe->flags = 0;
 412                 sqe->fd = f->real_fd;
 413         }
 414         if (fixedbufs) {
 415                 sqe->opcode = IORING_OP_READ_FIXED;
 416                 sqe->addr = (unsigned long) s->iovecs[index].iov_base;
 417                 sqe->len = bs;
 418                 sqe->buf_index = index;
 419         } else if (!vectored) {
 420                 sqe->opcode = IORING_OP_READ;
 421                 sqe->addr = (unsigned long) s->iovecs[index].iov_base;
 422                 sqe->len = bs;
 423                 sqe->buf_index = 0;
 424         } else {
 425                 sqe->opcode = IORING_OP_READV;
 426                 sqe->addr = (unsigned long) &s->iovecs[index];
 427                 sqe->len = 1;
 428                 sqe->buf_index = 0;
 429         }
 430         sqe->ioprio = 0;
 431         sqe->off = offset;
 432         sqe->user_data = (unsigned long) f->fileno;
 433         if (stats)
 434                 sqe->user_data |= ((unsigned long)s->clock_index << 32);
 435 }
 436
 437 static int prep_more_ios(struct submitter *s, int max_ios)
 438 {
 439         struct io_sq_ring *ring = &s->sq_ring;
 440         unsigned index, tail, next_tail, prepped = 0;
 441
 442         next_tail = tail = *ring->tail;
 443         do {
 444                 next_tail++;
 445                 if (next_tail == atomic_load_acquire(ring->head))
 446                         break;
 447
 448                 index = tail & sq_ring_mask;
 449                 init_io(s, index);
 450                 ring->array[index] = index;
 451                 prepped++;
 452                 tail = next_tail;
 453         } while (prepped < max_ios);
 454
 455         if (prepped)
 456                 atomic_store_release(ring->tail, tail);
 457         return prepped;
 458 }
 459
 460 static int get_file_size(struct file *f)
 461 {
 462         struct stat st;
 463
 464         if (fstat(f->real_fd, &st) < 0)
 465                 return -1;
 466         if (S_ISBLK(st.st_mode)) {
 467                 unsigned long long bytes;
 468
 469                 if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
 470                         return -1;
 471
 472                 f->max_blocks = bytes / bs;
 473                 return 0;
 474         } else if (S_ISREG(st.st_mode)) {
 475                 f->max_blocks = st.st_size / bs;
 476                 return 0;
 477         }
 478
 479         return -1;
 480 }
 481
 482 static int reap_events(struct submitter *s)
 483 {
 484         struct io_cq_ring *ring = &s->cq_ring;
 485         struct io_uring_cqe *cqe;
 486         unsigned head, reaped = 0;
 487
 488         head = *ring->head;
 489         do {
 490                 struct file *f;
 491
 492                 read_barrier();
 493                 if (head == atomic_load_acquire(ring->tail))
 494                         break;
 495                 cqe = &ring->cqes[head & cq_ring_mask];
 496                 if (!do_nop) {
 497                         int fileno = cqe->user_data & 0xffffffff;
 498
 499                         f = &s->files[fileno];
 500                         f->pending_ios--;
 501                         if (cqe->res != bs) {
 502                                 printf("io: unexpected ret=%d\n", cqe->res);
 503                                 if (polled && cqe->res == -EOPNOTSUPP)
 504                                         printf("Your filesystem/driver/kernel doesn't support polled IO\n");
 505                                 return -1;
 506                         }
 507                 }
 508                 if (stats) {
 509                         int clock_index = cqe->user_data >> 32;
 510
 511                         add_stat(s, clock_index, 1);
 512                 }
 513                 reaped++;
 514                 head++;
 515         } while (1);
 516
 517         if (reaped) {
 518                 s->inflight -= reaped;
 519                 atomic_store_release(ring->head, head);
 520         }
 521         return reaped;
 522 }
 523
 524 static void *submitter_fn(void *data)
 525 {
 526         struct submitter *s = data;
 527         struct io_sq_ring *ring = &s->sq_ring;
 528         int i, ret, prepped, nr_batch;
 529
 530         s->tid = gettid();
 531         printf("submitter=%d\n", s->tid);
 532
 533         srand48(pthread_self());
 534
 535         for (i = 0; i < MAX_FDS; i++)
 536                 s->files[i].fileno = i;
 537
 538         if (stats) {
 539                 nr_batch = roundup_pow2(depth / batch_submit);
 540                 s->clock_batch = calloc(nr_batch, sizeof(unsigned long));
 541                 s->clock_index = 0;
 542
 543                 s->plat = calloc(PLAT_NR, sizeof(unsigned long));
 544         } else {
 545                 s->clock_batch = NULL;
 546                 s->plat = NULL;
 547                 nr_batch = 0;
 548         }
 549
 550         prepped = 0;
 551         do {
 552                 int to_wait, to_submit, this_reap, to_prep;
 553                 unsigned ring_flags = 0;
 554
 555                 if (!prepped && s->inflight < depth) {
 556                         to_prep = min(depth - s->inflight, batch_submit);
 557                         prepped = prep_more_ios(s, to_prep);
 558 #ifdef ARCH_HAVE_CPU_CLOCK
 559                         if (prepped && stats) {
 560                                 s->clock_batch[s->clock_index] = get_cpu_clock();
 561                                 s->clock_index = (s->clock_index + 1) & (nr_batch - 1);
 562                         }
 563 #endif
 564                 }
 565                 s->inflight += prepped;
 566 submit_more:
 567                 to_submit = prepped;
 568 submit:
 569                 if (to_submit && (s->inflight + to_submit <= depth))
 570                         to_wait = 0;
 571                 else
 572                         to_wait = min(s->inflight + to_submit, batch_complete);
 573
 574                 /*
 575                  * Only need to call io_uring_enter if we're not using SQ thread
 576                  * poll, or if IORING_SQ_NEED_WAKEUP is set.
 577                  */
 578                 if (sq_thread_poll)
 579                         ring_flags = atomic_load_acquire(ring->flags);
 580                 if (!sq_thread_poll || ring_flags & IORING_SQ_NEED_WAKEUP) {
 581                         unsigned flags = 0;
 582
 583                         if (to_wait)
 584                                 flags = IORING_ENTER_GETEVENTS;
 585                         if (ring_flags & IORING_SQ_NEED_WAKEUP)
 586                                 flags |= IORING_ENTER_SQ_WAKEUP;
 587                         ret = io_uring_enter(s, to_submit, to_wait, flags);
 588                         s->calls++;
 589                 } else {
 590                         /* for SQPOLL, we submitted it all effectively */
 591                         ret = to_submit;
 592                 }
 593
 594                 /*
 595                  * For non SQ thread poll, we already got the events we needed
 596                  * through the io_uring_enter() above. For SQ thread poll, we
 597                  * need to loop here until we find enough events.
 598                  */
 599                 this_reap = 0;
 600                 do {
 601                         int r;
 602                         r = reap_events(s);
 603                         if (r == -1) {
 604                                 s->finish = 1;
 605                                 break;
 606                         } else if (r > 0)
 607                                 this_reap += r;
 608                 } while (sq_thread_poll && this_reap < to_wait);
 609                 s->reaps += this_reap;
 610
 611                 if (ret >= 0) {
 612                         if (!ret) {
 613                                 to_submit = 0;
 614                                 if (s->inflight)
 615                                         goto submit;
 616                                 continue;
 617                         } else if (ret < to_submit) {
 618                                 int diff = to_submit - ret;
 619
 620                                 s->done += ret;
 621                                 prepped -= diff;
 622                                 goto submit_more;
 623                         }
 624                         s->done += ret;
 625                         prepped = 0;
 626                         continue;
 627                 } else if (ret < 0) {
 628                         if (errno == EAGAIN) {
 629                                 if (s->finish)
 630                                         break;
 631                                 if (this_reap)
 632                                         goto submit;
 633                                 to_submit = 0;
 634                                 goto submit;
 635                         }
 636                         printf("io_submit: %s\n", strerror(errno));
 637                         break;
 638                 }
 639         } while (!s->finish);
 640
 641         finish = 1;
 642         return NULL;
 643 }
 644
 645 static struct submitter *get_submitter(int offset)
 646 {
 647         void *ret;
 648
 649         ret = submitter;
 650         if (offset)
 651                 ret += offset * (sizeof(*submitter) + depth * sizeof(struct iovec));
 652         return ret;
 653 }
 654
 655 static void sig_int(int sig)
 656 {
 657         int j;
 658
 659         printf("Exiting on signal %d\n", sig);
 660         for (j = 0; j < nthreads; j++) {
 661                 struct submitter *s = get_submitter(j);
 662                 s->finish = 1;
 663         }
 664         finish = 1;
 665 }
 666
 667 static void arm_sig_int(void)
 668 {
 669         struct sigaction act;
 670
 671         memset(&act, 0, sizeof(act));
 672         act.sa_handler = sig_int;
 673         act.sa_flags = SA_RESTART;
 674         sigaction(SIGINT, &act, NULL);
 675 }
 676
 677 static int setup_ring(struct submitter *s)
 678 {
 679         struct io_sq_ring *sring = &s->sq_ring;
 680         struct io_cq_ring *cring = &s->cq_ring;
 681         struct io_uring_params p;
 682         int ret, fd;
 683         void *ptr;
 684
 685         memset(&p, 0, sizeof(p));
 686
 687         if (polled && !do_nop)
 688                 p.flags |= IORING_SETUP_IOPOLL;
 689         if (sq_thread_poll) {
 690                 p.flags |= IORING_SETUP_SQPOLL;
 691                 if (sq_thread_cpu != -1) {
 692                         p.flags |= IORING_SETUP_SQ_AFF;
 693                         p.sq_thread_cpu = sq_thread_cpu;
 694                 }
 695         }
 696
 697         fd = io_uring_setup(depth, &p);
 698         if (fd < 0) {
 699                 perror("io_uring_setup");
 700                 return 1;
 701         }
 702         s->ring_fd = fd;
 703
 704         io_uring_probe(fd);
 705
 706         if (fixedbufs) {
 707                 struct rlimit rlim;
 708
 709                 rlim.rlim_cur = RLIM_INFINITY;
 710                 rlim.rlim_max = RLIM_INFINITY;
 711                 /* ignore potential error, not needed on newer kernels */
 712                 setrlimit(RLIMIT_MEMLOCK, &rlim);
 713
 714                 ret = io_uring_register_buffers(s);
 715                 if (ret < 0) {
 716                         perror("io_uring_register_buffers");
 717                         return 1;
 718                 }
 719         }
 720
 721         if (register_files) {
 722                 ret = io_uring_register_files(s);
 723                 if (ret < 0) {
 724                         perror("io_uring_register_files");
 725                         return 1;
 726                 }
 727         }
 728
 729         ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
 730                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 731                         IORING_OFF_SQ_RING);
 732         printf("sq_ring ptr = 0x%p\n", ptr);
 733         sring->head = ptr + p.sq_off.head;
 734         sring->tail = ptr + p.sq_off.tail;
 735         sring->ring_mask = ptr + p.sq_off.ring_mask;
 736         sring->ring_entries = ptr + p.sq_off.ring_entries;
 737         sring->flags = ptr + p.sq_off.flags;
 738         sring->array = ptr + p.sq_off.array;
 739         sq_ring_mask = *sring->ring_mask;
 740
 741         s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
 742                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 743                         IORING_OFF_SQES);
 744         printf("sqes ptr    = 0x%p\n", s->sqes);
 745
 746         ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
 747                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 748                         IORING_OFF_CQ_RING);
 749         printf("cq_ring ptr = 0x%p\n", ptr);
 750         cring->head = ptr + p.cq_off.head;
 751         cring->tail = ptr + p.cq_off.tail;
 752         cring->ring_mask = ptr + p.cq_off.ring_mask;
 753         cring->ring_entries = ptr + p.cq_off.ring_entries;
 754         cring->cqes = ptr + p.cq_off.cqes;
 755         cq_ring_mask = *cring->ring_mask;
 756         return 0;
 757 }
 758
 759 static void file_depths(char *buf)
 760 {
 761         bool prev = false;
 762         char *p;
 763         int i, j;
 764
 765         buf[0] = '\0';
 766         p = buf;
 767         for (j = 0; j < nthreads; j++) {
 768                 struct submitter *s = get_submitter(j);
 769
 770                 for (i = 0; i < s->nr_files; i++) {
 771                         struct file *f = &s->files[i];
 772
 773                         if (prev)
 774                                 p += sprintf(p, " %d", f->pending_ios);
 775                         else
 776                                 p += sprintf(p, "%d", f->pending_ios);
 777                         prev = true;
 778                 }
 779         }
 780 }
 781
 782 static void usage(char *argv, int status)
 783 {
 784         printf("%s [options] -- [filenames]\n"
 785                 " -d <int>  : IO Depth, default %d\n"
 786                 " -s <int>  : Batch submit, default %d\n"
 787                 " -c <int>  : Batch complete, default %d\n"
 788                 " -b <int>  : Block size, default %d\n"
 789                 " -p <bool> : Polled IO, default %d\n"
 790                 " -B <bool> : Fixed buffers, default %d\n"
 791                 " -F <bool> : Register files, default %d\n"
 792                 " -n <int>  : Number of threads, default %d\n"
 793                 " -O <bool> : Use O_DIRECT, default %d\n"
 794                 " -N <bool> : Perform just no-op requests, default %d"
 795                 " -t <bool> : Track IO latencies, default %d"
 796                 " -T <int>  : TSC rate in HZ\n",
 797                 argv, DEPTH, BATCH_SUBMIT, BATCH_COMPLETE, BS, polled,
 798                 fixedbufs, register_files, nthreads, !buffered, do_nop, stats);
 799         exit(status);
 800 }
 801
 802 int main(int argc, char *argv[])
 803 {
 804         struct submitter *s;
 805         unsigned long done, calls, reap;
 806         int err, i, j, flags, fd, opt, threads_per_f, threads_rem = 0, nfiles;
 807         struct file f;
 808         char *fdepths;
 809         void *ret;
 810
 811         if (!do_nop && argc < 2)
 812                 usage(argv[0], 1);
 813
 814         while ((opt = getopt(argc, argv, "d:s:c:b:p:B:F:n:N:O:t:T:h?")) != -1) {
 815                 switch (opt) {
 816                 case 'd':
 817                         depth = atoi(optarg);
 818                         break;
 819                 case 's':
 820                         batch_submit = atoi(optarg);
 821                         if (!batch_submit)
 822                                 batch_submit = 1;
 823                         break;
 824                 case 'c':
 825                         batch_complete = atoi(optarg);
 826                         if (!batch_complete)
 827                                 batch_complete = 1;
 828                         break;
 829                 case 'b':
 830                         bs = atoi(optarg);
 831                         break;
 832                 case 'p':
 833                         polled = !!atoi(optarg);
 834                         break;
 835                 case 'B':
 836                         fixedbufs = !!atoi(optarg);
 837                         break;
 838                 case 'F':
 839                         register_files = !!atoi(optarg);
 840                         break;
 841                 case 'n':
 842                         nthreads = atoi(optarg);
 843                         if (!nthreads) {
 844                                 printf("Threads must be non-zero\n");
 845                                 usage(argv[0], 1);
 846                         }
 847                         break;
 848                 case 'N':
 849                         do_nop = !!atoi(optarg);
 850                         break;
 851                 case 'O':
 852                         buffered = !atoi(optarg);
 853                         break;
 854                 case 't':
 855 #ifndef ARCH_HAVE_CPU_CLOCK
 856                         fprintf(stderr, "Stats not supported on this CPU\n");
 857                         return 1;
 858 #endif
 859                         stats = !!atoi(optarg);
 860                         break;
 861                 case 'T':
 862 #ifndef ARCH_HAVE_CPU_CLOCK
 863                         fprintf(stderr, "Stats not supported on this CPU\n");
 864                         return 1;
 865 #endif
 866                         tsc_rate = strtoul(optarg, NULL, 10);
 867                         break;
 868                 case 'h':
 869                 case '?':
 870                 default:
 871                         usage(argv[0], 0);
 872                         break;
 873                 }
 874         }
 875
 876         if (batch_complete > depth)
 877                 batch_complete = depth;
 878         if (batch_submit > depth)
 879                 batch_submit = depth;
 880
 881         submitter = calloc(nthreads, sizeof(*submitter) +
 882                                 depth * sizeof(struct iovec));
 883         for (j = 0; j < nthreads; j++) {
 884                 s = get_submitter(j);
 885                 s->index = j;
 886                 s->done = s->calls = s->reaps = 0;
 887         }
 888
 889         flags = O_RDONLY | O_NOATIME;
 890         if (!buffered)
 891                 flags |= O_DIRECT;
 892
 893         j = 0;
 894         i = optind;
 895         nfiles = argc - i;
 896         if (!do_nop) {
 897                 if (!nfiles) {
 898                         printf("No files specified\n");
 899                         usage(argv[0], 1);
 900                 }
 901                 threads_per_f = nthreads / nfiles;
 902                 /* make sure each thread gets assigned files */
 903                 if (threads_per_f == 0) {
 904                         threads_per_f = 1;
 905                 } else {
 906                         threads_rem = nthreads - threads_per_f * nfiles;
 907                 }
 908         }
 909         while (!do_nop && i < argc) {
 910                 int k, limit;
 911
 912                 memset(&f, 0, sizeof(f));
 913
 914                 fd = open(argv[i], flags);
 915                 if (fd < 0) {
 916                         perror("open");
 917                         return 1;
 918                 }
 919                 f.real_fd = fd;
 920                 if (get_file_size(&f)) {
 921                         printf("failed getting size of device/file\n");
 922                         return 1;
 923                 }
 924                 if (f.max_blocks <= 1) {
 925                         printf("Zero file/device size?\n");
 926                         return 1;
 927                 }
 928                 f.max_blocks--;
 929
 930                 limit = threads_per_f;
 931                 limit += threads_rem > 0 ? 1 : 0;
 932                 for (k = 0; k < limit; k++) {
 933                         s = get_submitter((j + k) % nthreads);
 934
 935                         if (s->nr_files == MAX_FDS) {
 936                                 printf("Max number of files (%d) reached\n", MAX_FDS);
 937                                 break;
 938                         }
 939
 940                         memcpy(&s->files[s->nr_files], &f, sizeof(f));
 941
 942                         printf("Added file %s (submitter %d)\n", argv[i], s->index);
 943                         s->nr_files++;
 944                 }
 945                 threads_rem--;
 946                 i++;
 947                 j += limit;
 948         }
 949
 950         arm_sig_int();
 951
 952         for (j = 0; j < nthreads; j++) {
 953                 s = get_submitter(j);
 954                 for (i = 0; i < depth; i++) {
 955                         void *buf;
 956
 957                         if (posix_memalign(&buf, bs, bs)) {
 958                                 printf("failed alloc\n");
 959                                 return 1;
 960                         }
 961                         s->iovecs[i].iov_base = buf;
 962                         s->iovecs[i].iov_len = bs;
 963                 }
 964         }
 965
 966         for (j = 0; j < nthreads; j++) {
 967                 s = get_submitter(j);
 968
 969                 err = setup_ring(s);
 970                 if (err) {
 971                         printf("ring setup failed: %s, %d\n", strerror(errno), err);
 972                         return 1;
 973                 }
 974         }
 975         s = get_submitter(0);
 976         printf("polled=%d, fixedbufs=%d, register_files=%d, buffered=%d", polled, fixedbufs, register_files, buffered);
 977         printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", depth, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
 978
 979         for (j = 0; j < nthreads; j++) {
 980                 s = get_submitter(j);
 981                 pthread_create(&s->thread, NULL, submitter_fn, s);
 982         }
 983
 984         fdepths = malloc(8 * s->nr_files * nthreads);
 985         reap = calls = done = 0;
 986         do {
 987                 unsigned long this_done = 0;
 988                 unsigned long this_reap = 0;
 989                 unsigned long this_call = 0;
 990                 unsigned long rpc = 0, ipc = 0;
 991                 unsigned long iops, bw;
 992
 993                 sleep(1);
 994                 for (j = 0; j < nthreads; j++) {
 995                         this_done += s->done;
 996                         this_call += s->calls;
 997                         this_reap += s->reaps;
 998                 }
 999                 if (this_call - calls) {
1000                         rpc = (this_done - done) / (this_call - calls);
1001                         ipc = (this_reap - reap) / (this_call - calls);
1002                 } else
1003                         rpc = ipc = -1;
1004                 file_depths(fdepths);
1005                 iops = this_done - done;
1006                 if (bs > 1048576)
1007                         bw = iops * (bs / 1048576);
1008                 else
1009                         bw = iops / (1048576 / bs);
1010                 printf("IOPS=%lu, ", iops);
1011                 if (!do_nop)
1012                         printf("BW=%luMiB/s, ", bw);
1013                 printf("IOS/call=%ld/%ld, inflight=(%s)\n", rpc, ipc, fdepths);
1014                 done = this_done;
1015                 calls = this_call;
1016                 reap = this_reap;
1017         } while (!finish);
1018
1019         for (j = 0; j < nthreads; j++) {
1020                 s = get_submitter(j);
1021                 pthread_join(s->thread, &ret);
1022                 close(s->ring_fd);
1023
1024                 if (stats) {
1025                         unsigned long nr;
1026
1027                         printf("%d: Latency percentiles:\n", s->tid);
1028                         for (i = 0, nr = 0; i < PLAT_NR; i++)
1029                                 nr += s->plat[i];
1030                         show_clat_percentiles(s->plat, nr, 4);
1031                         free(s->clock_batch);
1032                         free(s->plat);
1033                 }
1034         }
1035
1036         free(fdepths);
1037         free(submitter);
1038         return 0;
1039 }