fio.c

   1 /*
   2  * fio - the flexible io tester
   3  *
   4  * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 2 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <fcntl.h>
  25 #include <string.h>
  26 #include <errno.h>
  27 #include <signal.h>
  28 #include <time.h>
  29 #include <ctype.h>
  30 #include <sched.h>
  31 #include <libaio.h>
  32 #include <math.h>
  33 #include <limits.h>
  34 #include <sys/time.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <sys/wait.h>
  38 #include <semaphore.h>
  39 #include <sys/ipc.h>
  40 #include <sys/shm.h>
  41 #include <asm/unistd.h>
  42 #include <asm/types.h>
  43
  44 #include "list.h"
  45 #include "md5.h"
  46
  47 #define MAX_JOBS        (1024)
  48
  49 /*
  50  * assume we don't have _get either, if _set isn't defined
  51  */
  52 #ifndef __NR_ioprio_set
  53 #if defined(__i386__)
  54 #define __NR_ioprio_set         289
  55 #define __NR_ioprio_get         290
  56 #elif defined(__powerpc__) || defined(__powerpc64__)
  57 #define __NR_ioprio_set         273
  58 #define __NR_ioprio_get         274
  59 #elif defined(__x86_64__)
  60 #define __NR_ioprio_set         251
  61 #define __NR_ioprio_get         252
  62 #elif defined(__ia64__)
  63 #define __NR_ioprio_set         1274
  64 #define __NR_ioprio_get         1275
  65 #elif defined(__alpha__)
  66 #define __NR_ioprio_set         442
  67 #define __NR_ioprio_get         443
  68 #elif defined(__s390x__) || defined(__s390__)
  69 #define __NR_ioprio_set         282
  70 #define __NR_ioprio_get         283
  71 #else
  72 #error "Unsupported arch"
  73 #endif
  74 #endif
  75
  76 #ifndef __NR_fadvise64
  77 #if defined(__i386__)
  78 #define __NR_fadvise64          250
  79 #elif defined(__powerpc__) || defined(__powerpc64__)
  80 #define __NR_fadvise64          233
  81 #elif defined(__x86_64__)
  82 #define __NR_fadvise64          221
  83 #elif defined(__ia64__)
  84 #define __NR_fadvise64          1234
  85 #elif defined(__alpha__)
  86 #define __NR_fadvise64          413
  87 #elif defined(__s390x__) || defined(__s390__)
  88 #define __NR_fadvise64          253
  89 #else
  90 #error "Unsupported arch"
  91 #endif
  92 #endif
  93
  94 static int ioprio_set(int which, int who, int ioprio)
  95 {
  96         return syscall(__NR_ioprio_set, which, who, ioprio);
  97 }
  98
  99 /*
 100  * we want fadvise64 really, but it's so tangled... later
 101  */
 102 static int fadvise(int fd, loff_t offset, size_t len, int advice)
 103 {
 104 #if 0
 105         return syscall(__NR_fadvise64, fd, offset, offset >> 32, len, advice);
 106 #else
 107         return posix_fadvise(fd, (off_t) offset, len, advice);
 108 #endif
 109 }
 110
 111 enum {
 112         IOPRIO_WHO_PROCESS = 1,
 113         IOPRIO_WHO_PGRP,
 114         IOPRIO_WHO_USER,
 115 };
 116
 117 #define IOPRIO_CLASS_SHIFT      13
 118
 119 #define MASK    (4095)
 120
 121 #define DEF_BS          (4096)
 122 #define DEF_TIMEOUT     (30)
 123 #define DEF_RATE_CYCLE  (1000)
 124 #define DEF_ODIRECT     (1)
 125 #define DEF_SEQUENTIAL  (1)
 126 #define DEF_RAND_REPEAT (1)
 127 #define DEF_OVERWRITE   (0)
 128 #define DEF_CREATE      (1)
 129 #define DEF_INVALIDATE  (1)
 130 #define DEF_SYNCIO      (0)
 131 #define DEF_RANDSEED    (0xb1899bedUL)
 132 #define DEF_BWAVGTIME   (500)
 133 #define DEF_CREATE_SER  (1)
 134 #define DEF_CREATE_FSYNC        (1)
 135 #define DEF_LOOPS       (1)
 136 #define DEF_VERIFY      (0)
 137
 138 #define ALIGN(buf)      (char *) (((unsigned long) (buf) + MASK) & ~(MASK))
 139
 140 static int repeatable = DEF_RAND_REPEAT;
 141 static int rate_quit = 1;
 142 static int write_lat_log;
 143 static int write_bw_log;
 144 static int exitall_on_terminate;
 145
 146 static int thread_number;
 147 static char *ini_file;
 148
 149 static int max_jobs = MAX_JOBS;
 150
 151 static char run_str[MAX_JOBS + 1];
 152
 153 static int shm_id;
 154
 155 enum {
 156         DDIR_READ = 0,
 157         DDIR_WRITE,
 158 };
 159
 160 /*
 161  * thread life cycle
 162  */
 163 enum {
 164         TD_NOT_CREATED = 0,
 165         TD_CREATED,
 166         TD_RUNNING,
 167         TD_VERIFYING,
 168         TD_EXITED,
 169         TD_REAPED,
 170 };
 171
 172 enum {
 173         MEM_MALLOC,
 174         MEM_SHM,
 175 };
 176
 177 /*
 178  * The io unit
 179  */
 180 struct io_u {
 181         struct iocb iocb;
 182         struct timeval start_time;
 183         struct timeval issue_time;
 184
 185         char *buf;
 186         unsigned int buflen;
 187         unsigned long long offset;
 188
 189         struct list_head list;
 190 };
 191
 192 struct io_stat {
 193         unsigned long val;
 194         unsigned long val_sq;
 195         unsigned long max_val;
 196         unsigned long min_val;
 197         unsigned long samples;
 198 };
 199
 200 struct io_sample {
 201         unsigned long time;
 202         unsigned long val;
 203 };
 204
 205 struct io_log {
 206         unsigned long nr_samples;
 207         unsigned long max_samples;
 208         struct io_sample *log;
 209 };
 210
 211 #define FIO_HDR_MAGIC   0xf00baaef
 212
 213 struct verify_header {
 214         unsigned int fio_magic;
 215         unsigned int len;
 216         char md5_digest[MD5_HASH_WORDS * 4];
 217 };
 218
 219 #define td_read(td)             ((td)->ddir == DDIR_READ)
 220 #define should_fsync(td)        (!td_read(td) && !(td)->odirect)
 221
 222 struct thread_data {
 223         char file_name[256];
 224         int thread_number;
 225         int error;
 226         int fd;
 227         pid_t pid;
 228         char *orig_buffer;
 229         volatile int terminate;
 230         volatile int runstate;
 231         volatile int old_runstate;
 232         unsigned int ddir;
 233         unsigned int ioprio;
 234         unsigned int sequential;
 235         unsigned int bs;
 236         unsigned int min_bs;
 237         unsigned int max_bs;
 238         unsigned int odirect;
 239         unsigned int thinktime;
 240         unsigned int fsync_blocks;
 241         unsigned int start_delay;
 242         unsigned int timeout;
 243         unsigned int use_aio;
 244         unsigned int create_file;
 245         unsigned int overwrite;
 246         unsigned int invalidate_cache;
 247         unsigned int bw_avg_time;
 248         unsigned int create_serialize;
 249         unsigned int create_fsync;
 250         unsigned int loops;
 251         unsigned long long file_size;
 252         unsigned long long file_offset;
 253         unsigned int sync_io;
 254         unsigned int mem_type;
 255         unsigned int verify;
 256         cpu_set_t cpumask;
 257
 258         struct drand48_data bsrange_state;
 259         struct drand48_data verify_state;
 260
 261         int shm_id;
 262
 263         off_t cur_off;
 264
 265         io_context_t aio_ctx;
 266         unsigned int aio_depth;
 267         struct io_event *aio_events;
 268
 269         unsigned int cur_depth;
 270         struct list_head io_u_freelist;
 271         struct list_head io_u_busylist;
 272
 273         unsigned int rate;
 274         unsigned int ratemin;
 275         unsigned int ratecycle;
 276         unsigned long rate_usec_cycle;
 277         long rate_pending_usleep;
 278         unsigned long rate_kb;
 279         struct timeval lastrate;
 280
 281         unsigned long runtime;          /* sec */
 282         unsigned long kb;
 283         unsigned long io_blocks;
 284         unsigned long io_kb;
 285         unsigned long this_io_kb;
 286         unsigned long last_kb;
 287         sem_t mutex;
 288         struct drand48_data random_state;
 289
 290         /*
 291          * bandwidth and latency stats
 292          */
 293         struct io_stat clat_stat;               /* completion latency */
 294         struct io_stat slat_stat;               /* submission latency */
 295
 296         struct io_stat bw_stat;                 /* bandwidth stats */
 297         unsigned long stat_io_kb;
 298         struct timeval stat_sample_time;
 299
 300         struct io_log *lat_log;
 301         struct io_log *bw_log;
 302
 303         struct timeval start;
 304 };
 305
 306 static struct thread_data *threads;
 307 static struct thread_data def_thread;
 308
 309 static sem_t startup_sem;
 310
 311 static void sig_handler(int sig)
 312 {
 313         int i;
 314
 315         for (i = 0; i < thread_number; i++) {
 316                 struct thread_data *td = &threads[i];
 317
 318                 td->terminate = 1;
 319                 td->start_delay = 0;
 320         }
 321 }
 322
 323 static int init_random_state(struct thread_data *td)
 324 {
 325         unsigned long seed;
 326         int fd;
 327
 328         fd = open("/dev/random", O_RDONLY);
 329         if (fd == -1) {
 330                 td->error = errno;
 331                 return 1;
 332         }
 333
 334         if (read(fd, &seed, sizeof(seed)) < (int) sizeof(seed)) {
 335                 td->error = EIO;
 336                 close(fd);
 337                 return 1;
 338         }
 339
 340         close(fd);
 341
 342         srand48_r(seed, &td->bsrange_state);
 343         srand48_r(seed, &td->verify_state);
 344
 345         if (td->sequential)
 346                 return 0;
 347
 348         if (repeatable)
 349                 seed = DEF_RANDSEED;
 350
 351         srand48_r(seed, &td->random_state);
 352         return 0;
 353 }
 354
 355 static unsigned long utime_since(struct timeval *s, struct timeval *e)
 356 {
 357         double sec, usec;
 358
 359         sec = e->tv_sec - s->tv_sec;
 360         usec = e->tv_usec - s->tv_usec;
 361         if (sec > 0 && usec < 0) {
 362                 sec--;
 363                 usec += 1000000;
 364         }
 365
 366         sec *= (double) 1000000;
 367
 368         return sec + usec;
 369 }
 370
 371 static unsigned long mtime_since(struct timeval *s, struct timeval *e)
 372 {
 373         double sec, usec;
 374
 375         sec = e->tv_sec - s->tv_sec;
 376         usec = e->tv_usec - s->tv_usec;
 377         if (sec > 0 && usec < 0) {
 378                 sec--;
 379                 usec += 1000000;
 380         }
 381
 382         sec *= (double) 1000;
 383         usec /= (double) 1000;
 384
 385         return sec + usec;
 386 }
 387
 388 static unsigned long mtime_since_now(struct timeval *s)
 389 {
 390         struct timeval t;
 391
 392         gettimeofday(&t, NULL);
 393         return mtime_since(s, &t);
 394 }
 395
 396 static inline unsigned long msec_now(struct timeval *s)
 397 {
 398         return s->tv_sec * 1000 + s->tv_usec / 1000;
 399 }
 400
 401 static unsigned long long get_next_offset(struct thread_data *td)
 402 {
 403         unsigned long long kb;
 404         double r;
 405
 406         if (!td->sequential) {
 407                 drand48_r(&td->random_state, &r);
 408                 kb = (1+(double) (td->kb-1) * r / (RAND_MAX+1.0));
 409         } else
 410                 kb = td->last_kb;
 411
 412         return (kb << 10) + td->file_offset;
 413 }
 414
 415 static unsigned int get_next_buflen(struct thread_data *td)
 416 {
 417         unsigned int buflen;
 418         double r;
 419
 420         if (td->min_bs == td->max_bs)
 421                 buflen = td->min_bs;
 422         else {
 423                 drand48_r(&td->bsrange_state, &r);
 424                 buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0));
 425                 buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1);
 426         }
 427
 428         if (buflen > ((td->kb - td->this_io_kb) << 10))
 429                 buflen = (td->kb - td->this_io_kb) << 10;
 430
 431         td->last_kb += buflen >> 10;
 432         return buflen;
 433 }
 434
 435 static inline void add_stat_sample(struct thread_data *td, struct io_stat *is,
 436                                    unsigned long val)
 437 {
 438         if (val > is->max_val)
 439                 is->max_val = val;
 440         if (val < is->min_val)
 441                 is->min_val = val;
 442
 443         is->val += val;
 444         is->val_sq += val * val;
 445         is->samples++;
 446 }
 447
 448 static void add_log_sample(struct thread_data *td, struct io_log *log,
 449                            unsigned long val)
 450 {
 451         if (log->nr_samples == log->max_samples) {
 452                 int new_size = sizeof(struct io_sample) * log->max_samples * 2;
 453
 454                 log->log = realloc(log->log, new_size);
 455                 log->max_samples <<= 1;
 456         }
 457
 458         log->log[log->nr_samples].val = val;
 459         log->log[log->nr_samples].time = mtime_since_now(&td->start);
 460         log->nr_samples++;
 461 }
 462
 463 static void add_clat_sample(struct thread_data *td, unsigned long msec)
 464 {
 465         add_stat_sample(td, &td->clat_stat, msec);
 466
 467         if (td->lat_log)
 468                 add_log_sample(td, td->lat_log, msec);
 469 }
 470
 471 static void add_slat_sample(struct thread_data *td, unsigned long msec)
 472 {
 473         add_stat_sample(td, &td->slat_stat, msec);
 474 }
 475
 476 static void add_bw_sample(struct thread_data *td, unsigned long msec)
 477 {
 478         unsigned long spent = mtime_since_now(&td->stat_sample_time);
 479         unsigned long rate;
 480
 481         if (spent < td->bw_avg_time)
 482                 return;
 483
 484         rate = ((td->this_io_kb - td->stat_io_kb) * 1024) / spent;
 485         add_stat_sample(td, &td->bw_stat, rate);
 486
 487         if (td->bw_log)
 488                 add_log_sample(td, td->bw_log, rate);
 489
 490         gettimeofday(&td->stat_sample_time, NULL);
 491         td->stat_io_kb = td->this_io_kb;
 492 }
 493
 494 static void usec_sleep(int usec)
 495 {
 496         struct timespec req = { .tv_sec = 0, .tv_nsec = usec * 1000 };
 497         struct timespec rem;
 498
 499         do {
 500                 rem.tv_sec = rem.tv_nsec = 0;
 501                 nanosleep(&req, &rem);
 502                 if (!rem.tv_nsec)
 503                         break;
 504
 505                 req.tv_nsec = rem.tv_nsec;
 506         } while (1);
 507 }
 508
 509 static void rate_throttle(struct thread_data *td, unsigned long time_spent,
 510                           unsigned int bytes)
 511 {
 512         unsigned long usec_cycle;
 513
 514         if (!td->rate)
 515                 return;
 516
 517         usec_cycle = td->rate_usec_cycle * (bytes / td->min_bs);
 518
 519         if (time_spent < usec_cycle) {
 520                 unsigned long s = usec_cycle - time_spent;
 521
 522                 td->rate_pending_usleep += s;
 523                 if (td->rate_pending_usleep >= 100000) {
 524                         usec_sleep(td->rate_pending_usleep);
 525                         td->rate_pending_usleep = 0;
 526                 }
 527         } else {
 528                 long overtime = time_spent - usec_cycle;
 529
 530                 td->rate_pending_usleep -= overtime;
 531         }
 532 }
 533
 534 static int check_min_rate(struct thread_data *td, struct timeval *now)
 535 {
 536         unsigned long spent;
 537         unsigned long rate;
 538
 539         /*
 540          * allow a 2 second settle period in the beginning
 541          */
 542         if (mtime_since(&td->start, now) < 2000)
 543                 return 0;
 544
 545         /*
 546          * if rate blocks is set, sample is running
 547          */
 548         if (td->rate_kb) {
 549                 spent = mtime_since(&td->lastrate, now);
 550                 if (spent < td->ratecycle)
 551                         return 0;
 552
 553                 rate = ((td->this_io_kb - td->rate_kb) * 1024) / spent;
 554                 if (rate < td->ratemin) {
 555                         printf("Client%d: min rate %d not met, got %ldKiB/sec\n", td->thread_number, td->ratemin, rate);
 556                         if (rate_quit)
 557                                 sig_handler(0);
 558                         return 1;
 559                 }
 560         }
 561
 562         td->rate_kb = td->this_io_kb;
 563         memcpy(&td->lastrate, now, sizeof(*now));
 564         return 0;
 565 }
 566
 567 static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
 568 {
 569         if (mtime_since(&td->start, t) >= td->timeout * 1000)
 570                 return 1;
 571
 572         return 0;
 573 }
 574
 575 static void fill_random_bytes(struct thread_data *td,
 576                               unsigned char *p, unsigned int len)
 577 {
 578         double r;
 579         int todo;
 580
 581         while (len) {
 582                 drand48_r(&td->verify_state, &r);
 583
 584                 /*
 585                  * lrand48_r seems to be broken and only fill the bottom
 586                  * 32-bits, even on 64-bit archs with 64-bit longs
 587                  */
 588                 todo = sizeof(r);
 589                 if (todo > len)
 590                         todo = len;
 591
 592                 memcpy(p, &r, todo);
 593
 594                 len -= todo;
 595                 p += todo;
 596         }
 597 }
 598
 599 static int verify_io_u(struct thread_data *td, struct io_u *io_u)
 600 {
 601         struct verify_header *hdr = (struct verify_header *) io_u->buf;
 602         unsigned char *p = (unsigned char *) io_u->buf;
 603         struct md5_ctx md5_ctx;
 604
 605         if (hdr->fio_magic != FIO_HDR_MAGIC)
 606                 return 1;
 607
 608         memset(&md5_ctx, 0, sizeof(md5_ctx));
 609         p += sizeof(*hdr);
 610         md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
 611
 612         return memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
 613 }
 614
 615 static int verify_io_us(struct thread_data *td, struct io_u *io_u, int *back)
 616 {
 617         struct verify_header *hdr;
 618         unsigned int left;
 619         struct io_u i;
 620         char *buf;
 621         off_t off;
 622
 623         if (back)
 624                 *back = 0;
 625
 626         left = io_u->buflen;
 627         buf = io_u->buf;
 628         off = io_u->offset;
 629         while (left) {
 630                 hdr = (struct verify_header *) buf;
 631                 i.buf = buf;
 632                 i.buflen = hdr->len;
 633
 634                 if (hdr->len > left) {
 635                         if (back)
 636                                 *back = left;
 637                         return 0;
 638                 }
 639
 640                 if (verify_io_u(td, &i)) {
 641                         printf("failed verify at offset %lu\n", (unsigned long) off);
 642                         td->error = EBADMSG;
 643                         return 1;
 644                 }
 645
 646                 buf += hdr->len;
 647                 left -= hdr->len;
 648                 off += hdr->len;
 649         }
 650
 651         return 0;
 652 }
 653
 654 /*
 655  * fill body of io_u->buf with random data and add a header with the
 656  * (eg) sha1sum of that data.
 657  */
 658 static void populate_io_u(struct thread_data *td, struct io_u *io_u)
 659 {
 660         struct md5_ctx md5_ctx;
 661         struct verify_header hdr;
 662         unsigned char *p = (unsigned char *) io_u->buf;
 663
 664         hdr.fio_magic = FIO_HDR_MAGIC;
 665         hdr.len = io_u->buflen;
 666         p += sizeof(hdr);
 667         fill_random_bytes(td, p, io_u->buflen - sizeof(hdr));
 668
 669         memset(&md5_ctx, 0, sizeof(md5_ctx));
 670         md5_update(&md5_ctx, p, io_u->buflen - sizeof(hdr));
 671         memcpy(hdr.md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
 672         memcpy(io_u->buf, &hdr, sizeof(hdr));
 673 }
 674
 675 static void put_io_u(struct thread_data *td, struct io_u *io_u)
 676 {
 677         list_del(&io_u->list);
 678         list_add(&io_u->list, &td->io_u_freelist);
 679         td->cur_depth--;
 680 }
 681
 682 #define queue_full(td)  (list_empty(&(td)->io_u_freelist))
 683
 684 static struct io_u *__get_io_u(struct thread_data *td)
 685 {
 686         struct io_u *io_u;
 687
 688         if (queue_full(td))
 689                 return NULL;
 690
 691         io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
 692         list_del(&io_u->list);
 693         list_add(&io_u->list, &td->io_u_busylist);
 694         td->cur_depth++;
 695         return io_u;
 696 }
 697
 698 static struct io_u *get_io_u(struct thread_data *td)
 699 {
 700         struct io_u *io_u;
 701
 702         io_u = __get_io_u(td);
 703         if (!io_u)
 704                 return NULL;
 705
 706         io_u->offset = get_next_offset(td);
 707         io_u->buflen = get_next_buflen(td);
 708         if (!io_u->buflen) {
 709                 put_io_u(td, io_u);
 710                 return NULL;
 711         }
 712
 713         if (!td_read(td) && td->verify)
 714                 populate_io_u(td, io_u);
 715
 716         if (td->use_aio) {
 717                 if (td_read(td))
 718                         io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
 719                 else
 720                         io_prep_pwrite(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
 721         }
 722
 723         gettimeofday(&io_u->start_time, NULL);
 724         return io_u;
 725 }
 726
 727 static inline void td_set_runstate(struct thread_data *td, int runstate)
 728 {
 729         td->old_runstate = td->runstate;
 730         td->runstate = runstate;
 731 }
 732
 733 static int do_sync_verify(struct thread_data *td)
 734 {
 735         struct timeval t;
 736         struct io_u *io_u = NULL;
 737         loff_t off = 0;
 738         int back, ret;
 739
 740         td_set_runstate(td, TD_VERIFYING);
 741
 742         io_u = __get_io_u(td);
 743
 744         if (lseek(td->fd, 0, SEEK_SET) < 0) {
 745                 td->error = errno;
 746                 goto out;
 747         }
 748
 749         if (!td->odirect) {
 750                 unsigned long size = td->kb << 10;
 751
 752                 if (fadvise(td->fd, 0, size, POSIX_FADV_DONTNEED) < 0) {
 753                         td->error = errno;
 754                         goto out;
 755                 }
 756         }
 757
 758         do {
 759                 if (td->terminate)
 760                         break;
 761
 762                 gettimeofday(&t, NULL);
 763                 if (runtime_exceeded(td, &t))
 764                         break;
 765
 766                 io_u->offset = off;
 767                 io_u->buflen = td->max_bs;
 768
 769                 ret = read(td->fd, io_u->buf, io_u->buflen);
 770                 if (ret < (int) io_u->buflen) {
 771                         if (ret == -1) {
 772                                 td->error = errno;
 773                                 break;
 774                         } else if (!ret)
 775                                 break;
 776                         else
 777                                 io_u->buflen = ret;
 778                 }
 779
 780                 if (verify_io_us(td, io_u, &back))
 781                         break;
 782
 783                 if (back) {
 784                         ret -= back;
 785                         if (lseek(td->fd, -back, SEEK_CUR) < 0) {
 786                                 td->error = errno;
 787                                 break;
 788                         }
 789                 }
 790
 791                 off += ret;
 792         } while (1);
 793
 794 out:
 795         td_set_runstate(td, TD_RUNNING);
 796         put_io_u(td, io_u);
 797
 798         return td->error == 0;
 799 }
 800
 801 static void do_sync_io(struct thread_data *td)
 802 {
 803         unsigned long msec, usec;
 804         struct io_u *io_u = NULL;
 805         struct timeval e;
 806
 807         td->cur_off = 0;
 808         td->last_kb = 0;
 809
 810         for (td->this_io_kb = 0; td->this_io_kb < td->kb;) {
 811                 int ret;
 812
 813                 if (td->terminate)
 814                         break;
 815
 816                 io_u = get_io_u(td);
 817                 if (!io_u)
 818                         break;
 819
 820                 if (td->cur_off != io_u->offset) {
 821                         if (lseek(td->fd, io_u->offset, SEEK_SET) == -1) {
 822                                 td->error = errno;
 823                                 break;
 824                         }
 825                 }
 826
 827                 if (td_read(td))
 828                         ret = read(td->fd, io_u->buf, io_u->buflen);
 829                 else
 830                         ret = write(td->fd, io_u->buf, io_u->buflen);
 831
 832                 if (ret < (int) io_u->buflen) {
 833                         if (ret == -1)
 834                                 td->error = errno;
 835                         break;
 836                 }
 837
 838                 td->io_blocks++;
 839                 td->io_kb += io_u->buflen >> 10;
 840                 td->this_io_kb += io_u->buflen >> 10;
 841                 td->cur_off = io_u->offset + io_u->buflen;
 842
 843                 gettimeofday(&e, NULL);
 844
 845                 usec = utime_since(&io_u->start_time, &e);
 846
 847                 rate_throttle(td, usec, io_u->buflen);
 848
 849                 if (check_min_rate(td, &e)) {
 850                         td->error = ENODATA;
 851                         break;
 852                 }
 853
 854                 msec = usec / 1000;
 855                 add_clat_sample(td, msec);
 856                 add_bw_sample(td, msec);
 857
 858                 if (runtime_exceeded(td, &e))
 859                         break;
 860
 861                 put_io_u(td, io_u);
 862                 io_u = NULL;
 863
 864                 if (td->thinktime)
 865                         usec_sleep(td->thinktime);
 866
 867                 if (should_fsync(td) && td->fsync_blocks &&
 868                     (td->io_blocks % td->fsync_blocks) == 0)
 869                         fsync(td->fd);
 870         }
 871
 872         if (io_u)
 873                 put_io_u(td, io_u);
 874
 875         if (should_fsync(td))
 876                 fsync(td->fd);
 877 }
 878
 879 static int io_u_queue(struct thread_data *td, struct io_u *io_u)
 880 {
 881         struct iocb *iocb = &io_u->iocb;
 882         int ret;
 883
 884         do {
 885                 ret = io_submit(td->aio_ctx, 1, &iocb);
 886                 if (ret == 1)
 887                         return 0;
 888                 else if (ret == EAGAIN)
 889                         usleep(100);
 890                 else if (ret == EINTR)
 891                         continue;
 892                 else
 893                         break;
 894         } while (1);
 895
 896         return ret;
 897 }
 898
 899 #define iocb_time(iocb) ((unsigned long) (iocb)->data)
 900 #define ev_to_iou(ev)   (struct io_u *) ((unsigned long) (ev)->obj)
 901
 902 static int ios_completed(struct thread_data *td, int nr)
 903 {
 904         unsigned long msec;
 905         struct io_u *io_u;
 906         struct timeval e;
 907         int i, bytes_done;
 908
 909         gettimeofday(&e, NULL);
 910
 911         for (i = 0, bytes_done = 0; i < nr; i++) {
 912                 io_u = ev_to_iou(td->aio_events + i);
 913
 914                 td->io_blocks++;
 915                 td->io_kb += io_u->buflen >> 10;
 916                 td->this_io_kb += io_u->buflen >> 10;
 917
 918                 msec = mtime_since(&io_u->issue_time, &e);
 919
 920                 add_clat_sample(td, msec);
 921                 add_bw_sample(td, msec);
 922
 923                 bytes_done += io_u->buflen;
 924                 put_io_u(td, io_u);
 925         }
 926
 927         return bytes_done;
 928 }
 929
 930 static void cleanup_pending_aio(struct thread_data *td)
 931 {
 932         struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
 933         struct list_head *entry, *n;
 934         struct io_u *io_u;
 935         int r;
 936
 937         /*
 938          * get immediately available events, if any
 939          */
 940         r = io_getevents(td->aio_ctx, 0, td->cur_depth, td->aio_events, &ts);
 941         if (r > 0)
 942                 ios_completed(td, r);
 943
 944         /*
 945          * now cancel remaining active events
 946          */
 947         list_for_each_safe(entry, n, &td->io_u_busylist) {
 948                 io_u = list_entry(entry, struct io_u, list);
 949
 950                 r = io_cancel(td->aio_ctx, &io_u->iocb, td->aio_events);
 951                 if (!r)
 952                         put_io_u(td, io_u);
 953         }
 954
 955         if (td->cur_depth) {
 956                 r = io_getevents(td->aio_ctx, td->cur_depth, td->cur_depth, td->aio_events, NULL);
 957                 if (r > 0)
 958                         ios_completed(td, r);
 959         }
 960 }
 961
 962 static int do_async_verify(struct thread_data *td)
 963 {
 964         struct timeval t;
 965         struct io_u *io_u, *v_io_u;
 966         struct verify_header *hdr;
 967         int ret, back;
 968         char *p;
 969
 970         td_set_runstate(td, TD_VERIFYING);
 971
 972         td->cur_off = 0;
 973         td->last_kb = 0;
 974         v_io_u = NULL;
 975
 976         do {
 977                 if (td->terminate)
 978                         break;
 979
 980                 gettimeofday(&t, NULL);
 981                 if (runtime_exceeded(td, &t))
 982                         break;
 983
 984                 io_u = __get_io_u(td);
 985                 if (!io_u)
 986                         break;
 987
 988                 io_u->offset = td->cur_off;
 989                 io_u->buflen = td->max_bs;
 990
 991                 if (io_u->offset + io_u->buflen > (td->kb << 10)) {
 992                         io_u->buflen = (td->kb << 10) - io_u->offset;
 993                         if (!io_u->buflen) {
 994                                 put_io_u(td, io_u);
 995                                 break;
 996                         }
 997                 }
 998
 999                 io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
1000                 ret = io_u_queue(td, io_u);
1001                 if (ret) {
1002                         put_io_u(td, io_u);
1003                         td->error = ret;
1004                         break;
1005                 }
1006
1007                 /*
1008                  * we have one pending to verify, do that while the next
1009                  * we are doing io on the next one
1010                  */
1011                 if (v_io_u) {
1012                         ret = verify_io_us(td, v_io_u, NULL);
1013                         put_io_u(td, v_io_u);
1014                         v_io_u = NULL;
1015                         if (ret)
1016                                 break;
1017                 }
1018
1019                 ret = io_getevents(td->aio_ctx, 1, 1, td->aio_events, NULL);
1020                 if (ret != 1) {
1021                         if (ret < 0)
1022                                 td->error = ret;
1023                         break;
1024                 }
1025
1026                 /*
1027                  * got our io_u to verify, find back offset so we can
1028                  * submit the next one before verifying this one
1029                  */
1030                 v_io_u = ev_to_iou(td->aio_events);
1031                 p = v_io_u->buf;
1032                 back = v_io_u->buflen;
1033                 do {
1034                         hdr = (struct verify_header *) p;
1035
1036                         if (hdr->len > back)
1037                                 break;
1038
1039                         back -= hdr->len;
1040                         p += hdr->len;
1041                 } while (back);
1042
1043                 td->cur_off += (v_io_u->buflen - back);
1044
1045                 /*
1046                  * if max depth is 1, we need to verify now
1047                  */
1048                 if (queue_full(td)) {
1049                         ret = verify_io_us(td, v_io_u, NULL);
1050                         put_io_u(td, v_io_u);
1051                         v_io_u = NULL;
1052                         if (ret)
1053                                 break;
1054                 }
1055         } while (1);
1056
1057         if (v_io_u) {
1058                 verify_io_us(td, v_io_u, NULL);
1059                 put_io_u(td, v_io_u);
1060         }
1061
1062         if (td->cur_depth)
1063                 cleanup_pending_aio(td);
1064
1065         td_set_runstate(td, TD_RUNNING);
1066         return td->error == 0;
1067 }
1068
1069 static void do_async_io(struct thread_data *td)
1070 {
1071         struct timeval s, e;
1072         unsigned long usec;
1073
1074         td->cur_off = 0;
1075         td->last_kb = 0;
1076
1077         for (td->this_io_kb = 0; td->this_io_kb < td->kb;) {
1078                 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
1079                 struct timespec *timeout;
1080                 int ret, min_evts = 0;
1081                 struct io_u *io_u;
1082                 unsigned int bytes_done;
1083
1084                 if (td->terminate)
1085                         break;
1086
1087                 io_u = get_io_u(td);
1088                 if (!io_u)
1089                         break;
1090
1091                 memcpy(&s, &io_u->start_time, sizeof(s));
1092
1093                 ret = io_u_queue(td, io_u);
1094                 if (ret) {
1095                         put_io_u(td, io_u);
1096                         td->error = ret;
1097                         break;
1098                 }
1099
1100                 gettimeofday(&io_u->issue_time, NULL);
1101                 add_slat_sample(td, mtime_since(&io_u->start_time, &io_u->issue_time));
1102                 if (td->cur_depth < td->aio_depth) {
1103                         timeout = &ts;
1104                         min_evts = 0;
1105                 } else {
1106                         timeout = NULL;
1107                         min_evts = 1;
1108                 }
1109
1110                 ret = io_getevents(td->aio_ctx, min_evts, td->cur_depth, td->aio_events, timeout);
1111                 if (ret < 0) {
1112                         td->error = errno;
1113                         break;
1114                 } else if (!ret)
1115                         continue;
1116
1117                 bytes_done = ios_completed(td, ret);
1118
1119                 /*
1120                  * the rate is batched for now, it should work for batches
1121                  * of completions except the very first one which may look
1122                  * a little bursty
1123                  */
1124                 gettimeofday(&e, NULL);
1125                 usec = utime_since(&s, &e);
1126
1127                 rate_throttle(td, usec, bytes_done);
1128
1129                 if (check_min_rate(td, &e)) {
1130                         td->error = ENODATA;
1131                         break;
1132                 }
1133
1134                 if (runtime_exceeded(td, &e))
1135                         break;
1136
1137                 if (td->thinktime)
1138                         usec_sleep(td->thinktime);
1139
1140                 if (should_fsync(td) && td->fsync_blocks &&
1141                     (td->io_blocks % td->fsync_blocks) == 0)
1142                         fsync(td->fd);
1143         }
1144
1145         if (td->cur_depth)
1146                 cleanup_pending_aio(td);
1147
1148         if (should_fsync(td))
1149                 fsync(td->fd);
1150 }
1151
1152 static void cleanup_aio(struct thread_data *td)
1153 {
1154         io_destroy(td->aio_ctx);
1155
1156         if (td->aio_events)
1157                 free(td->aio_events);
1158 }
1159
1160 static int init_aio(struct thread_data *td)
1161 {
1162         if (io_queue_init(td->aio_depth, &td->aio_ctx)) {
1163                 td->error = errno;
1164                 return 1;
1165         }
1166
1167         td->aio_events = malloc(td->aio_depth * sizeof(struct io_event));
1168         return 0;
1169 }
1170
1171 static void cleanup_io_u(struct thread_data *td)
1172 {
1173         struct list_head *entry, *n;
1174         struct io_u *io_u;
1175
1176         list_for_each_safe(entry, n, &td->io_u_freelist) {
1177                 io_u = list_entry(entry, struct io_u, list);
1178
1179                 list_del(&io_u->list);
1180                 free(io_u);
1181         }
1182
1183         if (td->mem_type == MEM_MALLOC)
1184                 free(td->orig_buffer);
1185         else if (td->mem_type == MEM_SHM) {
1186                 struct shmid_ds sbuf;
1187
1188                 shmdt(td->orig_buffer);
1189                 shmctl(td->shm_id, IPC_RMID, &sbuf);
1190         }
1191 }
1192
1193 static int init_io_u(struct thread_data *td)
1194 {
1195         struct io_u *io_u;
1196         int i, max_units, mem_size;
1197         char *p;
1198
1199         if (!td->use_aio)
1200                 max_units = 1;
1201         else
1202                 max_units = td->aio_depth;
1203
1204         mem_size = td->max_bs * max_units + MASK;
1205
1206         if (td->mem_type == MEM_MALLOC)
1207                 td->orig_buffer = malloc(mem_size);
1208         else if (td->mem_type == MEM_SHM) {
1209                 td->shm_id = shmget(IPC_PRIVATE, mem_size, IPC_CREAT | 0600);
1210                 if (td->shm_id < 0) {
1211                         td->error = errno;
1212                         perror("shmget");
1213                         return 1;
1214                 }
1215
1216                 td->orig_buffer = shmat(td->shm_id, NULL, 0);
1217                 if (td->orig_buffer == (void *) -1) {
1218                         td->error = errno;
1219                         perror("shmat");
1220                         return 1;
1221                 }
1222         }
1223
1224         INIT_LIST_HEAD(&td->io_u_freelist);
1225         INIT_LIST_HEAD(&td->io_u_busylist);
1226
1227         p = ALIGN(td->orig_buffer);
1228         for (i = 0; i < max_units; i++) {
1229                 io_u = malloc(sizeof(*io_u));
1230                 memset(io_u, 0, sizeof(*io_u));
1231                 INIT_LIST_HEAD(&io_u->list);
1232
1233                 io_u->buf = p + td->max_bs * i;
1234                 list_add(&io_u->list, &td->io_u_freelist);
1235         }
1236
1237         return 0;
1238 }
1239
1240 static void setup_log(struct io_log **log)
1241 {
1242         struct io_log *l = malloc(sizeof(*l));
1243
1244         l->nr_samples = 0;
1245         l->max_samples = 1024;
1246         l->log = malloc(l->max_samples * sizeof(struct io_sample));
1247         *log = l;
1248 }
1249
1250 static void finish_log(struct thread_data *td, struct io_log *log, char *name)
1251 {
1252         char file_name[128];
1253         FILE *f;
1254         int i;
1255
1256         sprintf(file_name, "client%d_%s.log", td->thread_number, name);
1257         f = fopen(file_name, "w");
1258         if (!f) {
1259                 perror("fopen log");
1260                 return;
1261         }
1262
1263         for (i = 0; i < log->nr_samples; i++)
1264                 fprintf(f, "%lu, %lu\n", log->log[i].time, log->log[i].val);
1265
1266         fclose(f);
1267         free(log->log);
1268         free(log);
1269 }
1270
1271 static int create_file(struct thread_data *td)
1272 {
1273         unsigned long long left;
1274         char *b;
1275         int r, bs;
1276
1277         /*
1278          * unless specifically asked for overwrite, let normal io extend it
1279          */
1280         if (!td_read(td) && !td->overwrite)
1281                 return 0;
1282
1283         if (!td->file_size) {
1284                 fprintf(stderr, "Need size for create\n");
1285                 td->error = EINVAL;
1286                 return 1;
1287         }
1288
1289         printf("Client%d: Laying out IO file\n", td->thread_number);
1290
1291         td->fd = open(td->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
1292         if (td->fd < 0) {
1293                 td->error = errno;
1294                 return 1;
1295         }
1296
1297         td->kb = td->file_size >> 10;
1298         b = malloc(td->max_bs);
1299         memset(b, 0, td->max_bs);
1300
1301         left = td->file_size;
1302         while (left) {
1303                 bs = td->max_bs;
1304                 if (bs > left)
1305                         bs = left;
1306
1307                 r = write(td->fd, b, bs);
1308
1309                 if (r == bs) {
1310                         left -= bs;
1311                         continue;
1312                 } else {
1313                         if (r < 0)
1314                                 td->error = errno;
1315                         else
1316                                 td->error = EIO;
1317
1318                         break;
1319                 }
1320         }
1321
1322         if (td->create_fsync)
1323                 fsync(td->fd);
1324
1325         close(td->fd);
1326         td->fd = -1;
1327         free(b);
1328         return 0;
1329 }
1330
1331 static int file_exists(struct thread_data *td)
1332 {
1333         struct stat st;
1334
1335         if (stat(td->file_name, &st) != -1)
1336                 return 1;
1337
1338         return errno != ENOENT;
1339 }
1340
1341 static int setup_file(struct thread_data *td)
1342 {
1343         struct stat st;
1344         int flags = 0;
1345
1346         if (!file_exists(td)) {
1347                 if (!td->create_file) {
1348                         td->error = ENOENT;
1349                         return 1;
1350                 }
1351                 if (create_file(td))
1352                         return 1;
1353         }
1354
1355         if (td->odirect)
1356                 flags |= O_DIRECT;
1357
1358         if (td_read(td))
1359                 td->fd = open(td->file_name, flags | O_RDONLY);
1360         else {
1361                 if (!td->overwrite)
1362                         flags |= O_TRUNC;
1363                 if (td->sync_io)
1364                         flags |= O_SYNC;
1365                 if (td->verify)
1366                         flags |= O_RDWR;
1367                 else
1368                         flags |= O_WRONLY;
1369
1370                 td->fd = open(td->file_name, flags | O_CREAT, 0600);
1371         }
1372
1373         if (td->fd == -1) {
1374                 td->error = errno;
1375                 return 1;
1376         }
1377
1378         if (td_read(td)) {
1379                 if (fstat(td->fd, &st) == -1) {
1380                         td->error = errno;
1381                         return 1;
1382                 }
1383
1384                 if (td->file_size > st.st_size)
1385                         st.st_size = td->file_size;
1386         } else {
1387                 if (!td->file_size)
1388                         td->file_size = 1024 * 1024 * 1024;
1389
1390                 st.st_size = td->file_size;
1391         }
1392
1393         td->kb = (st.st_size - td->file_offset) / 1024;
1394         if (!td->kb) {
1395                 fprintf(stderr, "Client%d: no io blocks\n", td->thread_number);
1396                 td->error = EINVAL;
1397                 return 1;
1398         }
1399
1400         if (td->invalidate_cache) {
1401                 if (fadvise(td->fd, 0, st.st_size, POSIX_FADV_DONTNEED) < 0) {
1402                         td->error = errno;
1403                         return 1;
1404                 }
1405         }
1406
1407         return 0;
1408 }
1409
1410 static void *thread_main(int shm_id, int offset, char *argv[])
1411 {
1412         struct thread_data *td;
1413         int ret = 1;
1414         void *data;
1415
1416         setsid();
1417
1418         data = shmat(shm_id, NULL, 0);
1419         if (data == (void *) -1) {
1420                 perror("shmat");
1421                 return NULL;
1422         }
1423
1424         td = data + offset * sizeof(struct thread_data);
1425         td->pid = getpid();
1426
1427         if (init_io_u(td))
1428                 goto err;
1429
1430         if (sched_setaffinity(td->pid, sizeof(td->cpumask), &td->cpumask) == -1) {
1431                 td->error = errno;
1432                 goto err;
1433         }
1434
1435         sprintf(argv[0], "fio%d", offset);
1436
1437         if (td->use_aio && init_aio(td))
1438                 goto err;
1439
1440         if (init_random_state(td))
1441                 goto err;
1442
1443         if (td->ioprio) {
1444                 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
1445                         td->error = errno;
1446                         goto err;
1447                 }
1448         }
1449
1450         sem_post(&startup_sem);
1451         sem_wait(&td->mutex);
1452
1453         if (!td->create_serialize && setup_file(td))
1454                 goto err;
1455
1456         gettimeofday(&td->start, NULL);
1457
1458         if (td->ratemin)
1459                 memcpy(&td->lastrate, &td->start, sizeof(td->start));
1460
1461         memcpy(&td->stat_sample_time, &td->start, sizeof(td->start));
1462
1463         while (td->loops--) {
1464                 if (!td->use_aio) {
1465                         do_sync_io(td);
1466
1467                         if (td->verify) {
1468                                 if (!do_sync_verify(td))
1469                                         break;
1470                         }
1471                 } else {
1472                         do_async_io(td);
1473
1474                         if (td->verify) {
1475                                 if (!do_async_verify(td))
1476                                         break;
1477                         }
1478                 }
1479         }
1480
1481         td->runtime = mtime_since_now(&td->start);
1482         ret = 0;
1483
1484         if (td->bw_log)
1485                 finish_log(td, td->bw_log, "bw");
1486         if (td->lat_log)
1487                 finish_log(td, td->lat_log, "lat");
1488
1489         if (exitall_on_terminate)
1490                 sig_handler(0);
1491
1492 err:
1493         if (td->fd != -1) {
1494                 close(td->fd);
1495                 td->fd = -1;
1496         }
1497         if (td->use_aio)
1498                 cleanup_aio(td);
1499         cleanup_io_u(td);
1500         if (ret) {
1501                 sem_post(&startup_sem);
1502                 sem_wait(&td->mutex);
1503         }
1504         td_set_runstate(td, TD_EXITED);
1505         shmdt(data);
1506         return NULL;
1507 }
1508
1509 static void free_shm(void)
1510 {
1511         struct shmid_ds sbuf;
1512
1513         if (threads) {
1514                 shmdt(threads);
1515                 threads = NULL;
1516                 shmctl(shm_id, IPC_RMID, &sbuf);
1517         }
1518 }
1519
1520 static int calc_lat(struct io_stat *is, unsigned long *min, unsigned long *max,
1521                     double *mean, double *dev)
1522 {
1523         double n;
1524
1525         if (is->samples == 0)
1526                 return 0;
1527
1528         *min = is->min_val;
1529         *max = is->max_val;
1530
1531         n = (double) is->samples;
1532         *mean = (double) is->val / n;
1533         *dev = sqrt(((double) is->val_sq - (*mean * *mean) / n) / (n - 1));
1534         return 1;
1535 }
1536
1537 static void show_thread_status(struct thread_data *td)
1538 {
1539         int prio, prio_class;
1540         unsigned long min, max, bw = 0;
1541         double mean, dev;
1542
1543         if (!td->io_kb && !td->error)
1544                 return;
1545
1546         if (td->runtime)
1547                 bw = td->io_kb * 1024 / td->runtime;
1548
1549         prio = td->ioprio & 0xff;
1550         prio_class = td->ioprio >> IOPRIO_CLASS_SHIFT;
1551
1552         printf("Client%d: err=%2d, io=%6luMiB, bw=%6luKiB/s, runt=%6lumsec\n", td->thread_number, td->error, td->io_kb >> 10, bw, td->runtime);
1553
1554         if (calc_lat(&td->slat_stat, &min, &max, &mean, &dev))
1555                 printf("  slat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1556         if (calc_lat(&td->clat_stat, &min, &max, &mean, &dev))
1557                 printf("  clat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1558         if (calc_lat(&td->bw_stat, &min, &max, &mean, &dev))
1559                 printf("  bw (KiB/s) : min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1560 }
1561
1562 static int setup_rate(struct thread_data *td)
1563 {
1564         int nr_reads_per_sec;
1565
1566         if (!td->rate)
1567                 return 0;
1568
1569         if (td->rate < td->ratemin) {
1570                 fprintf(stderr, "min rate larger than nominal rate\n");
1571                 return -1;
1572         }
1573
1574         nr_reads_per_sec = td->rate * 1024 / td->min_bs;
1575         td->rate_usec_cycle = 1000000 / nr_reads_per_sec;
1576         td->rate_pending_usleep = 0;
1577         return 0;
1578 }
1579
1580 static struct thread_data *get_new_job(int global)
1581 {
1582         struct thread_data *td;
1583
1584         if (global)
1585                 return &def_thread;
1586         if (thread_number >= max_jobs)
1587                 return NULL;
1588
1589         td = &threads[thread_number++];
1590         memset(td, 0, sizeof(*td));
1591
1592         td->fd = -1;
1593         td->thread_number = thread_number;
1594
1595         td->ddir = def_thread.ddir;
1596         td->ioprio = def_thread.ioprio;
1597         td->sequential = def_thread.sequential;
1598         td->bs = def_thread.bs;
1599         td->min_bs = def_thread.min_bs;
1600         td->max_bs = def_thread.max_bs;
1601         td->odirect = def_thread.odirect;
1602         td->thinktime = def_thread.thinktime;
1603         td->fsync_blocks = def_thread.fsync_blocks;
1604         td->start_delay = def_thread.start_delay;
1605         td->timeout = def_thread.timeout;
1606         td->use_aio = def_thread.use_aio;
1607         td->create_file = def_thread.create_file;
1608         td->overwrite = def_thread.overwrite;
1609         td->invalidate_cache = def_thread.invalidate_cache;
1610         td->file_size = def_thread.file_size;
1611         td->file_offset = def_thread.file_offset;
1612         td->rate = def_thread.rate;
1613         td->ratemin = def_thread.ratemin;
1614         td->ratecycle = def_thread.ratecycle;
1615         td->aio_depth = def_thread.aio_depth;
1616         td->sync_io = def_thread.sync_io;
1617         td->mem_type = def_thread.mem_type;
1618         td->bw_avg_time = def_thread.bw_avg_time;
1619         td->create_serialize = def_thread.create_serialize;
1620         td->create_fsync = def_thread.create_fsync;
1621         td->loops = def_thread.loops;
1622         td->verify = def_thread.verify;
1623         memcpy(&td->cpumask, &def_thread.cpumask, sizeof(td->cpumask));
1624
1625         return td;
1626 }
1627
1628 static void put_job(struct thread_data *td)
1629 {
1630         memset(&threads[td->thread_number - 1], 0, sizeof(*td));
1631         thread_number--;
1632 }
1633
1634 static int add_job(struct thread_data *td, const char *filename, int prioclass,
1635                    int prio)
1636 {
1637         if (td == &def_thread)
1638                 return 0;
1639
1640         strcpy(td->file_name, filename);
1641         sem_init(&td->mutex, 1, 0);
1642         td->ioprio = (prioclass << IOPRIO_CLASS_SHIFT) | prio;
1643
1644         td->clat_stat.min_val = ULONG_MAX;
1645         td->slat_stat.min_val = ULONG_MAX;
1646         td->bw_stat.min_val = ULONG_MAX;
1647
1648         run_str[td->thread_number - 1] = 'P';
1649
1650         if (td->use_aio && !td->aio_depth)
1651                 td->aio_depth = 1;
1652
1653         if (td->min_bs == -1)
1654                 td->min_bs = td->bs;
1655         if (td->max_bs == -1)
1656                 td->max_bs = td->bs;
1657
1658         if (setup_rate(td))
1659                 return -1;
1660
1661         if (write_lat_log)
1662                 setup_log(&td->lat_log);
1663         if (write_bw_log)
1664                 setup_log(&td->bw_log);
1665
1666         printf("Client%d: file=%s, rw=%d, prio=%d/%d, seq=%d, odir=%d, bs=%d-%d, rate=%d, aio=%d, aio_depth=%d\n", td->thread_number, filename, td->ddir, prioclass, prio, td->sequential, td->odirect, td->min_bs, td->max_bs, td->rate, td->use_aio, td->aio_depth);
1667         return 0;
1668 }
1669
1670 static void fill_cpu_mask(cpu_set_t cpumask, int cpu)
1671 {
1672         unsigned int i;
1673
1674         CPU_ZERO(&cpumask);
1675
1676         for (i = 0; i < sizeof(int) * 8; i++) {
1677                 if ((1 << i) & cpu)
1678                         CPU_SET(i, &cpumask);
1679         }
1680 }
1681
1682 unsigned long get_mult(char c)
1683 {
1684         switch (c) {
1685                 case 'k':
1686                 case 'K':
1687                         return 1024;
1688                 case 'm':
1689                 case 'M':
1690                         return 1024 * 1024;
1691                 case 'g':
1692                 case 'G':
1693                         return 1024 * 1024 * 1024;
1694                 default:
1695                         return 1;
1696         }
1697 }
1698
1699 /*
1700  * convert string after '=' into decimal value, noting any size suffix
1701  */
1702 static int str_cnv(char *p, unsigned long long *val)
1703 {
1704         char *str;
1705         int len;
1706
1707         str = strstr(p, "=");
1708         if (!str)
1709                 return 1;
1710
1711         str++;
1712         len = strlen(str);
1713
1714         *val = strtoul(str, NULL, 10);
1715         if (*val == ULONG_MAX && errno == ERANGE)
1716                 return 1;
1717
1718         *val *= get_mult(str[len - 2]);
1719         return 0;
1720 }
1721
1722 static int check_strcnv(char *p, char *name, unsigned long long *val)
1723 {
1724         if (!strstr(p, name))
1725                 return 1;
1726
1727         return str_cnv(p, val);
1728 }
1729
1730 static int check_str(char *p, char *name, char *option)
1731 {
1732         char *s = strstr(p, name);
1733
1734         if (!s)
1735                 return 1;
1736
1737         s += strlen(name);
1738         if (strstr(s, option))
1739                 return 0;
1740
1741         return 1;
1742 }
1743
1744 static int check_range(char *p, char *name, unsigned long *s, unsigned long *e)
1745 {
1746         char str[128];
1747         char s1, s2;
1748
1749         sprintf(str, "%s=%%lu%%c-%%lu%%c", name);
1750         if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1751                 *s *= get_mult(s1);
1752                 *e *= get_mult(s2);
1753                 return 0;
1754         }
1755
1756         sprintf(str, "%s = %%lu%%c-%%lu%%c", name);
1757         if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1758                 *s *= get_mult(s1);
1759                 *e *= get_mult(s2);
1760                 return 0;
1761         }
1762
1763         sprintf(str, "%s=%%lu-%%lu", name);
1764         if (sscanf(p, str, s, e) == 2)
1765                 return 0;
1766
1767         sprintf(str, "%s = %%lu-%%lu", name);
1768         if (sscanf(p, str, s, e) == 2)
1769                 return 0;
1770
1771         return 1;
1772
1773 }
1774
1775 static int check_int(char *p, char *name, unsigned int *val)
1776 {
1777         char str[128];
1778
1779         sprintf(str, "%s=%%d", name);
1780         if (sscanf(p, str, val) == 1)
1781                 return 0;
1782
1783         sprintf(str, "%s = %%d", name);
1784         if (sscanf(p, str, val) == 1)
1785                 return 0;
1786
1787         return 1;
1788 }
1789
1790 static int is_empty_or_comment(char *line)
1791 {
1792         unsigned int i;
1793
1794         for (i = 0; i < strlen(line); i++) {
1795                 if (line[i] == ';')
1796                         return 1;
1797                 if (!isspace(line[i]) && !iscntrl(line[i]))
1798                         return 0;
1799         }
1800
1801         return 1;
1802 }
1803
1804 static int parse_jobs_ini(char *file)
1805 {
1806         unsigned int prioclass, prio, cpu, global;
1807         unsigned long long ull;
1808         unsigned long ul1, ul2;
1809         struct thread_data *td;
1810         char *string, *name;
1811         fpos_t off;
1812         FILE *f;
1813         char *p;
1814
1815         f = fopen(file, "r");
1816         if (!f) {
1817                 perror("fopen");
1818                 return 1;
1819         }
1820
1821         string = malloc(4096);
1822         name = malloc(256);
1823
1824         while ((p = fgets(string, 4096, f)) != NULL) {
1825                 if (is_empty_or_comment(p))
1826                         continue;
1827                 if (sscanf(p, "[%s]", name) != 1)
1828                         continue;
1829
1830                 global = !strncmp(name, "global", 6);
1831
1832                 name[strlen(name) - 1] = '\0';
1833
1834                 td = get_new_job(global);
1835                 if (!td)
1836                         break;
1837
1838                 prioclass = 2;
1839                 prio = 4;
1840
1841                 fgetpos(f, &off);
1842                 while ((p = fgets(string, 4096, f)) != NULL) {
1843                         if (is_empty_or_comment(p))
1844                                 continue;
1845                         if (strstr(p, "["))
1846                                 break;
1847                         if (!check_int(p, "rw", &td->ddir)) {
1848                                 fgetpos(f, &off);
1849                                 continue;
1850                         }
1851                         if (!check_int(p, "prio", &prio)) {
1852                                 fgetpos(f, &off);
1853                                 continue;
1854                         }
1855                         if (!check_int(p, "prioclass", &prioclass)) {
1856                                 fgetpos(f, &off);
1857                                 continue;
1858                         }
1859                         if (!check_int(p, "direct", &td->odirect)) {
1860                                 fgetpos(f, &off);
1861                                 continue;
1862                         }
1863                         if (!check_int(p, "rate", &td->rate)) {
1864                                 fgetpos(f, &off);
1865                                 continue;
1866                         }
1867                         if (!check_int(p, "ratemin", &td->ratemin)) {
1868                                 fgetpos(f, &off);
1869                                 continue;
1870                         }
1871                         if (!check_int(p, "ratecycle", &td->ratecycle)) {
1872                                 fgetpos(f, &off);
1873                                 continue;
1874                         }
1875                         if (!check_int(p, "thinktime", &td->thinktime)) {
1876                                 fgetpos(f, &off);
1877                                 continue;
1878                         }
1879                         if (!check_int(p, "cpumask", &cpu)) {
1880                                 fill_cpu_mask(td->cpumask, cpu);
1881                                 fgetpos(f, &off);
1882                                 continue;
1883                         }
1884                         if (!check_int(p, "fsync", &td->fsync_blocks)) {
1885                                 fgetpos(f, &off);
1886                                 continue;
1887                         }
1888                         if (!check_int(p, "startdelay", &td->start_delay)) {
1889                                 fgetpos(f, &off);
1890                                 continue;
1891                         }
1892                         if (!check_int(p, "timeout", &td->timeout)) {
1893                                 fgetpos(f, &off);
1894                                 continue;
1895                         }
1896                         if (!check_int(p, "invalidate",&td->invalidate_cache)) {
1897                                 fgetpos(f, &off);
1898                                 continue;
1899                         }
1900                         if (!check_int(p, "aio_depth", &td->aio_depth)) {
1901                                 fgetpos(f, &off);
1902                                 continue;
1903                         }
1904                         if (!check_int(p, "sync", &td->sync_io)) {
1905                                 fgetpos(f, &off);
1906                                 continue;
1907                         }
1908                         if (!check_int(p, "bwavgtime", &td->bw_avg_time)) {
1909                                 fgetpos(f, &off);
1910                                 continue;
1911                         }
1912                         if (!check_int(p, "create_serialize", &td->create_serialize)) {
1913                                 fgetpos(f, &off);
1914                                 continue;
1915                         }
1916                         if (!check_int(p, "create_fsync", &td->create_fsync)) {
1917                                 fgetpos(f, &off);
1918                                 continue;
1919                         }
1920                         if (!check_int(p, "loops", &td->loops)) {
1921                                 fgetpos(f, &off);
1922                                 continue;
1923                         }
1924                         if (!check_int(p, "verify", &td->verify)) {
1925                                 fgetpos(f, &off);
1926                                 continue;
1927                         }
1928                         if (!check_range(p, "bsrange", &ul1, &ul2)) {
1929                                 td->min_bs = ul1;
1930                                 td->max_bs = ul2;
1931                                 fgetpos(f, &off);
1932                                 continue;
1933                         }
1934                         if (!check_strcnv(p, "bs", &ull)) {
1935                                 td->bs = ull;
1936                                 fgetpos(f, &off);
1937                                 continue;
1938                         }
1939                         if (!check_strcnv(p, "size", &td->file_size)) {
1940                                 fgetpos(f, &off);
1941                                 continue;
1942                         }
1943                         if (!check_strcnv(p, "offset", &td->file_offset)) {
1944                                 fgetpos(f, &off);
1945                                 continue;
1946                         }
1947                         if (!check_str(p, "mem", "malloc")) {
1948                                 td->mem_type = MEM_MALLOC;
1949                                 fgetpos(f, &off);
1950                                 continue;
1951                         }
1952                         if (!check_str(p, "mem", "shm")) {
1953                                 td->mem_type = MEM_SHM;
1954                                 fgetpos(f, &off);
1955                                 continue;
1956                         }
1957                         if (!strncmp(p, "sequential", 10)) {
1958                                 td->sequential = 1;
1959                                 fgetpos(f, &off);
1960                                 continue;
1961                         }
1962                         if (!strncmp(p, "random", 6)) {
1963                                 td->sequential = 0;
1964                                 fgetpos(f, &off);
1965                                 continue;
1966                         }
1967                         if (!strncmp(p, "aio", 3)) {
1968                                 td->use_aio = 1;
1969                                 fgetpos(f, &off);
1970                                 continue;
1971                         }
1972                         if (!strncmp(p, "create", 6)) {
1973                                 td->create_file = 1;
1974                                 fgetpos(f, &off);
1975                                 continue;
1976                         }
1977                         if (!strncmp(p, "overwrite", 9)) {
1978                                 td->overwrite = 1;
1979                                 fgetpos(f, &off);
1980                                 continue;
1981                         }
1982                         if (!strncmp(p, "exitall", 7)) {
1983                                 exitall_on_terminate = 1;
1984                                 fgetpos(f, &off);
1985                                 continue;
1986                         }
1987                         printf("Client%d: bad option %s\n",td->thread_number,p);
1988                 }
1989                 fsetpos(f, &off);
1990
1991                 if (add_job(td, name, prioclass, prio))
1992                         put_job(td);
1993         }
1994
1995         free(string);
1996         free(name);
1997         fclose(f);
1998         return 0;
1999 }
2000
2001 static int parse_options(int argc, char *argv[])
2002 {
2003         int i;
2004
2005         for (i = 1; i < argc; i++) {
2006                 char *parm = argv[i];
2007
2008                 if (parm[0] != '-')
2009                         break;
2010
2011                 parm++;
2012                 switch (*parm) {
2013                         case 's':
2014                                 parm++;
2015                                 def_thread.sequential = !!atoi(parm);
2016                                 break;
2017                         case 'b':
2018                                 parm++;
2019                                 def_thread.bs = atoi(parm);
2020                                 def_thread.bs <<= 10;
2021                                 if (!def_thread.bs) {
2022                                         printf("bad block size\n");
2023                                         def_thread.bs = DEF_BS;
2024                                 }
2025                                 break;
2026                         case 't':
2027                                 parm++;
2028                                 def_thread.timeout = atoi(parm);
2029                                 break;
2030                         case 'r':
2031                                 parm++;
2032                                 repeatable = !!atoi(parm);
2033                                 break;
2034                         case 'R':
2035                                 parm++;
2036                                 rate_quit = !!atoi(parm);
2037                                 break;
2038                         case 'o':
2039                                 parm++;
2040                                 def_thread.odirect = !!atoi(parm);
2041                                 break;
2042                         case 'f':
2043                                 if (i + 1 >= argc) {
2044                                         printf("-f needs file as arg\n");
2045                                         break;
2046                                 }
2047                                 ini_file = strdup(argv[i+1]);
2048                                 i++;
2049                                 break;
2050                         case 'l':
2051                                 write_lat_log = 1;
2052                                 break;
2053                         case 'w':
2054                                 write_bw_log = 1;
2055                                 break;
2056                         default:
2057                                 printf("bad option %s\n", argv[i]);
2058                                 break;
2059                 }
2060         }
2061
2062         return i;
2063 }
2064
2065 static void print_thread_status(struct thread_data *td, int nr_running,
2066                                 int t_rate, int m_rate)
2067 {
2068         printf("Threads now running: %d", nr_running);
2069         if (m_rate || t_rate)
2070                 printf(", commitrate %d/%dKiB/sec", t_rate, m_rate);
2071         printf(" : [%s]\r", run_str);
2072         fflush(stdout);
2073 }
2074
2075 static void check_str_update(struct thread_data *td, int n, int t, int m)
2076 {
2077         char c = run_str[td->thread_number - 1];
2078
2079         if (td->runstate == td->old_runstate)
2080                 return;
2081
2082         switch (td->runstate) {
2083                 case TD_REAPED:
2084                         c = '_';
2085                         break;
2086                 case TD_EXITED:
2087                         c = 'E';
2088                         break;
2089                 case TD_RUNNING:
2090                         c = '+';
2091                         break;
2092                 case TD_VERIFYING:
2093                         c = 'V';
2094                         break;
2095                 case TD_CREATED:
2096                         c = 'C';
2097                         break;
2098                 case TD_NOT_CREATED:
2099                         c = 'P';
2100                         break;
2101                 default:
2102                         printf("state %d\n", td->runstate);
2103         }
2104
2105         run_str[td->thread_number - 1] = c;
2106         print_thread_status(td, n, t, m);
2107         td->old_runstate = td->runstate;
2108 }
2109
2110 static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
2111 {
2112         int i;
2113
2114         /*
2115          * reap exited threads (TD_EXITED -> TD_REAPED)
2116          */
2117         for (i = 0; i < thread_number; i++) {
2118                 struct thread_data *td = &threads[i];
2119
2120                 check_str_update(td, *nr_running, *t_rate, *m_rate);
2121
2122                 if (td->runstate != TD_EXITED)
2123                         continue;
2124
2125                 td_set_runstate(td, TD_REAPED);
2126                 waitpid(td->pid, NULL, 0);
2127                 (*nr_running)--;
2128                 (*m_rate) -= td->ratemin;
2129                 (*t_rate) -= td->rate;
2130                 check_str_update(td, *nr_running, *t_rate, *m_rate);
2131
2132                 if (td->terminate)
2133                         continue;
2134         }
2135 }
2136
2137 static void run_threads(char *argv[])
2138 {
2139         struct timeval genesis;
2140         struct thread_data *td;
2141         unsigned long spent;
2142         int i, todo, nr_running, m_rate, t_rate;
2143
2144         printf("Starting %d threads\n", thread_number);
2145         fflush(stdout);
2146
2147         signal(SIGINT, sig_handler);
2148
2149         todo = thread_number;
2150         nr_running = 0;
2151         m_rate = t_rate = 0;
2152
2153         for (i = 0; i < thread_number; i++) {
2154                 td = &threads[i];
2155
2156                 if (!td->create_serialize)
2157                         continue;
2158
2159                 /*
2160                  * do file setup here so it happens sequentially,
2161                  * we don't want X number of threads getting their
2162                  * client data interspersed on disk
2163                  */
2164                 if (setup_file(td)) {
2165                         td_set_runstate(td, TD_REAPED);
2166                         todo--;
2167                 }
2168         }
2169
2170         gettimeofday(&genesis, NULL);
2171
2172         while (todo) {
2173                 /*
2174                  * create threads (TD_NOT_CREATED -> TD_CREATED)
2175                  */
2176                 for (i = 0; i < thread_number; i++) {
2177                         td = &threads[i];
2178
2179                         if (td->runstate != TD_NOT_CREATED)
2180                                 continue;
2181
2182                         /*
2183                          * never got a chance to start, killed by other
2184                          * thread for some reason
2185                          */
2186                         if (td->terminate) {
2187                                 todo--;
2188                                 continue;
2189                         }
2190
2191                         if (td->start_delay) {
2192                                 spent = mtime_since_now(&genesis);
2193
2194                                 if (td->start_delay * 1000 > spent)
2195                                         continue;
2196                         }
2197
2198                         td_set_runstate(td, TD_CREATED);
2199                         check_str_update(td, nr_running, t_rate, m_rate);
2200                         sem_init(&startup_sem, 1, 1);
2201                         todo--;
2202
2203                         if (fork())
2204                                 sem_wait(&startup_sem);
2205                         else {
2206                                 thread_main(shm_id, i, argv);
2207                                 exit(0);
2208                         }
2209                 }
2210
2211                 /*
2212                  * start created threads (TD_CREATED -> TD_RUNNING)
2213                  */
2214                 for (i = 0; i < thread_number; i++) {
2215                         struct thread_data *td = &threads[i];
2216
2217                         if (td->runstate != TD_CREATED)
2218                                 continue;
2219
2220                         td_set_runstate(td, TD_RUNNING);
2221                         nr_running++;
2222                         m_rate += td->ratemin;
2223                         t_rate += td->rate;
2224                         check_str_update(td, nr_running, t_rate, m_rate);
2225                         sem_post(&td->mutex);
2226                 }
2227
2228                 for (i = 0; i < thread_number; i++) {
2229                         struct thread_data *td = &threads[i];
2230
2231                         if (td->runstate == TD_RUNNING)
2232                                 run_str[td->thread_number - 1] = '+';
2233                         else if (td->runstate == TD_VERIFYING)
2234                                 run_str[td->thread_number - 1] = 'V';
2235                         else
2236                                 continue;
2237
2238                         check_str_update(td, nr_running, t_rate, m_rate);
2239                 }
2240
2241                 reap_threads(&nr_running, &t_rate, &m_rate);
2242
2243                 if (todo)
2244                         usleep(100000);
2245         }
2246
2247         while (nr_running) {
2248                 reap_threads(&nr_running, &t_rate, &m_rate);
2249                 usleep(10000);
2250         }
2251 }
2252
2253 int setup_thread_area(void)
2254 {
2255         /*
2256          * 1024 is too much on some machines, scale max_jobs if
2257          * we get a failure that looks like too large a shm segment
2258          */
2259         do {
2260                 int s = max_jobs * sizeof(struct thread_data);
2261
2262                 shm_id = shmget(0, s, IPC_CREAT | 0600);
2263                 if (shm_id != -1)
2264                         break;
2265                 if (errno != EINVAL) {
2266                         perror("shmget");
2267                         break;
2268                 }
2269
2270                 max_jobs >>= 1;
2271         } while (max_jobs);
2272
2273         if (shm_id == -1)
2274                 return 1;
2275
2276         threads = shmat(shm_id, NULL, 0);
2277         if (threads == (void *) -1) {
2278                 perror("shmat");
2279                 return 1;
2280         }
2281
2282         atexit(free_shm);
2283         return 0;
2284 }
2285
2286 int main(int argc, char *argv[])
2287 {
2288         static unsigned long max_run[2], min_run[2];
2289         static unsigned long max_bw[2], min_bw[2];
2290         static unsigned long io_mb[2], agg[2];
2291         int i;
2292
2293         if (setup_thread_area())
2294                 return 1;
2295
2296         if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &def_thread.cpumask) == -1) {
2297                 perror("sched_getaffinity");
2298                 return 1;
2299         }
2300
2301         /*
2302          * fill globals
2303          */
2304         def_thread.ddir = DDIR_READ;
2305         def_thread.bs = DEF_BS;
2306         def_thread.min_bs = -1;
2307         def_thread.max_bs = -1;
2308         def_thread.odirect = DEF_ODIRECT;
2309         def_thread.ratecycle = DEF_RATE_CYCLE;
2310         def_thread.sequential = DEF_SEQUENTIAL;
2311         def_thread.timeout = DEF_TIMEOUT;
2312         def_thread.create_file = DEF_CREATE;
2313         def_thread.overwrite = DEF_OVERWRITE;
2314         def_thread.invalidate_cache = DEF_INVALIDATE;
2315         def_thread.sync_io = DEF_SYNCIO;
2316         def_thread.mem_type = MEM_MALLOC;
2317         def_thread.bw_avg_time = DEF_BWAVGTIME;
2318         def_thread.create_serialize = DEF_CREATE_SER;
2319         def_thread.create_fsync = DEF_CREATE_FSYNC;
2320         def_thread.loops = DEF_LOOPS;
2321         def_thread.verify = DEF_VERIFY;
2322
2323         i = parse_options(argc, argv);
2324
2325         if (!ini_file) {
2326                 printf("Need job file\n");
2327                 return 1;
2328         }
2329
2330         if (parse_jobs_ini(ini_file))
2331                 return 1;
2332
2333         if (!thread_number) {
2334                 printf("Nothing to do\n");
2335                 return 1;
2336         }
2337
2338         run_threads(argv);
2339
2340         min_bw[0] = min_run[0] = ~0UL;
2341         min_bw[1] = min_run[1] = ~0UL;
2342         io_mb[0] = io_mb[1] = 0;
2343         agg[0] = agg[1] = 0;
2344         for (i = 0; i < thread_number; i++) {
2345                 struct thread_data *td = &threads[i];
2346                 unsigned long bw = 0;
2347
2348                 if (!td->error) {
2349                         if (td->runtime < min_run[td->ddir])
2350                                 min_run[td->ddir] = td->runtime;
2351                         if (td->runtime > max_run[td->ddir])
2352                                 max_run[td->ddir] = td->runtime;
2353
2354                         if (td->runtime)
2355                                 bw = td->io_kb * 1024 / td->runtime;
2356                         if (bw < min_bw[td->ddir])
2357                                 min_bw[td->ddir] = bw;
2358                         if (bw > max_bw[td->ddir])
2359                                 max_bw[td->ddir] = bw;
2360
2361                         io_mb[td->ddir] += td->io_kb >> 10;
2362                 }
2363
2364                 show_thread_status(td);
2365         }
2366
2367         if (max_run[0])
2368                 agg[0] = io_mb[0] * 1024 * 1000 / max_run[0];
2369         if (max_run[1])
2370                 agg[1] = io_mb[1] * 1024 * 1000 / max_run[1];
2371
2372         printf("\nRun status:\n");
2373         if (max_run[DDIR_READ])
2374                 printf("   READ: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[0], agg[0], min_bw[0], max_bw[0], min_run[0], max_run[0]);
2375         if (max_run[DDIR_WRITE])
2376                 printf("  WRITE: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[1], agg[1], min_bw[1], max_bw[1], min_run[1], max_run[1]);
2377
2378         return 0;
2379 }