fio.c

   1 /*
   2  * fio - the flexible io tester
   3  *
   4  * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 2 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <fcntl.h>
  25 #include <string.h>
  26 #include <errno.h>
  27 #include <signal.h>
  28 #include <time.h>
  29 #include <ctype.h>
  30 #include <sched.h>
  31 #include <libaio.h>
  32 #include <math.h>
  33 #include <limits.h>
  34 #include <sys/time.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <sys/wait.h>
  38 #include <semaphore.h>
  39 #include <sys/ipc.h>
  40 #include <sys/shm.h>
  41 #include <asm/unistd.h>
  42 #include <asm/types.h>
  43
  44 #include "list.h"
  45 #include "md5.h"
  46
  47 #define MAX_JOBS        (1024)
  48
  49 /*
  50  * assume we don't have _get either, if _set isn't defined
  51  */
  52 #ifndef __NR_ioprio_set
  53 #if defined(__i386__)
  54 #define __NR_ioprio_set         289
  55 #define __NR_ioprio_get         290
  56 #elif defined(__powerpc__) || defined(__powerpc64__)
  57 #define __NR_ioprio_set         273
  58 #define __NR_ioprio_get         274
  59 #elif defined(__x86_64__)
  60 #define __NR_ioprio_set         251
  61 #define __NR_ioprio_get         252
  62 #elif defined(__ia64__)
  63 #define __NR_ioprio_set         1274
  64 #define __NR_ioprio_get         1275
  65 #elif defined(__alpha__)
  66 #define __NR_ioprio_set         442
  67 #define __NR_ioprio_get         443
  68 #elif defined(__s390x__) || defined(__s390__)
  69 #define __NR_ioprio_set         282
  70 #define __NR_ioprio_get         283
  71 #else
  72 #error "Unsupported arch"
  73 #endif
  74 #endif
  75
  76 #ifndef __NR_fadvise64
  77 #if defined(__i386__)
  78 #define __NR_fadvise64          250
  79 #elif defined(__powerpc__) || defined(__powerpc64__)
  80 #define __NR_fadvise64          233
  81 #elif defined(__x86_64__)
  82 #define __NR_fadvise64          221
  83 #elif defined(__ia64__)
  84 #define __NR_fadvise64          1234
  85 #elif defined(__alpha__)
  86 #define __NR_fadvise64          413
  87 #elif defined(__s390x__) || defined(__s390__)
  88 #define __NR_fadvise64          253
  89 #else
  90 #error "Unsupported arch"
  91 #endif
  92 #endif
  93
  94 static int ioprio_set(int which, int who, int ioprio)
  95 {
  96         return syscall(__NR_ioprio_set, which, who, ioprio);
  97 }
  98
  99 /*
 100  * we want fadvise64 really, but it's so tangled... later
 101  */
 102 static int fadvise(int fd, loff_t offset, size_t len, int advice)
 103 {
 104 #if 0
 105         return syscall(__NR_fadvise64, fd, offset, offset >> 32, len, advice);
 106 #else
 107         return posix_fadvise(fd, (off_t) offset, len, advice);
 108 #endif
 109 }
 110
 111 enum {
 112         IOPRIO_WHO_PROCESS = 1,
 113         IOPRIO_WHO_PGRP,
 114         IOPRIO_WHO_USER,
 115 };
 116
 117 #define IOPRIO_CLASS_SHIFT      13
 118
 119 #define MASK    (4095)
 120
 121 #define DEF_BS          (4096)
 122 #define DEF_TIMEOUT     (30)
 123 #define DEF_RATE_CYCLE  (1000)
 124 #define DEF_ODIRECT     (1)
 125 #define DEF_SEQUENTIAL  (1)
 126 #define DEF_RAND_REPEAT (1)
 127 #define DEF_OVERWRITE   (0)
 128 #define DEF_CREATE      (1)
 129 #define DEF_INVALIDATE  (1)
 130 #define DEF_SYNCIO      (0)
 131 #define DEF_RANDSEED    (0xb1899bedUL)
 132 #define DEF_BWAVGTIME   (500)
 133 #define DEF_CREATE_SER  (1)
 134 #define DEF_CREATE_FSYNC        (1)
 135 #define DEF_LOOPS       (1)
 136 #define DEF_VERIFY      (0)
 137
 138 #define ALIGN(buf)      (char *) (((unsigned long) (buf) + MASK) & ~(MASK))
 139
 140 static int repeatable = DEF_RAND_REPEAT;
 141 static int rate_quit = 1;
 142 static int write_lat_log;
 143 static int write_bw_log;
 144 static int exitall_on_terminate;
 145
 146 static int thread_number;
 147 static char *ini_file;
 148
 149 static int max_jobs = MAX_JOBS;
 150
 151 static char run_str[MAX_JOBS + 1];
 152
 153 static int shm_id;
 154
 155 enum {
 156         DDIR_READ = 0,
 157         DDIR_WRITE,
 158 };
 159
 160 /*
 161  * thread life cycle
 162  */
 163 enum {
 164         TD_NOT_CREATED = 0,
 165         TD_CREATED,
 166         TD_RUNNING,
 167         TD_VERIFYING,
 168         TD_EXITED,
 169         TD_REAPED,
 170 };
 171
 172 enum {
 173         MEM_MALLOC,
 174         MEM_SHM,
 175 };
 176
 177 /*
 178  * The io unit
 179  */
 180 struct io_u {
 181         struct iocb iocb;
 182         struct timeval start_time;
 183         struct timeval issue_time;
 184
 185         char *buf;
 186         unsigned int buflen;
 187         unsigned long long offset;
 188
 189         struct list_head list;
 190 };
 191
 192 struct io_stat {
 193         unsigned long val;
 194         unsigned long val_sq;
 195         unsigned long max_val;
 196         unsigned long min_val;
 197         unsigned long samples;
 198 };
 199
 200 struct io_sample {
 201         unsigned long time;
 202         unsigned long val;
 203 };
 204
 205 struct io_log {
 206         unsigned long nr_samples;
 207         unsigned long max_samples;
 208         struct io_sample *log;
 209 };
 210
 211 #define FIO_HDR_MAGIC   0xf00baaef
 212
 213 struct verify_header {
 214         unsigned int fio_magic;
 215         unsigned int len;
 216         char md5_digest[MD5_HASH_WORDS * 4];
 217 };
 218
 219 #define td_read(td)             ((td)->ddir == DDIR_READ)
 220 #define should_fsync(td)        (!td_read(td) && !(td)->odirect)
 221
 222 struct thread_data {
 223         char file_name[256];
 224         int thread_number;
 225         int error;
 226         int fd;
 227         pid_t pid;
 228         char *orig_buffer;
 229         volatile int terminate;
 230         volatile int runstate;
 231         volatile int old_runstate;
 232         unsigned int ddir;
 233         unsigned int ioprio;
 234         unsigned int sequential;
 235         unsigned int bs;
 236         unsigned int min_bs;
 237         unsigned int max_bs;
 238         unsigned int odirect;
 239         unsigned int thinktime;
 240         unsigned int fsync_blocks;
 241         unsigned int start_delay;
 242         unsigned int timeout;
 243         unsigned int use_aio;
 244         unsigned int create_file;
 245         unsigned int overwrite;
 246         unsigned int invalidate_cache;
 247         unsigned int bw_avg_time;
 248         unsigned int create_serialize;
 249         unsigned int create_fsync;
 250         unsigned int loops;
 251         unsigned long long file_size;
 252         unsigned long long file_offset;
 253         unsigned int sync_io;
 254         unsigned int mem_type;
 255         unsigned int verify;
 256         cpu_set_t cpumask;
 257
 258         struct drand48_data bsrange_state;
 259         struct drand48_data verify_state;
 260
 261         int shm_id;
 262
 263         off_t cur_off;
 264
 265         io_context_t aio_ctx;
 266         unsigned int aio_depth;
 267         struct io_event *aio_events;
 268
 269         unsigned int cur_depth;
 270         struct list_head io_u_freelist;
 271         struct list_head io_u_busylist;
 272
 273         unsigned int rate;
 274         unsigned int ratemin;
 275         unsigned int ratecycle;
 276         unsigned long rate_usec_cycle;
 277         long rate_pending_usleep;
 278         unsigned long rate_kb;
 279         struct timeval lastrate;
 280
 281         unsigned long runtime;          /* sec */
 282         unsigned long kb;
 283         unsigned long io_blocks;
 284         unsigned long io_kb;
 285         unsigned long this_io_kb;
 286         unsigned long last_kb;
 287         sem_t mutex;
 288         struct drand48_data random_state;
 289
 290         /*
 291          * bandwidth and latency stats
 292          */
 293         struct io_stat clat_stat;               /* completion latency */
 294         struct io_stat slat_stat;               /* submission latency */
 295
 296         struct io_stat bw_stat;                 /* bandwidth stats */
 297         unsigned long stat_io_kb;
 298         struct timeval stat_sample_time;
 299
 300         struct io_log *lat_log;
 301         struct io_log *bw_log;
 302
 303         struct timeval start;
 304 };
 305
 306 static struct thread_data *threads;
 307 static struct thread_data def_thread;
 308
 309 static sem_t startup_sem;
 310
 311 static void sig_handler(int sig)
 312 {
 313         int i;
 314
 315         for (i = 0; i < thread_number; i++) {
 316                 struct thread_data *td = &threads[i];
 317
 318                 td->terminate = 1;
 319                 td->start_delay = 0;
 320         }
 321 }
 322
 323 static int init_random_state(struct thread_data *td)
 324 {
 325         unsigned long seed;
 326         int fd;
 327
 328         fd = open("/dev/random", O_RDONLY);
 329         if (fd == -1) {
 330                 td->error = errno;
 331                 return 1;
 332         }
 333
 334         if (read(fd, &seed, sizeof(seed)) < (int) sizeof(seed)) {
 335                 td->error = EIO;
 336                 close(fd);
 337                 return 1;
 338         }
 339
 340         close(fd);
 341
 342         srand48_r(seed, &td->bsrange_state);
 343         srand48_r(seed, &td->verify_state);
 344
 345         if (td->sequential)
 346                 return 0;
 347
 348         if (repeatable)
 349                 seed = DEF_RANDSEED;
 350
 351         srand48_r(seed, &td->random_state);
 352         return 0;
 353 }
 354
 355 static unsigned long utime_since(struct timeval *s, struct timeval *e)
 356 {
 357         double sec, usec;
 358
 359         sec = e->tv_sec - s->tv_sec;
 360         usec = e->tv_usec - s->tv_usec;
 361         if (sec > 0 && usec < 0) {
 362                 sec--;
 363                 usec += 1000000;
 364         }
 365
 366         sec *= (double) 1000000;
 367
 368         return sec + usec;
 369 }
 370
 371 static unsigned long mtime_since(struct timeval *s, struct timeval *e)
 372 {
 373         double sec, usec;
 374
 375         sec = e->tv_sec - s->tv_sec;
 376         usec = e->tv_usec - s->tv_usec;
 377         if (sec > 0 && usec < 0) {
 378                 sec--;
 379                 usec += 1000000;
 380         }
 381
 382         sec *= (double) 1000;
 383         usec /= (double) 1000;
 384
 385         return sec + usec;
 386 }
 387
 388 static unsigned long mtime_since_now(struct timeval *s)
 389 {
 390         struct timeval t;
 391
 392         gettimeofday(&t, NULL);
 393         return mtime_since(s, &t);
 394 }
 395
 396 static inline unsigned long msec_now(struct timeval *s)
 397 {
 398         return s->tv_sec * 1000 + s->tv_usec / 1000;
 399 }
 400
 401 static unsigned long long get_next_offset(struct thread_data *td)
 402 {
 403         unsigned long long kb;
 404         long r;
 405
 406         if (!td->sequential) {
 407                 lrand48_r(&td->random_state, &r);
 408                 kb = (1+(double) (td->kb-1) * r / (RAND_MAX+1.0));
 409         } else
 410                 kb = td->last_kb;
 411
 412         return (kb << 10) + td->file_offset;
 413 }
 414
 415 static unsigned int get_next_buflen(struct thread_data *td)
 416 {
 417         unsigned int buflen;
 418         long r;
 419
 420         if (td->min_bs == td->max_bs)
 421                 buflen = td->min_bs;
 422         else {
 423                 lrand48_r(&td->bsrange_state, &r);
 424                 buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0));
 425                 buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1);
 426         }
 427
 428         if (buflen > ((td->kb - td->this_io_kb) << 10))
 429                 buflen = (td->kb - td->this_io_kb) << 10;
 430
 431         td->last_kb += buflen >> 10;
 432         return buflen;
 433 }
 434
 435 static inline void add_stat_sample(struct thread_data *td, struct io_stat *is,
 436                                    unsigned long val)
 437 {
 438         if (val > is->max_val)
 439                 is->max_val = val;
 440         if (val < is->min_val)
 441                 is->min_val = val;
 442
 443         is->val += val;
 444         is->val_sq += val * val;
 445         is->samples++;
 446 }
 447
 448 static void add_log_sample(struct thread_data *td, struct io_log *log,
 449                            unsigned long val)
 450 {
 451         if (log->nr_samples == log->max_samples) {
 452                 int new_size = sizeof(struct io_sample) * log->max_samples * 2;
 453
 454                 log->log = realloc(log->log, new_size);
 455                 log->max_samples <<= 1;
 456         }
 457
 458         log->log[log->nr_samples].val = val;
 459         log->log[log->nr_samples].time = mtime_since_now(&td->start);
 460         log->nr_samples++;
 461 }
 462
 463 static void add_clat_sample(struct thread_data *td, unsigned long msec)
 464 {
 465         add_stat_sample(td, &td->clat_stat, msec);
 466
 467         if (td->lat_log)
 468                 add_log_sample(td, td->lat_log, msec);
 469 }
 470
 471 static void add_slat_sample(struct thread_data *td, unsigned long msec)
 472 {
 473         add_stat_sample(td, &td->slat_stat, msec);
 474 }
 475
 476 static void add_bw_sample(struct thread_data *td, unsigned long msec)
 477 {
 478         unsigned long spent = mtime_since_now(&td->stat_sample_time);
 479         unsigned long rate;
 480
 481         if (spent < td->bw_avg_time)
 482                 return;
 483
 484         rate = ((td->this_io_kb - td->stat_io_kb) * 1024) / spent;
 485         add_stat_sample(td, &td->bw_stat, rate);
 486
 487         if (td->bw_log)
 488                 add_log_sample(td, td->bw_log, rate);
 489
 490         gettimeofday(&td->stat_sample_time, NULL);
 491         td->stat_io_kb = td->this_io_kb;
 492 }
 493
 494 static void usec_sleep(int usec)
 495 {
 496         struct timespec req = { .tv_sec = 0, .tv_nsec = usec * 1000 };
 497         struct timespec rem;
 498
 499         do {
 500                 rem.tv_sec = rem.tv_nsec = 0;
 501                 nanosleep(&req, &rem);
 502                 if (!rem.tv_nsec)
 503                         break;
 504
 505                 req.tv_nsec = rem.tv_nsec;
 506         } while (1);
 507 }
 508
 509 static void rate_throttle(struct thread_data *td, unsigned long time_spent,
 510                           unsigned int bytes)
 511 {
 512         unsigned long usec_cycle;
 513
 514         if (!td->rate)
 515                 return;
 516
 517         usec_cycle = td->rate_usec_cycle * (bytes / td->min_bs);
 518
 519         if (time_spent < usec_cycle) {
 520                 unsigned long s = usec_cycle - time_spent;
 521
 522                 td->rate_pending_usleep += s;
 523                 if (td->rate_pending_usleep >= 100000) {
 524                         usec_sleep(td->rate_pending_usleep);
 525                         td->rate_pending_usleep = 0;
 526                 }
 527         } else {
 528                 long overtime = time_spent - usec_cycle;
 529
 530                 td->rate_pending_usleep -= overtime;
 531         }
 532 }
 533
 534 static int check_min_rate(struct thread_data *td, struct timeval *now)
 535 {
 536         unsigned long spent;
 537         unsigned long rate;
 538
 539         /*
 540          * allow a 2 second settle period in the beginning
 541          */
 542         if (mtime_since(&td->start, now) < 2000)
 543                 return 0;
 544
 545         /*
 546          * if rate blocks is set, sample is running
 547          */
 548         if (td->rate_kb) {
 549                 spent = mtime_since(&td->lastrate, now);
 550                 if (spent < td->ratecycle)
 551                         return 0;
 552
 553                 rate = ((td->this_io_kb - td->rate_kb) * 1024) / spent;
 554                 if (rate < td->ratemin) {
 555                         printf("Client%d: min rate %d not met, got %ldKiB/sec\n", td->thread_number, td->ratemin, rate);
 556                         if (rate_quit)
 557                                 sig_handler(0);
 558                         return 1;
 559                 }
 560         }
 561
 562         td->rate_kb = td->this_io_kb;
 563         memcpy(&td->lastrate, now, sizeof(*now));
 564         return 0;
 565 }
 566
 567 static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
 568 {
 569         if (mtime_since(&td->start, t) >= td->timeout * 1000)
 570                 return 1;
 571
 572         return 0;
 573 }
 574
 575 static void fill_random_bytes(struct thread_data *td,
 576                               unsigned char *p, unsigned int len)
 577 {
 578         double r;
 579         int todo;
 580
 581         while (len) {
 582                 drand48_r(&td->verify_state, &r);
 583
 584                 /*
 585                  * lrand48_r seems to be broken and only fill the bottom
 586                  * 32-bits, even on 64-bit archs with 64-bit longs
 587                  */
 588                 todo = sizeof(r);
 589                 if (todo > len)
 590                         todo = len;
 591
 592                 memcpy(p, &r, todo);
 593
 594                 len -= todo;
 595                 p += todo;
 596         }
 597 }
 598
 599 static int verify_io_u(struct thread_data *td, struct io_u *io_u)
 600 {
 601         struct verify_header *hdr = (struct verify_header *) io_u->buf;
 602         unsigned char *p = (unsigned char *) io_u->buf;
 603         struct md5_ctx md5_ctx;
 604
 605         /*
 606          * assume this is a file hole, if we have been doing random writes
 607          * and magic is zero
 608          */
 609         if (hdr->fio_magic != FIO_HDR_MAGIC) {
 610                 if (!td->sequential && !hdr->fio_magic)
 611                         return 0;
 612
 613                 return 1;
 614         }
 615
 616         memset(&md5_ctx, 0, sizeof(md5_ctx));
 617         p += sizeof(*hdr);
 618         md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
 619
 620         return memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
 621 }
 622
 623 static int verify_io_us(struct thread_data *td, struct io_u *io_u, int *back)
 624 {
 625         struct verify_header *hdr;
 626         unsigned int left;
 627         struct io_u i;
 628         char *buf;
 629         off_t off;
 630
 631         if (back)
 632                 *back = 0;
 633
 634         left = io_u->buflen;
 635         buf = io_u->buf;
 636         off = io_u->offset;
 637         while (left) {
 638                 hdr = (struct verify_header *) buf;
 639                 i.buf = buf;
 640                 i.buflen = hdr->len;
 641
 642                 if (hdr->len > left) {
 643                         if (back)
 644                                 *back = left;
 645                         return 0;
 646                 }
 647
 648                 if (verify_io_u(td, &i)) {
 649                         printf("failed verify at offset %lu\n", (unsigned long) off);
 650                         td->error = EBADMSG;
 651                         return 1;
 652                 }
 653
 654                 buf += hdr->len;
 655                 left -= hdr->len;
 656                 off += hdr->len;
 657         }
 658
 659         return 0;
 660 }
 661
 662 /*
 663  * fill body of io_u->buf with random data and add a header with the
 664  * (eg) sha1sum of that data.
 665  */
 666 static void populate_io_u(struct thread_data *td, struct io_u *io_u)
 667 {
 668         struct md5_ctx md5_ctx;
 669         struct verify_header hdr;
 670         unsigned char *p = (unsigned char *) io_u->buf;
 671
 672         hdr.fio_magic = FIO_HDR_MAGIC;
 673         hdr.len = io_u->buflen;
 674         p += sizeof(hdr);
 675         fill_random_bytes(td, p, io_u->buflen - sizeof(hdr));
 676
 677         memset(&md5_ctx, 0, sizeof(md5_ctx));
 678         md5_update(&md5_ctx, p, io_u->buflen - sizeof(hdr));
 679         memcpy(hdr.md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
 680         memcpy(io_u->buf, &hdr, sizeof(hdr));
 681 }
 682
 683 static void put_io_u(struct thread_data *td, struct io_u *io_u)
 684 {
 685         list_del(&io_u->list);
 686         list_add(&io_u->list, &td->io_u_freelist);
 687         td->cur_depth--;
 688 }
 689
 690 #define queue_full(td)  (list_empty(&(td)->io_u_freelist))
 691
 692 static struct io_u *__get_io_u(struct thread_data *td)
 693 {
 694         struct io_u *io_u;
 695
 696         if (queue_full(td))
 697                 return NULL;
 698
 699         io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
 700         list_del(&io_u->list);
 701         list_add(&io_u->list, &td->io_u_busylist);
 702         td->cur_depth++;
 703         return io_u;
 704 }
 705
 706 static struct io_u *get_io_u(struct thread_data *td)
 707 {
 708         struct io_u *io_u;
 709
 710         io_u = __get_io_u(td);
 711         if (!io_u)
 712                 return NULL;
 713
 714         io_u->offset = get_next_offset(td);
 715         io_u->buflen = get_next_buflen(td);
 716         if (!io_u->buflen) {
 717                 put_io_u(td, io_u);
 718                 return NULL;
 719         }
 720
 721         if (!td_read(td) && td->verify)
 722                 populate_io_u(td, io_u);
 723
 724         if (td->use_aio) {
 725                 if (td_read(td))
 726                         io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
 727                 else
 728                         io_prep_pwrite(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
 729         }
 730
 731         gettimeofday(&io_u->start_time, NULL);
 732         return io_u;
 733 }
 734
 735 static inline void td_set_runstate(struct thread_data *td, int runstate)
 736 {
 737         td->old_runstate = td->runstate;
 738         td->runstate = runstate;
 739 }
 740
 741 static int do_sync_verify(struct thread_data *td)
 742 {
 743         struct timeval t;
 744         struct io_u *io_u = NULL;
 745         loff_t off = 0;
 746         int back, ret;
 747
 748         td_set_runstate(td, TD_VERIFYING);
 749
 750         io_u = __get_io_u(td);
 751
 752         if (lseek(td->fd, 0, SEEK_SET) < 0) {
 753                 td->error = errno;
 754                 goto out;
 755         }
 756
 757         if (!td->odirect) {
 758                 unsigned long size = td->kb << 10;
 759
 760                 if (fadvise(td->fd, 0, size, POSIX_FADV_DONTNEED) < 0) {
 761                         td->error = errno;
 762                         goto out;
 763                 }
 764         }
 765
 766         do {
 767                 if (td->terminate)
 768                         break;
 769
 770                 gettimeofday(&t, NULL);
 771                 if (runtime_exceeded(td, &t))
 772                         break;
 773
 774                 io_u->offset = off;
 775                 io_u->buflen = td->max_bs;
 776
 777                 ret = read(td->fd, io_u->buf, io_u->buflen);
 778                 if (ret < (int) io_u->buflen) {
 779                         if (ret == -1) {
 780                                 td->error = errno;
 781                                 break;
 782                         } else if (!ret)
 783                                 break;
 784                         else
 785                                 io_u->buflen = ret;
 786                 }
 787
 788                 if (verify_io_us(td, io_u, &back))
 789                         break;
 790
 791                 if (back) {
 792                         ret -= back;
 793                         if (lseek(td->fd, -back, SEEK_CUR) < 0) {
 794                                 td->error = errno;
 795                                 break;
 796                         }
 797                 }
 798
 799                 off += ret;
 800         } while (1);
 801
 802 out:
 803         td_set_runstate(td, TD_RUNNING);
 804         put_io_u(td, io_u);
 805
 806         return td->error == 0;
 807 }
 808
 809 static void do_sync_io(struct thread_data *td)
 810 {
 811         unsigned long msec, usec;
 812         struct io_u *io_u = NULL;
 813         struct timeval e;
 814
 815         while (td->this_io_kb < td->kb) {
 816                 int ret;
 817
 818                 if (td->terminate)
 819                         break;
 820
 821                 io_u = get_io_u(td);
 822                 if (!io_u)
 823                         break;
 824
 825                 if (td->cur_off != io_u->offset) {
 826                         if (lseek(td->fd, io_u->offset, SEEK_SET) == -1) {
 827                                 td->error = errno;
 828                                 break;
 829                         }
 830                 }
 831
 832                 if (td_read(td))
 833                         ret = read(td->fd, io_u->buf, io_u->buflen);
 834                 else
 835                         ret = write(td->fd, io_u->buf, io_u->buflen);
 836
 837                 if (ret < (int) io_u->buflen) {
 838                         if (ret == -1)
 839                                 td->error = errno;
 840                         break;
 841                 }
 842
 843                 td->io_blocks++;
 844                 td->io_kb += io_u->buflen >> 10;
 845                 td->this_io_kb += io_u->buflen >> 10;
 846                 td->cur_off = io_u->offset + io_u->buflen;
 847
 848                 gettimeofday(&e, NULL);
 849
 850                 usec = utime_since(&io_u->start_time, &e);
 851
 852                 rate_throttle(td, usec, io_u->buflen);
 853
 854                 if (check_min_rate(td, &e)) {
 855                         td->error = ENODATA;
 856                         break;
 857                 }
 858
 859                 msec = usec / 1000;
 860                 add_clat_sample(td, msec);
 861                 add_bw_sample(td, msec);
 862
 863                 if (runtime_exceeded(td, &e))
 864                         break;
 865
 866                 put_io_u(td, io_u);
 867                 io_u = NULL;
 868
 869                 if (td->thinktime)
 870                         usec_sleep(td->thinktime);
 871
 872                 if (should_fsync(td) && td->fsync_blocks &&
 873                     (td->io_blocks % td->fsync_blocks) == 0)
 874                         fsync(td->fd);
 875         }
 876
 877         if (io_u)
 878                 put_io_u(td, io_u);
 879
 880         if (should_fsync(td))
 881                 fsync(td->fd);
 882 }
 883
 884 static int io_u_queue(struct thread_data *td, struct io_u *io_u)
 885 {
 886         struct iocb *iocb = &io_u->iocb;
 887         int ret;
 888
 889         do {
 890                 ret = io_submit(td->aio_ctx, 1, &iocb);
 891                 if (ret == 1)
 892                         return 0;
 893                 else if (ret == EAGAIN)
 894                         usleep(100);
 895                 else if (ret == EINTR)
 896                         continue;
 897                 else
 898                         break;
 899         } while (1);
 900
 901         return ret;
 902 }
 903
 904 #define iocb_time(iocb) ((unsigned long) (iocb)->data)
 905 #define ev_to_iou(ev)   (struct io_u *) ((unsigned long) (ev)->obj)
 906
 907 static int ios_completed(struct thread_data *td, int nr)
 908 {
 909         unsigned long msec;
 910         struct io_u *io_u;
 911         struct timeval e;
 912         int i, bytes_done;
 913
 914         gettimeofday(&e, NULL);
 915
 916         for (i = 0, bytes_done = 0; i < nr; i++) {
 917                 io_u = ev_to_iou(td->aio_events + i);
 918
 919                 td->io_blocks++;
 920                 td->io_kb += io_u->buflen >> 10;
 921                 td->this_io_kb += io_u->buflen >> 10;
 922
 923                 msec = mtime_since(&io_u->issue_time, &e);
 924
 925                 add_clat_sample(td, msec);
 926                 add_bw_sample(td, msec);
 927
 928                 bytes_done += io_u->buflen;
 929                 put_io_u(td, io_u);
 930         }
 931
 932         return bytes_done;
 933 }
 934
 935 static void cleanup_pending_aio(struct thread_data *td)
 936 {
 937         struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
 938         struct list_head *entry, *n;
 939         struct io_u *io_u;
 940         int r;
 941
 942         /*
 943          * get immediately available events, if any
 944          */
 945         r = io_getevents(td->aio_ctx, 0, td->cur_depth, td->aio_events, &ts);
 946         if (r > 0)
 947                 ios_completed(td, r);
 948
 949         /*
 950          * now cancel remaining active events
 951          */
 952         list_for_each_safe(entry, n, &td->io_u_busylist) {
 953                 io_u = list_entry(entry, struct io_u, list);
 954
 955                 r = io_cancel(td->aio_ctx, &io_u->iocb, td->aio_events);
 956                 if (!r)
 957                         put_io_u(td, io_u);
 958         }
 959
 960         if (td->cur_depth) {
 961                 r = io_getevents(td->aio_ctx, td->cur_depth, td->cur_depth, td->aio_events, NULL);
 962                 if (r > 0)
 963                         ios_completed(td, r);
 964         }
 965 }
 966
 967 static int async_do_verify(struct thread_data *td, struct io_u **io_u)
 968 {
 969         struct io_u *v_io_u = *io_u;
 970         int ret = 0;
 971
 972         if (v_io_u) {
 973                 ret = verify_io_us(td, v_io_u, NULL);
 974                 put_io_u(td, v_io_u);
 975                 *io_u = NULL;
 976         }
 977
 978         return ret;
 979 }
 980
 981 static int do_async_verify(struct thread_data *td)
 982 {
 983         struct timeval t;
 984         struct io_u *io_u, *v_io_u = NULL;
 985         struct verify_header *hdr;
 986         int ret, back;
 987         char *p;
 988
 989         td_set_runstate(td, TD_VERIFYING);
 990
 991         do {
 992                 if (td->terminate)
 993                         break;
 994
 995                 gettimeofday(&t, NULL);
 996                 if (runtime_exceeded(td, &t))
 997                         break;
 998
 999                 io_u = __get_io_u(td);
1000                 if (!io_u)
1001                         break;
1002
1003                 io_u->offset = td->cur_off;
1004                 io_u->buflen = td->max_bs;
1005
1006                 if (io_u->offset + io_u->buflen > (td->kb << 10)) {
1007                         io_u->buflen = (td->kb << 10) - io_u->offset;
1008                         if (!io_u->buflen) {
1009                                 put_io_u(td, io_u);
1010                                 break;
1011                         }
1012                 }
1013
1014                 io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
1015                 ret = io_u_queue(td, io_u);
1016                 if (ret) {
1017                         put_io_u(td, io_u);
1018                         td->error = ret;
1019                         break;
1020                 }
1021
1022                 /*
1023                  * we have one pending to verify, do that while the next
1024                  * we are doing io on the next one
1025                  */
1026                 if (async_do_verify(td, &v_io_u))
1027                         break;
1028
1029                 ret = io_getevents(td->aio_ctx, 1, 1, td->aio_events, NULL);
1030                 if (ret != 1) {
1031                         if (ret < 0)
1032                                 td->error = ret;
1033                         break;
1034                 }
1035
1036                 /*
1037                  * got our io_u to verify, find back offset so we can
1038                  * submit the next one before verifying this one
1039                  */
1040                 v_io_u = ev_to_iou(td->aio_events);
1041                 p = v_io_u->buf;
1042                 back = v_io_u->buflen;
1043                 do {
1044                         hdr = (struct verify_header *) p;
1045
1046                         if (hdr->len > back)
1047                                 break;
1048
1049                         back -= hdr->len;
1050                         p += hdr->len;
1051                 } while (back);
1052
1053                 td->cur_off += (v_io_u->buflen - back);
1054
1055                 /*
1056                  * if we can't submit more io, we need to verify now
1057                  */
1058                 if (queue_full(td) && async_do_verify(td, &v_io_u))
1059                         break;
1060
1061         } while (1);
1062
1063         async_do_verify(td, &v_io_u);
1064
1065         if (td->cur_depth)
1066                 cleanup_pending_aio(td);
1067
1068         td_set_runstate(td, TD_RUNNING);
1069         return td->error == 0;
1070 }
1071
1072 static void do_async_io(struct thread_data *td)
1073 {
1074         struct timeval s, e;
1075         unsigned long usec;
1076
1077         while (td->this_io_kb < td->kb) {
1078                 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
1079                 struct timespec *timeout;
1080                 int ret, min_evts = 0;
1081                 struct io_u *io_u;
1082                 unsigned int bytes_done;
1083
1084                 if (td->terminate)
1085                         break;
1086
1087                 io_u = get_io_u(td);
1088                 if (!io_u)
1089                         break;
1090
1091                 memcpy(&s, &io_u->start_time, sizeof(s));
1092
1093                 ret = io_u_queue(td, io_u);
1094                 if (ret) {
1095                         put_io_u(td, io_u);
1096                         td->error = ret;
1097                         break;
1098                 }
1099
1100                 gettimeofday(&io_u->issue_time, NULL);
1101                 add_slat_sample(td, mtime_since(&io_u->start_time, &io_u->issue_time));
1102                 if (td->cur_depth < td->aio_depth) {
1103                         timeout = &ts;
1104                         min_evts = 0;
1105                 } else {
1106                         timeout = NULL;
1107                         min_evts = 1;
1108                 }
1109
1110                 ret = io_getevents(td->aio_ctx, min_evts, td->cur_depth, td->aio_events, timeout);
1111                 if (ret < 0) {
1112                         td->error = errno;
1113                         break;
1114                 } else if (!ret)
1115                         continue;
1116
1117                 bytes_done = ios_completed(td, ret);
1118
1119                 /*
1120                  * the rate is batched for now, it should work for batches
1121                  * of completions except the very first one which may look
1122                  * a little bursty
1123                  */
1124                 gettimeofday(&e, NULL);
1125                 usec = utime_since(&s, &e);
1126
1127                 rate_throttle(td, usec, bytes_done);
1128
1129                 if (check_min_rate(td, &e)) {
1130                         td->error = ENODATA;
1131                         break;
1132                 }
1133
1134                 if (runtime_exceeded(td, &e))
1135                         break;
1136
1137                 if (td->thinktime)
1138                         usec_sleep(td->thinktime);
1139
1140                 if (should_fsync(td) && td->fsync_blocks &&
1141                     (td->io_blocks % td->fsync_blocks) == 0)
1142                         fsync(td->fd);
1143         }
1144
1145         if (td->cur_depth)
1146                 cleanup_pending_aio(td);
1147
1148         if (should_fsync(td))
1149                 fsync(td->fd);
1150 }
1151
1152 static void cleanup_aio(struct thread_data *td)
1153 {
1154         io_destroy(td->aio_ctx);
1155
1156         if (td->aio_events)
1157                 free(td->aio_events);
1158 }
1159
1160 static int init_aio(struct thread_data *td)
1161 {
1162         if (io_queue_init(td->aio_depth, &td->aio_ctx)) {
1163                 td->error = errno;
1164                 return 1;
1165         }
1166
1167         td->aio_events = malloc(td->aio_depth * sizeof(struct io_event));
1168         return 0;
1169 }
1170
1171 static void cleanup_io_u(struct thread_data *td)
1172 {
1173         struct list_head *entry, *n;
1174         struct io_u *io_u;
1175
1176         list_for_each_safe(entry, n, &td->io_u_freelist) {
1177                 io_u = list_entry(entry, struct io_u, list);
1178
1179                 list_del(&io_u->list);
1180                 free(io_u);
1181         }
1182
1183         if (td->mem_type == MEM_MALLOC)
1184                 free(td->orig_buffer);
1185         else if (td->mem_type == MEM_SHM) {
1186                 struct shmid_ds sbuf;
1187
1188                 shmdt(td->orig_buffer);
1189                 shmctl(td->shm_id, IPC_RMID, &sbuf);
1190         }
1191 }
1192
1193 static int init_io_u(struct thread_data *td)
1194 {
1195         struct io_u *io_u;
1196         int i, max_units, mem_size;
1197         char *p;
1198
1199         if (!td->use_aio)
1200                 max_units = 1;
1201         else
1202                 max_units = td->aio_depth;
1203
1204         mem_size = td->max_bs * max_units + MASK;
1205
1206         if (td->mem_type == MEM_MALLOC)
1207                 td->orig_buffer = malloc(mem_size);
1208         else if (td->mem_type == MEM_SHM) {
1209                 td->shm_id = shmget(IPC_PRIVATE, mem_size, IPC_CREAT | 0600);
1210                 if (td->shm_id < 0) {
1211                         td->error = errno;
1212                         perror("shmget");
1213                         return 1;
1214                 }
1215
1216                 td->orig_buffer = shmat(td->shm_id, NULL, 0);
1217                 if (td->orig_buffer == (void *) -1) {
1218                         td->error = errno;
1219                         perror("shmat");
1220                         return 1;
1221                 }
1222         }
1223
1224         INIT_LIST_HEAD(&td->io_u_freelist);
1225         INIT_LIST_HEAD(&td->io_u_busylist);
1226
1227         p = ALIGN(td->orig_buffer);
1228         for (i = 0; i < max_units; i++) {
1229                 io_u = malloc(sizeof(*io_u));
1230                 memset(io_u, 0, sizeof(*io_u));
1231                 INIT_LIST_HEAD(&io_u->list);
1232
1233                 io_u->buf = p + td->max_bs * i;
1234                 list_add(&io_u->list, &td->io_u_freelist);
1235         }
1236
1237         return 0;
1238 }
1239
1240 static void setup_log(struct io_log **log)
1241 {
1242         struct io_log *l = malloc(sizeof(*l));
1243
1244         l->nr_samples = 0;
1245         l->max_samples = 1024;
1246         l->log = malloc(l->max_samples * sizeof(struct io_sample));
1247         *log = l;
1248 }
1249
1250 static void finish_log(struct thread_data *td, struct io_log *log, char *name)
1251 {
1252         char file_name[128];
1253         FILE *f;
1254         int i;
1255
1256         sprintf(file_name, "client%d_%s.log", td->thread_number, name);
1257         f = fopen(file_name, "w");
1258         if (!f) {
1259                 perror("fopen log");
1260                 return;
1261         }
1262
1263         for (i = 0; i < log->nr_samples; i++)
1264                 fprintf(f, "%lu, %lu\n", log->log[i].time, log->log[i].val);
1265
1266         fclose(f);
1267         free(log->log);
1268         free(log);
1269 }
1270
1271 static int create_file(struct thread_data *td)
1272 {
1273         unsigned long long left;
1274         char *b;
1275         int r, bs;
1276
1277         /*
1278          * unless specifically asked for overwrite, let normal io extend it
1279          */
1280         if (!td_read(td) && !td->overwrite)
1281                 return 0;
1282
1283         if (!td->file_size) {
1284                 fprintf(stderr, "Need size for create\n");
1285                 td->error = EINVAL;
1286                 return 1;
1287         }
1288
1289         printf("Client%d: Laying out IO file\n", td->thread_number);
1290
1291         td->fd = open(td->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
1292         if (td->fd < 0) {
1293                 td->error = errno;
1294                 return 1;
1295         }
1296
1297         td->kb = td->file_size >> 10;
1298         b = malloc(td->max_bs);
1299         memset(b, 0, td->max_bs);
1300
1301         left = td->file_size;
1302         while (left) {
1303                 bs = td->max_bs;
1304                 if (bs > left)
1305                         bs = left;
1306
1307                 r = write(td->fd, b, bs);
1308
1309                 if (r == bs) {
1310                         left -= bs;
1311                         continue;
1312                 } else {
1313                         if (r < 0)
1314                                 td->error = errno;
1315                         else
1316                                 td->error = EIO;
1317
1318                         break;
1319                 }
1320         }
1321
1322         if (td->create_fsync)
1323                 fsync(td->fd);
1324
1325         close(td->fd);
1326         td->fd = -1;
1327         free(b);
1328         return 0;
1329 }
1330
1331 static int file_exists(struct thread_data *td)
1332 {
1333         struct stat st;
1334
1335         if (stat(td->file_name, &st) != -1)
1336                 return 1;
1337
1338         return errno != ENOENT;
1339 }
1340
1341 static int setup_file(struct thread_data *td)
1342 {
1343         struct stat st;
1344         int flags = 0;
1345
1346         if (!file_exists(td)) {
1347                 if (!td->create_file) {
1348                         td->error = ENOENT;
1349                         return 1;
1350                 }
1351                 if (create_file(td))
1352                         return 1;
1353         }
1354
1355         if (td->odirect)
1356                 flags |= O_DIRECT;
1357
1358         if (td_read(td))
1359                 td->fd = open(td->file_name, flags | O_RDONLY);
1360         else {
1361                 if (!td->overwrite)
1362                         flags |= O_TRUNC;
1363                 if (td->sync_io)
1364                         flags |= O_SYNC;
1365                 if (td->verify)
1366                         flags |= O_RDWR;
1367                 else
1368                         flags |= O_WRONLY;
1369
1370                 td->fd = open(td->file_name, flags | O_CREAT, 0600);
1371         }
1372
1373         if (td->fd == -1) {
1374                 td->error = errno;
1375                 return 1;
1376         }
1377
1378         if (td_read(td)) {
1379                 if (fstat(td->fd, &st) == -1) {
1380                         td->error = errno;
1381                         return 1;
1382                 }
1383
1384                 if (td->file_size > st.st_size)
1385                         st.st_size = td->file_size;
1386         } else {
1387                 if (!td->file_size)
1388                         td->file_size = 1024 * 1024 * 1024;
1389
1390                 st.st_size = td->file_size;
1391         }
1392
1393         td->kb = (st.st_size - td->file_offset) / 1024;
1394         if (!td->kb) {
1395                 fprintf(stderr, "Client%d: no io blocks\n", td->thread_number);
1396                 td->error = EINVAL;
1397                 return 1;
1398         }
1399
1400         if (td->invalidate_cache) {
1401                 if (fadvise(td->fd, 0, st.st_size, POSIX_FADV_DONTNEED) < 0) {
1402                         td->error = errno;
1403                         return 1;
1404                 }
1405         }
1406
1407         return 0;
1408 }
1409
1410 static void clear_io_state(struct thread_data *td)
1411 {
1412         td->cur_off = 0;
1413         td->last_kb = 0;
1414         td->stat_io_kb = 0;
1415         td->this_io_kb = 0;
1416 }
1417
1418 static void *thread_main(int shm_id, int offset, char *argv[])
1419 {
1420         struct thread_data *td;
1421         int ret = 1;
1422         void *data;
1423
1424         setsid();
1425
1426         data = shmat(shm_id, NULL, 0);
1427         if (data == (void *) -1) {
1428                 perror("shmat");
1429                 return NULL;
1430         }
1431
1432         td = data + offset * sizeof(struct thread_data);
1433         td->pid = getpid();
1434
1435         if (init_io_u(td))
1436                 goto err;
1437
1438         if (sched_setaffinity(td->pid, sizeof(td->cpumask), &td->cpumask) == -1) {
1439                 td->error = errno;
1440                 goto err;
1441         }
1442
1443         sprintf(argv[0], "fio%d", offset);
1444
1445         if (td->use_aio && init_aio(td))
1446                 goto err;
1447
1448         if (init_random_state(td))
1449                 goto err;
1450
1451         if (td->ioprio) {
1452                 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
1453                         td->error = errno;
1454                         goto err;
1455                 }
1456         }
1457
1458         sem_post(&startup_sem);
1459         sem_wait(&td->mutex);
1460
1461         if (!td->create_serialize && setup_file(td))
1462                 goto err;
1463
1464         gettimeofday(&td->start, NULL);
1465
1466         while (td->loops--) {
1467                 gettimeofday(&td->stat_sample_time, NULL);
1468
1469                 if (td->ratemin)
1470                         memcpy(&td->lastrate, &td->stat_sample_time, sizeof(td->lastrate));
1471
1472                 clear_io_state(td);
1473
1474                 if (!td->use_aio) {
1475                         do_sync_io(td);
1476
1477                         if (!td->verify || td_read(td))
1478                                 continue;
1479
1480                         clear_io_state(td);
1481                         if (!do_sync_verify(td))
1482                                 break;
1483                 } else {
1484                         do_async_io(td);
1485
1486                         if (!td->verify || td_read(td))
1487                                 continue;
1488
1489                         clear_io_state(td);
1490                         if (!do_async_verify(td))
1491                                 break;
1492                 }
1493         }
1494
1495         td->runtime = mtime_since_now(&td->start);
1496         ret = 0;
1497
1498         if (td->bw_log)
1499                 finish_log(td, td->bw_log, "bw");
1500         if (td->lat_log)
1501                 finish_log(td, td->lat_log, "lat");
1502
1503         if (exitall_on_terminate)
1504                 sig_handler(0);
1505
1506 err:
1507         if (td->fd != -1) {
1508                 close(td->fd);
1509                 td->fd = -1;
1510         }
1511         if (td->use_aio)
1512                 cleanup_aio(td);
1513         cleanup_io_u(td);
1514         if (ret) {
1515                 sem_post(&startup_sem);
1516                 sem_wait(&td->mutex);
1517         }
1518         td_set_runstate(td, TD_EXITED);
1519         shmdt(data);
1520         return NULL;
1521 }
1522
1523 static void free_shm(void)
1524 {
1525         struct shmid_ds sbuf;
1526
1527         if (threads) {
1528                 shmdt(threads);
1529                 threads = NULL;
1530                 shmctl(shm_id, IPC_RMID, &sbuf);
1531         }
1532 }
1533
1534 static int calc_lat(struct io_stat *is, unsigned long *min, unsigned long *max,
1535                     double *mean, double *dev)
1536 {
1537         double n;
1538
1539         if (is->samples == 0)
1540                 return 0;
1541
1542         *min = is->min_val;
1543         *max = is->max_val;
1544
1545         n = (double) is->samples;
1546         *mean = (double) is->val / n;
1547         *dev = sqrt(((double) is->val_sq - (*mean * *mean) / n) / (n - 1));
1548         return 1;
1549 }
1550
1551 static void show_thread_status(struct thread_data *td)
1552 {
1553         int prio, prio_class;
1554         unsigned long min, max, bw = 0;
1555         double mean, dev;
1556
1557         if (!td->io_kb && !td->error)
1558                 return;
1559
1560         if (td->runtime)
1561                 bw = td->io_kb * 1024 / td->runtime;
1562
1563         prio = td->ioprio & 0xff;
1564         prio_class = td->ioprio >> IOPRIO_CLASS_SHIFT;
1565
1566         printf("Client%d: err=%2d, io=%6luMiB, bw=%6luKiB/s, runt=%6lumsec\n", td->thread_number, td->error, td->io_kb >> 10, bw, td->runtime);
1567
1568         if (calc_lat(&td->slat_stat, &min, &max, &mean, &dev))
1569                 printf("  slat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1570         if (calc_lat(&td->clat_stat, &min, &max, &mean, &dev))
1571                 printf("  clat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1572         if (calc_lat(&td->bw_stat, &min, &max, &mean, &dev))
1573                 printf("  bw (KiB/s) : min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1574 }
1575
1576 static int setup_rate(struct thread_data *td)
1577 {
1578         int nr_reads_per_sec;
1579
1580         if (!td->rate)
1581                 return 0;
1582
1583         if (td->rate < td->ratemin) {
1584                 fprintf(stderr, "min rate larger than nominal rate\n");
1585                 return -1;
1586         }
1587
1588         nr_reads_per_sec = td->rate * 1024 / td->min_bs;
1589         td->rate_usec_cycle = 1000000 / nr_reads_per_sec;
1590         td->rate_pending_usleep = 0;
1591         return 0;
1592 }
1593
1594 static struct thread_data *get_new_job(int global)
1595 {
1596         struct thread_data *td;
1597
1598         if (global)
1599                 return &def_thread;
1600         if (thread_number >= max_jobs)
1601                 return NULL;
1602
1603         td = &threads[thread_number++];
1604         memset(td, 0, sizeof(*td));
1605
1606         td->fd = -1;
1607         td->thread_number = thread_number;
1608
1609         td->ddir = def_thread.ddir;
1610         td->ioprio = def_thread.ioprio;
1611         td->sequential = def_thread.sequential;
1612         td->bs = def_thread.bs;
1613         td->min_bs = def_thread.min_bs;
1614         td->max_bs = def_thread.max_bs;
1615         td->odirect = def_thread.odirect;
1616         td->thinktime = def_thread.thinktime;
1617         td->fsync_blocks = def_thread.fsync_blocks;
1618         td->start_delay = def_thread.start_delay;
1619         td->timeout = def_thread.timeout;
1620         td->use_aio = def_thread.use_aio;
1621         td->create_file = def_thread.create_file;
1622         td->overwrite = def_thread.overwrite;
1623         td->invalidate_cache = def_thread.invalidate_cache;
1624         td->file_size = def_thread.file_size;
1625         td->file_offset = def_thread.file_offset;
1626         td->rate = def_thread.rate;
1627         td->ratemin = def_thread.ratemin;
1628         td->ratecycle = def_thread.ratecycle;
1629         td->aio_depth = def_thread.aio_depth;
1630         td->sync_io = def_thread.sync_io;
1631         td->mem_type = def_thread.mem_type;
1632         td->bw_avg_time = def_thread.bw_avg_time;
1633         td->create_serialize = def_thread.create_serialize;
1634         td->create_fsync = def_thread.create_fsync;
1635         td->loops = def_thread.loops;
1636         td->verify = def_thread.verify;
1637         memcpy(&td->cpumask, &def_thread.cpumask, sizeof(td->cpumask));
1638
1639         return td;
1640 }
1641
1642 static void put_job(struct thread_data *td)
1643 {
1644         memset(&threads[td->thread_number - 1], 0, sizeof(*td));
1645         thread_number--;
1646 }
1647
1648 static int add_job(struct thread_data *td, const char *filename, int prioclass,
1649                    int prio)
1650 {
1651         if (td == &def_thread)
1652                 return 0;
1653
1654         strcpy(td->file_name, filename);
1655         sem_init(&td->mutex, 1, 0);
1656         td->ioprio = (prioclass << IOPRIO_CLASS_SHIFT) | prio;
1657
1658         td->clat_stat.min_val = ULONG_MAX;
1659         td->slat_stat.min_val = ULONG_MAX;
1660         td->bw_stat.min_val = ULONG_MAX;
1661
1662         run_str[td->thread_number - 1] = 'P';
1663
1664         if (td->use_aio && !td->aio_depth)
1665                 td->aio_depth = 1;
1666
1667         if (td->min_bs == -1)
1668                 td->min_bs = td->bs;
1669         if (td->max_bs == -1)
1670                 td->max_bs = td->bs;
1671
1672         if (setup_rate(td))
1673                 return -1;
1674
1675         if (write_lat_log)
1676                 setup_log(&td->lat_log);
1677         if (write_bw_log)
1678                 setup_log(&td->bw_log);
1679
1680         printf("Client%d: file=%s, rw=%d, prio=%d/%d, seq=%d, odir=%d, bs=%d-%d, rate=%d, aio=%d, aio_depth=%d\n", td->thread_number, filename, td->ddir, prioclass, prio, td->sequential, td->odirect, td->min_bs, td->max_bs, td->rate, td->use_aio, td->aio_depth);
1681         return 0;
1682 }
1683
1684 static void fill_cpu_mask(cpu_set_t cpumask, int cpu)
1685 {
1686         unsigned int i;
1687
1688         CPU_ZERO(&cpumask);
1689
1690         for (i = 0; i < sizeof(int) * 8; i++) {
1691                 if ((1 << i) & cpu)
1692                         CPU_SET(i, &cpumask);
1693         }
1694 }
1695
1696 unsigned long get_mult(char c)
1697 {
1698         switch (c) {
1699                 case 'k':
1700                 case 'K':
1701                         return 1024;
1702                 case 'm':
1703                 case 'M':
1704                         return 1024 * 1024;
1705                 case 'g':
1706                 case 'G':
1707                         return 1024 * 1024 * 1024;
1708                 default:
1709                         return 1;
1710         }
1711 }
1712
1713 /*
1714  * convert string after '=' into decimal value, noting any size suffix
1715  */
1716 static int str_cnv(char *p, unsigned long long *val)
1717 {
1718         char *str;
1719         int len;
1720
1721         str = strstr(p, "=");
1722         if (!str)
1723                 return 1;
1724
1725         str++;
1726         len = strlen(str);
1727
1728         *val = strtoul(str, NULL, 10);
1729         if (*val == ULONG_MAX && errno == ERANGE)
1730                 return 1;
1731
1732         *val *= get_mult(str[len - 2]);
1733         return 0;
1734 }
1735
1736 static int check_strcnv(char *p, char *name, unsigned long long *val)
1737 {
1738         if (!strstr(p, name))
1739                 return 1;
1740
1741         return str_cnv(p, val);
1742 }
1743
1744 static int check_str(char *p, char *name, char *option)
1745 {
1746         char *s = strstr(p, name);
1747
1748         if (!s)
1749                 return 1;
1750
1751         s += strlen(name);
1752         if (strstr(s, option))
1753                 return 0;
1754
1755         return 1;
1756 }
1757
1758 static int check_range(char *p, char *name, unsigned long *s, unsigned long *e)
1759 {
1760         char str[128];
1761         char s1, s2;
1762
1763         sprintf(str, "%s=%%lu%%c-%%lu%%c", name);
1764         if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1765                 *s *= get_mult(s1);
1766                 *e *= get_mult(s2);
1767                 return 0;
1768         }
1769
1770         sprintf(str, "%s = %%lu%%c-%%lu%%c", name);
1771         if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1772                 *s *= get_mult(s1);
1773                 *e *= get_mult(s2);
1774                 return 0;
1775         }
1776
1777         sprintf(str, "%s=%%lu-%%lu", name);
1778         if (sscanf(p, str, s, e) == 2)
1779                 return 0;
1780
1781         sprintf(str, "%s = %%lu-%%lu", name);
1782         if (sscanf(p, str, s, e) == 2)
1783                 return 0;
1784
1785         return 1;
1786
1787 }
1788
1789 static int check_int(char *p, char *name, unsigned int *val)
1790 {
1791         char str[128];
1792
1793         sprintf(str, "%s=%%d", name);
1794         if (sscanf(p, str, val) == 1)
1795                 return 0;
1796
1797         sprintf(str, "%s = %%d", name);
1798         if (sscanf(p, str, val) == 1)
1799                 return 0;
1800
1801         return 1;
1802 }
1803
1804 static int is_empty_or_comment(char *line)
1805 {
1806         unsigned int i;
1807
1808         for (i = 0; i < strlen(line); i++) {
1809                 if (line[i] == ';')
1810                         return 1;
1811                 if (!isspace(line[i]) && !iscntrl(line[i]))
1812                         return 0;
1813         }
1814
1815         return 1;
1816 }
1817
1818 static int parse_jobs_ini(char *file)
1819 {
1820         unsigned int prioclass, prio, cpu, global;
1821         unsigned long long ull;
1822         unsigned long ul1, ul2;
1823         struct thread_data *td;
1824         char *string, *name;
1825         fpos_t off;
1826         FILE *f;
1827         char *p;
1828
1829         f = fopen(file, "r");
1830         if (!f) {
1831                 perror("fopen");
1832                 return 1;
1833         }
1834
1835         string = malloc(4096);
1836         name = malloc(256);
1837
1838         while ((p = fgets(string, 4096, f)) != NULL) {
1839                 if (is_empty_or_comment(p))
1840                         continue;
1841                 if (sscanf(p, "[%s]", name) != 1)
1842                         continue;
1843
1844                 global = !strncmp(name, "global", 6);
1845
1846                 name[strlen(name) - 1] = '\0';
1847
1848                 td = get_new_job(global);
1849                 if (!td)
1850                         break;
1851
1852                 prioclass = 2;
1853                 prio = 4;
1854
1855                 fgetpos(f, &off);
1856                 while ((p = fgets(string, 4096, f)) != NULL) {
1857                         if (is_empty_or_comment(p))
1858                                 continue;
1859                         if (strstr(p, "["))
1860                                 break;
1861                         if (!check_int(p, "rw", &td->ddir)) {
1862                                 fgetpos(f, &off);
1863                                 continue;
1864                         }
1865                         if (!check_int(p, "prio", &prio)) {
1866                                 fgetpos(f, &off);
1867                                 continue;
1868                         }
1869                         if (!check_int(p, "prioclass", &prioclass)) {
1870                                 fgetpos(f, &off);
1871                                 continue;
1872                         }
1873                         if (!check_int(p, "direct", &td->odirect)) {
1874                                 fgetpos(f, &off);
1875                                 continue;
1876                         }
1877                         if (!check_int(p, "rate", &td->rate)) {
1878                                 fgetpos(f, &off);
1879                                 continue;
1880                         }
1881                         if (!check_int(p, "ratemin", &td->ratemin)) {
1882                                 fgetpos(f, &off);
1883                                 continue;
1884                         }
1885                         if (!check_int(p, "ratecycle", &td->ratecycle)) {
1886                                 fgetpos(f, &off);
1887                                 continue;
1888                         }
1889                         if (!check_int(p, "thinktime", &td->thinktime)) {
1890                                 fgetpos(f, &off);
1891                                 continue;
1892                         }
1893                         if (!check_int(p, "cpumask", &cpu)) {
1894                                 fill_cpu_mask(td->cpumask, cpu);
1895                                 fgetpos(f, &off);
1896                                 continue;
1897                         }
1898                         if (!check_int(p, "fsync", &td->fsync_blocks)) {
1899                                 fgetpos(f, &off);
1900                                 continue;
1901                         }
1902                         if (!check_int(p, "startdelay", &td->start_delay)) {
1903                                 fgetpos(f, &off);
1904                                 continue;
1905                         }
1906                         if (!check_int(p, "timeout", &td->timeout)) {
1907                                 fgetpos(f, &off);
1908                                 continue;
1909                         }
1910                         if (!check_int(p, "invalidate",&td->invalidate_cache)) {
1911                                 fgetpos(f, &off);
1912                                 continue;
1913                         }
1914                         if (!check_int(p, "aio_depth", &td->aio_depth)) {
1915                                 fgetpos(f, &off);
1916                                 continue;
1917                         }
1918                         if (!check_int(p, "sync", &td->sync_io)) {
1919                                 fgetpos(f, &off);
1920                                 continue;
1921                         }
1922                         if (!check_int(p, "bwavgtime", &td->bw_avg_time)) {
1923                                 fgetpos(f, &off);
1924                                 continue;
1925                         }
1926                         if (!check_int(p, "create_serialize", &td->create_serialize)) {
1927                                 fgetpos(f, &off);
1928                                 continue;
1929                         }
1930                         if (!check_int(p, "create_fsync", &td->create_fsync)) {
1931                                 fgetpos(f, &off);
1932                                 continue;
1933                         }
1934                         if (!check_int(p, "loops", &td->loops)) {
1935                                 fgetpos(f, &off);
1936                                 continue;
1937                         }
1938                         if (!check_int(p, "verify", &td->verify)) {
1939                                 fgetpos(f, &off);
1940                                 continue;
1941                         }
1942                         if (!check_range(p, "bsrange", &ul1, &ul2)) {
1943                                 td->min_bs = ul1;
1944                                 td->max_bs = ul2;
1945                                 fgetpos(f, &off);
1946                                 continue;
1947                         }
1948                         if (!check_strcnv(p, "bs", &ull)) {
1949                                 td->bs = ull;
1950                                 fgetpos(f, &off);
1951                                 continue;
1952                         }
1953                         if (!check_strcnv(p, "size", &td->file_size)) {
1954                                 fgetpos(f, &off);
1955                                 continue;
1956                         }
1957                         if (!check_strcnv(p, "offset", &td->file_offset)) {
1958                                 fgetpos(f, &off);
1959                                 continue;
1960                         }
1961                         if (!check_str(p, "mem", "malloc")) {
1962                                 td->mem_type = MEM_MALLOC;
1963                                 fgetpos(f, &off);
1964                                 continue;
1965                         }
1966                         if (!check_str(p, "mem", "shm")) {
1967                                 td->mem_type = MEM_SHM;
1968                                 fgetpos(f, &off);
1969                                 continue;
1970                         }
1971                         if (!strncmp(p, "sequential", 10)) {
1972                                 td->sequential = 1;
1973                                 fgetpos(f, &off);
1974                                 continue;
1975                         }
1976                         if (!strncmp(p, "random", 6)) {
1977                                 td->sequential = 0;
1978                                 fgetpos(f, &off);
1979                                 continue;
1980                         }
1981                         if (!strncmp(p, "aio", 3)) {
1982                                 td->use_aio = 1;
1983                                 fgetpos(f, &off);
1984                                 continue;
1985                         }
1986                         if (!strncmp(p, "create", 6)) {
1987                                 td->create_file = 1;
1988                                 fgetpos(f, &off);
1989                                 continue;
1990                         }
1991                         if (!strncmp(p, "overwrite", 9)) {
1992                                 td->overwrite = 1;
1993                                 fgetpos(f, &off);
1994                                 continue;
1995                         }
1996                         if (!strncmp(p, "exitall", 7)) {
1997                                 exitall_on_terminate = 1;
1998                                 fgetpos(f, &off);
1999                                 continue;
2000                         }
2001                         printf("Client%d: bad option %s\n",td->thread_number,p);
2002                 }
2003                 fsetpos(f, &off);
2004
2005                 if (add_job(td, name, prioclass, prio))
2006                         put_job(td);
2007         }
2008
2009         free(string);
2010         free(name);
2011         fclose(f);
2012         return 0;
2013 }
2014
2015 static int parse_options(int argc, char *argv[])
2016 {
2017         int i;
2018
2019         for (i = 1; i < argc; i++) {
2020                 char *parm = argv[i];
2021
2022                 if (parm[0] != '-')
2023                         break;
2024
2025                 parm++;
2026                 switch (*parm) {
2027                         case 's':
2028                                 parm++;
2029                                 def_thread.sequential = !!atoi(parm);
2030                                 break;
2031                         case 'b':
2032                                 parm++;
2033                                 def_thread.bs = atoi(parm);
2034                                 def_thread.bs <<= 10;
2035                                 if (!def_thread.bs) {
2036                                         printf("bad block size\n");
2037                                         def_thread.bs = DEF_BS;
2038                                 }
2039                                 break;
2040                         case 't':
2041                                 parm++;
2042                                 def_thread.timeout = atoi(parm);
2043                                 break;
2044                         case 'r':
2045                                 parm++;
2046                                 repeatable = !!atoi(parm);
2047                                 break;
2048                         case 'R':
2049                                 parm++;
2050                                 rate_quit = !!atoi(parm);
2051                                 break;
2052                         case 'o':
2053                                 parm++;
2054                                 def_thread.odirect = !!atoi(parm);
2055                                 break;
2056                         case 'f':
2057                                 if (i + 1 >= argc) {
2058                                         printf("-f needs file as arg\n");
2059                                         break;
2060                                 }
2061                                 ini_file = strdup(argv[i+1]);
2062                                 i++;
2063                                 break;
2064                         case 'l':
2065                                 write_lat_log = 1;
2066                                 break;
2067                         case 'w':
2068                                 write_bw_log = 1;
2069                                 break;
2070                         default:
2071                                 printf("bad option %s\n", argv[i]);
2072                                 break;
2073                 }
2074         }
2075
2076         return i;
2077 }
2078
2079 static void print_thread_status(struct thread_data *td, int nr_running,
2080                                 int t_rate, int m_rate)
2081 {
2082         printf("Threads now running: %d", nr_running);
2083         if (m_rate || t_rate)
2084                 printf(", commitrate %d/%dKiB/sec", t_rate, m_rate);
2085         printf(" : [%s]\r", run_str);
2086         fflush(stdout);
2087 }
2088
2089 static void check_str_update(struct thread_data *td, int n, int t, int m)
2090 {
2091         char c = run_str[td->thread_number - 1];
2092
2093         if (td->runstate == td->old_runstate)
2094                 return;
2095
2096         switch (td->runstate) {
2097                 case TD_REAPED:
2098                         c = '_';
2099                         break;
2100                 case TD_EXITED:
2101                         c = 'E';
2102                         break;
2103                 case TD_RUNNING:
2104                         c = '+';
2105                         break;
2106                 case TD_VERIFYING:
2107                         c = 'V';
2108                         break;
2109                 case TD_CREATED:
2110                         c = 'C';
2111                         break;
2112                 case TD_NOT_CREATED:
2113                         c = 'P';
2114                         break;
2115                 default:
2116                         printf("state %d\n", td->runstate);
2117         }
2118
2119         run_str[td->thread_number - 1] = c;
2120         print_thread_status(td, n, t, m);
2121         td->old_runstate = td->runstate;
2122 }
2123
2124 static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
2125 {
2126         int i;
2127
2128         /*
2129          * reap exited threads (TD_EXITED -> TD_REAPED)
2130          */
2131         for (i = 0; i < thread_number; i++) {
2132                 struct thread_data *td = &threads[i];
2133
2134                 check_str_update(td, *nr_running, *t_rate, *m_rate);
2135
2136                 if (td->runstate != TD_EXITED)
2137                         continue;
2138
2139                 td_set_runstate(td, TD_REAPED);
2140                 waitpid(td->pid, NULL, 0);
2141                 (*nr_running)--;
2142                 (*m_rate) -= td->ratemin;
2143                 (*t_rate) -= td->rate;
2144                 check_str_update(td, *nr_running, *t_rate, *m_rate);
2145
2146                 if (td->terminate)
2147                         continue;
2148         }
2149 }
2150
2151 static void run_threads(char *argv[])
2152 {
2153         struct timeval genesis;
2154         struct thread_data *td;
2155         unsigned long spent;
2156         int i, todo, nr_running, m_rate, t_rate;
2157
2158         printf("Starting %d threads\n", thread_number);
2159         fflush(stdout);
2160
2161         signal(SIGINT, sig_handler);
2162
2163         todo = thread_number;
2164         nr_running = 0;
2165         m_rate = t_rate = 0;
2166
2167         for (i = 0; i < thread_number; i++) {
2168                 td = &threads[i];
2169
2170                 if (!td->create_serialize)
2171                         continue;
2172
2173                 /*
2174                  * do file setup here so it happens sequentially,
2175                  * we don't want X number of threads getting their
2176                  * client data interspersed on disk
2177                  */
2178                 if (setup_file(td)) {
2179                         td_set_runstate(td, TD_REAPED);
2180                         todo--;
2181                 }
2182         }
2183
2184         gettimeofday(&genesis, NULL);
2185
2186         while (todo) {
2187                 /*
2188                  * create threads (TD_NOT_CREATED -> TD_CREATED)
2189                  */
2190                 for (i = 0; i < thread_number; i++) {
2191                         td = &threads[i];
2192
2193                         if (td->runstate != TD_NOT_CREATED)
2194                                 continue;
2195
2196                         /*
2197                          * never got a chance to start, killed by other
2198                          * thread for some reason
2199                          */
2200                         if (td->terminate) {
2201                                 todo--;
2202                                 continue;
2203                         }
2204
2205                         if (td->start_delay) {
2206                                 spent = mtime_since_now(&genesis);
2207
2208                                 if (td->start_delay * 1000 > spent)
2209                                         continue;
2210                         }
2211
2212                         td_set_runstate(td, TD_CREATED);
2213                         check_str_update(td, nr_running, t_rate, m_rate);
2214                         sem_init(&startup_sem, 1, 1);
2215                         todo--;
2216
2217                         if (fork())
2218                                 sem_wait(&startup_sem);
2219                         else {
2220                                 thread_main(shm_id, i, argv);
2221                                 exit(0);
2222                         }
2223                 }
2224
2225                 /*
2226                  * start created threads (TD_CREATED -> TD_RUNNING)
2227                  */
2228                 for (i = 0; i < thread_number; i++) {
2229                         struct thread_data *td = &threads[i];
2230
2231                         if (td->runstate != TD_CREATED)
2232                                 continue;
2233
2234                         td_set_runstate(td, TD_RUNNING);
2235                         nr_running++;
2236                         m_rate += td->ratemin;
2237                         t_rate += td->rate;
2238                         check_str_update(td, nr_running, t_rate, m_rate);
2239                         sem_post(&td->mutex);
2240                 }
2241
2242                 for (i = 0; i < thread_number; i++) {
2243                         struct thread_data *td = &threads[i];
2244
2245                         if (td->runstate == TD_RUNNING)
2246                                 run_str[td->thread_number - 1] = '+';
2247                         else if (td->runstate == TD_VERIFYING)
2248                                 run_str[td->thread_number - 1] = 'V';
2249                         else
2250                                 continue;
2251
2252                         check_str_update(td, nr_running, t_rate, m_rate);
2253                 }
2254
2255                 reap_threads(&nr_running, &t_rate, &m_rate);
2256
2257                 if (todo)
2258                         usleep(100000);
2259         }
2260
2261         while (nr_running) {
2262                 reap_threads(&nr_running, &t_rate, &m_rate);
2263                 usleep(10000);
2264         }
2265 }
2266
2267 int setup_thread_area(void)
2268 {
2269         /*
2270          * 1024 is too much on some machines, scale max_jobs if
2271          * we get a failure that looks like too large a shm segment
2272          */
2273         do {
2274                 int s = max_jobs * sizeof(struct thread_data);
2275
2276                 shm_id = shmget(0, s, IPC_CREAT | 0600);
2277                 if (shm_id != -1)
2278                         break;
2279                 if (errno != EINVAL) {
2280                         perror("shmget");
2281                         break;
2282                 }
2283
2284                 max_jobs >>= 1;
2285         } while (max_jobs);
2286
2287         if (shm_id == -1)
2288                 return 1;
2289
2290         threads = shmat(shm_id, NULL, 0);
2291         if (threads == (void *) -1) {
2292                 perror("shmat");
2293                 return 1;
2294         }
2295
2296         atexit(free_shm);
2297         return 0;
2298 }
2299
2300 int main(int argc, char *argv[])
2301 {
2302         static unsigned long max_run[2], min_run[2];
2303         static unsigned long max_bw[2], min_bw[2];
2304         static unsigned long io_mb[2], agg[2];
2305         int i;
2306
2307         if (setup_thread_area())
2308                 return 1;
2309
2310         if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &def_thread.cpumask) == -1) {
2311                 perror("sched_getaffinity");
2312                 return 1;
2313         }
2314
2315         /*
2316          * fill globals
2317          */
2318         def_thread.ddir = DDIR_READ;
2319         def_thread.bs = DEF_BS;
2320         def_thread.min_bs = -1;
2321         def_thread.max_bs = -1;
2322         def_thread.odirect = DEF_ODIRECT;
2323         def_thread.ratecycle = DEF_RATE_CYCLE;
2324         def_thread.sequential = DEF_SEQUENTIAL;
2325         def_thread.timeout = DEF_TIMEOUT;
2326         def_thread.create_file = DEF_CREATE;
2327         def_thread.overwrite = DEF_OVERWRITE;
2328         def_thread.invalidate_cache = DEF_INVALIDATE;
2329         def_thread.sync_io = DEF_SYNCIO;
2330         def_thread.mem_type = MEM_MALLOC;
2331         def_thread.bw_avg_time = DEF_BWAVGTIME;
2332         def_thread.create_serialize = DEF_CREATE_SER;
2333         def_thread.create_fsync = DEF_CREATE_FSYNC;
2334         def_thread.loops = DEF_LOOPS;
2335         def_thread.verify = DEF_VERIFY;
2336
2337         i = parse_options(argc, argv);
2338
2339         if (!ini_file) {
2340                 printf("Need job file\n");
2341                 return 1;
2342         }
2343
2344         if (parse_jobs_ini(ini_file))
2345                 return 1;
2346
2347         if (!thread_number) {
2348                 printf("Nothing to do\n");
2349                 return 1;
2350         }
2351
2352         run_threads(argv);
2353
2354         min_bw[0] = min_run[0] = ~0UL;
2355         min_bw[1] = min_run[1] = ~0UL;
2356         io_mb[0] = io_mb[1] = 0;
2357         agg[0] = agg[1] = 0;
2358         for (i = 0; i < thread_number; i++) {
2359                 struct thread_data *td = &threads[i];
2360                 unsigned long bw = 0;
2361
2362                 if (!td->error) {
2363                         if (td->runtime < min_run[td->ddir])
2364                                 min_run[td->ddir] = td->runtime;
2365                         if (td->runtime > max_run[td->ddir])
2366                                 max_run[td->ddir] = td->runtime;
2367
2368                         if (td->runtime)
2369                                 bw = td->io_kb * 1024 / td->runtime;
2370                         if (bw < min_bw[td->ddir])
2371                                 min_bw[td->ddir] = bw;
2372                         if (bw > max_bw[td->ddir])
2373                                 max_bw[td->ddir] = bw;
2374
2375                         io_mb[td->ddir] += td->io_kb >> 10;
2376                 }
2377
2378                 show_thread_status(td);
2379         }
2380
2381         if (max_run[0])
2382                 agg[0] = io_mb[0] * 1024 * 1000 / max_run[0];
2383         if (max_run[1])
2384                 agg[1] = io_mb[1] * 1024 * 1000 / max_run[1];
2385
2386         printf("\nRun status:\n");
2387         if (max_run[DDIR_READ])
2388                 printf("   READ: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[0], agg[0], min_bw[0], max_bw[0], min_run[0], max_run[0]);
2389         if (max_run[DDIR_WRITE])
2390                 printf("  WRITE: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[1], agg[1], min_bw[1], max_bw[1], min_run[1], max_run[1]);
2391
2392         return 0;
2393 }