fio.c

   1 /*
   2  * fio - the flexible io tester
   3  *
   4  * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 2 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <fcntl.h>
  25 #include <string.h>
  26 #include <errno.h>
  27 #include <signal.h>
  28 #include <time.h>
  29 #include <ctype.h>
  30 #include <sched.h>
  31 #include <libaio.h>
  32 #include <math.h>
  33 #include <limits.h>
  34 #include <sys/time.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <sys/wait.h>
  38 #include <semaphore.h>
  39 #include <sys/ipc.h>
  40 #include <sys/shm.h>
  41 #include <asm/unistd.h>
  42 #include <asm/types.h>
  43
  44 #include "list.h"
  45 #include "md5.h"
  46
  47 #define MAX_JOBS        (1024)
  48
  49 /*
  50  * assume we don't have _get either, if _set isn't defined
  51  */
  52 #ifndef __NR_ioprio_set
  53 #if defined(__i386__)
  54 #define __NR_ioprio_set         289
  55 #define __NR_ioprio_get         290
  56 #elif defined(__powerpc__) || defined(__powerpc64__)
  57 #define __NR_ioprio_set         273
  58 #define __NR_ioprio_get         274
  59 #elif defined(__x86_64__)
  60 #define __NR_ioprio_set         251
  61 #define __NR_ioprio_get         252
  62 #elif defined(__ia64__)
  63 #define __NR_ioprio_set         1274
  64 #define __NR_ioprio_get         1275
  65 #elif defined(__alpha__)
  66 #define __NR_ioprio_set         442
  67 #define __NR_ioprio_get         443
  68 #elif defined(__s390x__) || defined(__s390__)
  69 #define __NR_ioprio_set         282
  70 #define __NR_ioprio_get         283
  71 #else
  72 #error "Unsupported arch"
  73 #endif
  74 #endif
  75
  76 #ifndef __NR_fadvise64
  77 #if defined(__i386__)
  78 #define __NR_fadvise64          250
  79 #elif defined(__powerpc__) || defined(__powerpc64__)
  80 #define __NR_fadvise64          233
  81 #elif defined(__x86_64__)
  82 #define __NR_fadvise64          221
  83 #elif defined(__ia64__)
  84 #define __NR_fadvise64          1234
  85 #elif defined(__alpha__)
  86 #define __NR_fadvise64          413
  87 #elif defined(__s390x__) || defined(__s390__)
  88 #define __NR_fadvise64          253
  89 #else
  90 #error "Unsupported arch"
  91 #endif
  92 #endif
  93
  94 static int ioprio_set(int which, int who, int ioprio)
  95 {
  96         return syscall(__NR_ioprio_set, which, who, ioprio);
  97 }
  98
  99 /*
 100  * we want fadvise64 really, but it's so tangled... later
 101  */
 102 static int fadvise(int fd, loff_t offset, size_t len, int advice)
 103 {
 104 #if 0
 105         return syscall(__NR_fadvise64, fd, offset, offset >> 32, len, advice);
 106 #else
 107         return posix_fadvise(fd, (off_t) offset, len, advice);
 108 #endif
 109 }
 110
 111 enum {
 112         IOPRIO_WHO_PROCESS = 1,
 113         IOPRIO_WHO_PGRP,
 114         IOPRIO_WHO_USER,
 115 };
 116
 117 #define IOPRIO_CLASS_SHIFT      13
 118
 119 #define MASK    (4095)
 120
 121 #define DEF_BS          (4096)
 122 #define DEF_TIMEOUT     (30)
 123 #define DEF_RATE_CYCLE  (1000)
 124 #define DEF_ODIRECT     (1)
 125 #define DEF_SEQUENTIAL  (1)
 126 #define DEF_RAND_REPEAT (1)
 127 #define DEF_OVERWRITE   (0)
 128 #define DEF_CREATE      (1)
 129 #define DEF_INVALIDATE  (1)
 130 #define DEF_SYNCIO      (0)
 131 #define DEF_RANDSEED    (0xb1899bedUL)
 132 #define DEF_BWAVGTIME   (500)
 133 #define DEF_CREATE_SER  (1)
 134 #define DEF_CREATE_FSYNC        (1)
 135 #define DEF_LOOPS       (1)
 136 #define DEF_VERIFY      (0)
 137
 138 #define ALIGN(buf)      (char *) (((unsigned long) (buf) + MASK) & ~(MASK))
 139
 140 static int repeatable = DEF_RAND_REPEAT;
 141 static int rate_quit = 1;
 142 static int write_lat_log;
 143 static int write_bw_log;
 144 static int exitall_on_terminate;
 145
 146 static int thread_number;
 147 static char *ini_file;
 148
 149 static int max_jobs = MAX_JOBS;
 150
 151 static char run_str[MAX_JOBS + 1];
 152
 153 static int shm_id;
 154
 155 enum {
 156         DDIR_READ = 0,
 157         DDIR_WRITE,
 158 };
 159
 160 /*
 161  * thread life cycle
 162  */
 163 enum {
 164         TD_NOT_CREATED = 0,
 165         TD_CREATED,
 166         TD_RUNNING,
 167         TD_VERIFYING,
 168         TD_EXITED,
 169         TD_REAPED,
 170 };
 171
 172 enum {
 173         MEM_MALLOC,
 174         MEM_SHM,
 175 };
 176
 177 /*
 178  * The io unit
 179  */
 180 struct io_u {
 181         struct iocb iocb;
 182         struct timeval start_time;
 183         struct timeval issue_time;
 184
 185         char *buf;
 186         unsigned int buflen;
 187         unsigned long long offset;
 188
 189         struct list_head list;
 190 };
 191
 192 struct io_stat {
 193         unsigned long val;
 194         unsigned long val_sq;
 195         unsigned long max_val;
 196         unsigned long min_val;
 197         unsigned long samples;
 198 };
 199
 200 struct io_sample {
 201         unsigned long time;
 202         unsigned long val;
 203 };
 204
 205 struct io_log {
 206         unsigned long nr_samples;
 207         unsigned long max_samples;
 208         struct io_sample *log;
 209 };
 210
 211 #define FIO_HDR_MAGIC   0xf00baaef
 212
 213 struct verify_header {
 214         unsigned int fio_magic;
 215         unsigned int len;
 216         char md5_digest[MD5_HASH_WORDS * 4];
 217 };
 218
 219 #define td_read(td)             ((td)->ddir == DDIR_READ)
 220 #define should_fsync(td)        (!td_read(td) && !(td)->odirect)
 221
 222 struct thread_data {
 223         char file_name[256];
 224         int thread_number;
 225         int error;
 226         int fd;
 227         pid_t pid;
 228         char *orig_buffer;
 229         volatile int terminate;
 230         volatile int runstate;
 231         volatile int old_runstate;
 232         unsigned int ddir;
 233         unsigned int ioprio;
 234         unsigned int sequential;
 235         unsigned int bs;
 236         unsigned int min_bs;
 237         unsigned int max_bs;
 238         unsigned int odirect;
 239         unsigned int thinktime;
 240         unsigned int fsync_blocks;
 241         unsigned int start_delay;
 242         unsigned int timeout;
 243         unsigned int use_aio;
 244         unsigned int create_file;
 245         unsigned int overwrite;
 246         unsigned int invalidate_cache;
 247         unsigned int bw_avg_time;
 248         unsigned int create_serialize;
 249         unsigned int create_fsync;
 250         unsigned int loops;
 251         unsigned long long file_size;
 252         unsigned long long file_offset;
 253         unsigned int sync_io;
 254         unsigned int mem_type;
 255         unsigned int verify;
 256         cpu_set_t cpumask;
 257
 258         struct drand48_data bsrange_state;
 259         struct drand48_data verify_state;
 260
 261         int shm_id;
 262
 263         off_t cur_off;
 264
 265         io_context_t aio_ctx;
 266         unsigned int aio_depth;
 267         struct io_event *aio_events;
 268
 269         unsigned int cur_depth;
 270         struct list_head io_u_freelist;
 271         struct list_head io_u_busylist;
 272
 273         unsigned int rate;
 274         unsigned int ratemin;
 275         unsigned int ratecycle;
 276         unsigned long rate_usec_cycle;
 277         long rate_pending_usleep;
 278         unsigned long rate_kb;
 279         struct timeval lastrate;
 280
 281         unsigned long runtime;          /* sec */
 282         unsigned long kb;
 283         unsigned long io_blocks;
 284         unsigned long io_kb;
 285         unsigned long this_io_kb;
 286         unsigned long last_kb;
 287         sem_t mutex;
 288         struct drand48_data random_state;
 289
 290         /*
 291          * bandwidth and latency stats
 292          */
 293         struct io_stat clat_stat;               /* completion latency */
 294         struct io_stat slat_stat;               /* submission latency */
 295
 296         struct io_stat bw_stat;                 /* bandwidth stats */
 297         unsigned long stat_io_kb;
 298         struct timeval stat_sample_time;
 299
 300         struct io_log *lat_log;
 301         struct io_log *bw_log;
 302
 303         struct timeval start;
 304 };
 305
 306 static struct thread_data *threads;
 307 static struct thread_data def_thread;
 308
 309 static sem_t startup_sem;
 310
 311 static void sig_handler(int sig)
 312 {
 313         int i;
 314
 315         for (i = 0; i < thread_number; i++) {
 316                 struct thread_data *td = &threads[i];
 317
 318                 td->terminate = 1;
 319                 td->start_delay = 0;
 320         }
 321 }
 322
 323 static int init_random_state(struct thread_data *td)
 324 {
 325         unsigned long seed;
 326         int fd;
 327
 328         fd = open("/dev/random", O_RDONLY);
 329         if (fd == -1) {
 330                 td->error = errno;
 331                 return 1;
 332         }
 333
 334         if (read(fd, &seed, sizeof(seed)) < (int) sizeof(seed)) {
 335                 td->error = EIO;
 336                 close(fd);
 337                 return 1;
 338         }
 339
 340         close(fd);
 341
 342         srand48_r(seed, &td->bsrange_state);
 343         srand48_r(seed, &td->verify_state);
 344
 345         if (td->sequential)
 346                 return 0;
 347
 348         if (repeatable)
 349                 seed = DEF_RANDSEED;
 350
 351         srand48_r(seed, &td->random_state);
 352         return 0;
 353 }
 354
 355 static unsigned long utime_since(struct timeval *s, struct timeval *e)
 356 {
 357         double sec, usec;
 358
 359         sec = e->tv_sec - s->tv_sec;
 360         usec = e->tv_usec - s->tv_usec;
 361         if (sec > 0 && usec < 0) {
 362                 sec--;
 363                 usec += 1000000;
 364         }
 365
 366         sec *= (double) 1000000;
 367
 368         return sec + usec;
 369 }
 370
 371 static unsigned long mtime_since(struct timeval *s, struct timeval *e)
 372 {
 373         double sec, usec;
 374
 375         sec = e->tv_sec - s->tv_sec;
 376         usec = e->tv_usec - s->tv_usec;
 377         if (sec > 0 && usec < 0) {
 378                 sec--;
 379                 usec += 1000000;
 380         }
 381
 382         sec *= (double) 1000;
 383         usec /= (double) 1000;
 384
 385         return sec + usec;
 386 }
 387
 388 static unsigned long mtime_since_now(struct timeval *s)
 389 {
 390         struct timeval t;
 391
 392         gettimeofday(&t, NULL);
 393         return mtime_since(s, &t);
 394 }
 395
 396 static inline unsigned long msec_now(struct timeval *s)
 397 {
 398         return s->tv_sec * 1000 + s->tv_usec / 1000;
 399 }
 400
 401 static unsigned long long get_next_offset(struct thread_data *td)
 402 {
 403         unsigned long long kb;
 404         long r;
 405
 406         if (!td->sequential) {
 407                 lrand48_r(&td->random_state, &r);
 408                 kb = (1+(double) (td->kb-1) * r / (RAND_MAX+1.0));
 409         } else
 410                 kb = td->last_kb;
 411
 412         return (kb << 10) + td->file_offset;
 413 }
 414
 415 static unsigned int get_next_buflen(struct thread_data *td)
 416 {
 417         unsigned int buflen;
 418         long r;
 419
 420         if (td->min_bs == td->max_bs)
 421                 buflen = td->min_bs;
 422         else {
 423                 lrand48_r(&td->bsrange_state, &r);
 424                 buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0));
 425                 buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1);
 426         }
 427
 428         if (buflen > ((td->kb - td->this_io_kb) << 10))
 429                 buflen = (td->kb - td->this_io_kb) << 10;
 430
 431         td->last_kb += buflen >> 10;
 432         return buflen;
 433 }
 434
 435 static inline void add_stat_sample(struct thread_data *td, struct io_stat *is,
 436                                    unsigned long val)
 437 {
 438         if (val > is->max_val)
 439                 is->max_val = val;
 440         if (val < is->min_val)
 441                 is->min_val = val;
 442
 443         is->val += val;
 444         is->val_sq += val * val;
 445         is->samples++;
 446 }
 447
 448 static void add_log_sample(struct thread_data *td, struct io_log *log,
 449                            unsigned long val)
 450 {
 451         if (log->nr_samples == log->max_samples) {
 452                 int new_size = sizeof(struct io_sample) * log->max_samples * 2;
 453
 454                 log->log = realloc(log->log, new_size);
 455                 log->max_samples <<= 1;
 456         }
 457
 458         log->log[log->nr_samples].val = val;
 459         log->log[log->nr_samples].time = mtime_since_now(&td->start);
 460         log->nr_samples++;
 461 }
 462
 463 static void add_clat_sample(struct thread_data *td, unsigned long msec)
 464 {
 465         add_stat_sample(td, &td->clat_stat, msec);
 466
 467         if (td->lat_log)
 468                 add_log_sample(td, td->lat_log, msec);
 469 }
 470
 471 static void add_slat_sample(struct thread_data *td, unsigned long msec)
 472 {
 473         add_stat_sample(td, &td->slat_stat, msec);
 474 }
 475
 476 static void add_bw_sample(struct thread_data *td, unsigned long msec)
 477 {
 478         unsigned long spent = mtime_since_now(&td->stat_sample_time);
 479         unsigned long rate;
 480
 481         if (spent < td->bw_avg_time)
 482                 return;
 483
 484         rate = ((td->this_io_kb - td->stat_io_kb) * 1024) / spent;
 485         add_stat_sample(td, &td->bw_stat, rate);
 486
 487         if (td->bw_log)
 488                 add_log_sample(td, td->bw_log, rate);
 489
 490         gettimeofday(&td->stat_sample_time, NULL);
 491         td->stat_io_kb = td->this_io_kb;
 492 }
 493
 494 static void usec_sleep(int usec)
 495 {
 496         struct timespec req = { .tv_sec = 0, .tv_nsec = usec * 1000 };
 497         struct timespec rem;
 498
 499         do {
 500                 rem.tv_sec = rem.tv_nsec = 0;
 501                 nanosleep(&req, &rem);
 502                 if (!rem.tv_nsec)
 503                         break;
 504
 505                 req.tv_nsec = rem.tv_nsec;
 506         } while (1);
 507 }
 508
 509 static void rate_throttle(struct thread_data *td, unsigned long time_spent,
 510                           unsigned int bytes)
 511 {
 512         unsigned long usec_cycle;
 513
 514         if (!td->rate)
 515                 return;
 516
 517         usec_cycle = td->rate_usec_cycle * (bytes / td->min_bs);
 518
 519         if (time_spent < usec_cycle) {
 520                 unsigned long s = usec_cycle - time_spent;
 521
 522                 td->rate_pending_usleep += s;
 523                 if (td->rate_pending_usleep >= 100000) {
 524                         usec_sleep(td->rate_pending_usleep);
 525                         td->rate_pending_usleep = 0;
 526                 }
 527         } else {
 528                 long overtime = time_spent - usec_cycle;
 529
 530                 td->rate_pending_usleep -= overtime;
 531         }
 532 }
 533
 534 static int check_min_rate(struct thread_data *td, struct timeval *now)
 535 {
 536         unsigned long spent;
 537         unsigned long rate;
 538
 539         /*
 540          * allow a 2 second settle period in the beginning
 541          */
 542         if (mtime_since(&td->start, now) < 2000)
 543                 return 0;
 544
 545         /*
 546          * if rate blocks is set, sample is running
 547          */
 548         if (td->rate_kb) {
 549                 spent = mtime_since(&td->lastrate, now);
 550                 if (spent < td->ratecycle)
 551                         return 0;
 552
 553                 rate = ((td->this_io_kb - td->rate_kb) * 1024) / spent;
 554                 if (rate < td->ratemin) {
 555                         printf("Client%d: min rate %d not met, got %ldKiB/sec\n", td->thread_number, td->ratemin, rate);
 556                         if (rate_quit)
 557                                 sig_handler(0);
 558                         return 1;
 559                 }
 560         }
 561
 562         td->rate_kb = td->this_io_kb;
 563         memcpy(&td->lastrate, now, sizeof(*now));
 564         return 0;
 565 }
 566
 567 static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
 568 {
 569         if (mtime_since(&td->start, t) >= td->timeout * 1000)
 570                 return 1;
 571
 572         return 0;
 573 }
 574
 575 static void fill_random_bytes(struct thread_data *td,
 576                               unsigned char *p, unsigned int len)
 577 {
 578         double r;
 579         int todo;
 580
 581         while (len) {
 582                 drand48_r(&td->verify_state, &r);
 583
 584                 /*
 585                  * lrand48_r seems to be broken and only fill the bottom
 586                  * 32-bits, even on 64-bit archs with 64-bit longs
 587                  */
 588                 todo = sizeof(r);
 589                 if (todo > len)
 590                         todo = len;
 591
 592                 memcpy(p, &r, todo);
 593
 594                 len -= todo;
 595                 p += todo;
 596         }
 597 }
 598
 599 static int verify_io_u(struct thread_data *td, struct io_u *io_u)
 600 {
 601         struct verify_header *hdr = (struct verify_header *) io_u->buf;
 602         unsigned char *p = (unsigned char *) io_u->buf;
 603         struct md5_ctx md5_ctx;
 604
 605         if (hdr->fio_magic != FIO_HDR_MAGIC)
 606                 return 1;
 607
 608         memset(&md5_ctx, 0, sizeof(md5_ctx));
 609         p += sizeof(*hdr);
 610         md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
 611
 612         return memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
 613 }
 614
 615 static int verify_io_us(struct thread_data *td, struct io_u *io_u, int *back)
 616 {
 617         struct verify_header *hdr;
 618         unsigned int left;
 619         struct io_u i;
 620         char *buf;
 621         off_t off;
 622
 623         if (back)
 624                 *back = 0;
 625
 626         left = io_u->buflen;
 627         buf = io_u->buf;
 628         off = io_u->offset;
 629         while (left) {
 630                 hdr = (struct verify_header *) buf;
 631                 i.buf = buf;
 632                 i.buflen = hdr->len;
 633
 634                 if (hdr->len > left) {
 635                         if (back)
 636                                 *back = left;
 637                         return 0;
 638                 }
 639
 640                 if (verify_io_u(td, &i)) {
 641                         printf("failed verify at offset %lu\n", (unsigned long) off);
 642                         td->error = EBADMSG;
 643                         return 1;
 644                 }
 645
 646                 buf += hdr->len;
 647                 left -= hdr->len;
 648                 off += hdr->len;
 649         }
 650
 651         return 0;
 652 }
 653
 654 /*
 655  * fill body of io_u->buf with random data and add a header with the
 656  * (eg) sha1sum of that data.
 657  */
 658 static void populate_io_u(struct thread_data *td, struct io_u *io_u)
 659 {
 660         struct md5_ctx md5_ctx;
 661         struct verify_header hdr;
 662         unsigned char *p = (unsigned char *) io_u->buf;
 663
 664         hdr.fio_magic = FIO_HDR_MAGIC;
 665         hdr.len = io_u->buflen;
 666         p += sizeof(hdr);
 667         fill_random_bytes(td, p, io_u->buflen - sizeof(hdr));
 668
 669         memset(&md5_ctx, 0, sizeof(md5_ctx));
 670         md5_update(&md5_ctx, p, io_u->buflen - sizeof(hdr));
 671         memcpy(hdr.md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
 672         memcpy(io_u->buf, &hdr, sizeof(hdr));
 673 }
 674
 675 static void put_io_u(struct thread_data *td, struct io_u *io_u)
 676 {
 677         list_del(&io_u->list);
 678         list_add(&io_u->list, &td->io_u_freelist);
 679         td->cur_depth--;
 680 }
 681
 682 #define queue_full(td)  (list_empty(&(td)->io_u_freelist))
 683
 684 static struct io_u *__get_io_u(struct thread_data *td)
 685 {
 686         struct io_u *io_u;
 687
 688         if (queue_full(td))
 689                 return NULL;
 690
 691         io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
 692         list_del(&io_u->list);
 693         list_add(&io_u->list, &td->io_u_busylist);
 694         td->cur_depth++;
 695         return io_u;
 696 }
 697
 698 static struct io_u *get_io_u(struct thread_data *td)
 699 {
 700         struct io_u *io_u;
 701
 702         io_u = __get_io_u(td);
 703         if (!io_u)
 704                 return NULL;
 705
 706         io_u->offset = get_next_offset(td);
 707         io_u->buflen = get_next_buflen(td);
 708         if (!io_u->buflen) {
 709                 put_io_u(td, io_u);
 710                 return NULL;
 711         }
 712
 713         if (!td_read(td) && td->verify)
 714                 populate_io_u(td, io_u);
 715
 716         if (td->use_aio) {
 717                 if (td_read(td))
 718                         io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
 719                 else
 720                         io_prep_pwrite(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
 721         }
 722
 723         gettimeofday(&io_u->start_time, NULL);
 724         return io_u;
 725 }
 726
 727 static inline void td_set_runstate(struct thread_data *td, int runstate)
 728 {
 729         td->old_runstate = td->runstate;
 730         td->runstate = runstate;
 731 }
 732
 733 static int do_sync_verify(struct thread_data *td)
 734 {
 735         struct timeval t;
 736         struct io_u *io_u = NULL;
 737         loff_t off = 0;
 738         int back, ret;
 739
 740         td_set_runstate(td, TD_VERIFYING);
 741
 742         io_u = __get_io_u(td);
 743
 744         if (lseek(td->fd, 0, SEEK_SET) < 0) {
 745                 td->error = errno;
 746                 goto out;
 747         }
 748
 749         if (!td->odirect) {
 750                 unsigned long size = td->kb << 10;
 751
 752                 if (fadvise(td->fd, 0, size, POSIX_FADV_DONTNEED) < 0) {
 753                         td->error = errno;
 754                         goto out;
 755                 }
 756         }
 757
 758         do {
 759                 if (td->terminate)
 760                         break;
 761
 762                 gettimeofday(&t, NULL);
 763                 if (runtime_exceeded(td, &t))
 764                         break;
 765
 766                 io_u->offset = off;
 767                 io_u->buflen = td->max_bs;
 768
 769                 ret = read(td->fd, io_u->buf, io_u->buflen);
 770                 if (ret < (int) io_u->buflen) {
 771                         if (ret == -1) {
 772                                 td->error = errno;
 773                                 break;
 774                         } else if (!ret)
 775                                 break;
 776                         else
 777                                 io_u->buflen = ret;
 778                 }
 779
 780                 if (verify_io_us(td, io_u, &back))
 781                         break;
 782
 783                 if (back) {
 784                         printf("will seek %d %d\n", ret, back);
 785                         ret -= back;
 786                         if (lseek(td->fd, -back, SEEK_CUR) < 0) {
 787                                 td->error = errno;
 788                                 break;
 789                         }
 790                 }
 791
 792                 off += ret;
 793         } while (1);
 794
 795 out:
 796         td_set_runstate(td, TD_RUNNING);
 797         put_io_u(td, io_u);
 798
 799         return td->error == 0;
 800 }
 801
 802 static void do_sync_io(struct thread_data *td)
 803 {
 804         unsigned long msec, usec;
 805         struct io_u *io_u = NULL;
 806         struct timeval e;
 807
 808         while (td->this_io_kb < td->kb) {
 809                 int ret;
 810
 811                 if (td->terminate)
 812                         break;
 813
 814                 io_u = get_io_u(td);
 815                 if (!io_u)
 816                         break;
 817
 818                 if (td->cur_off != io_u->offset) {
 819                         if (lseek(td->fd, io_u->offset, SEEK_SET) == -1) {
 820                                 td->error = errno;
 821                                 break;
 822                         }
 823                 }
 824
 825                 if (td_read(td))
 826                         ret = read(td->fd, io_u->buf, io_u->buflen);
 827                 else
 828                         ret = write(td->fd, io_u->buf, io_u->buflen);
 829
 830                 if (ret < (int) io_u->buflen) {
 831                         if (ret == -1)
 832                                 td->error = errno;
 833                         break;
 834                 }
 835
 836                 td->io_blocks++;
 837                 td->io_kb += io_u->buflen >> 10;
 838                 td->this_io_kb += io_u->buflen >> 10;
 839                 td->cur_off = io_u->offset + io_u->buflen;
 840
 841                 gettimeofday(&e, NULL);
 842
 843                 usec = utime_since(&io_u->start_time, &e);
 844
 845                 rate_throttle(td, usec, io_u->buflen);
 846
 847                 if (check_min_rate(td, &e)) {
 848                         td->error = ENODATA;
 849                         break;
 850                 }
 851
 852                 msec = usec / 1000;
 853                 add_clat_sample(td, msec);
 854                 add_bw_sample(td, msec);
 855
 856                 if (runtime_exceeded(td, &e))
 857                         break;
 858
 859                 put_io_u(td, io_u);
 860                 io_u = NULL;
 861
 862                 if (td->thinktime)
 863                         usec_sleep(td->thinktime);
 864
 865                 if (should_fsync(td) && td->fsync_blocks &&
 866                     (td->io_blocks % td->fsync_blocks) == 0)
 867                         fsync(td->fd);
 868         }
 869
 870         if (io_u)
 871                 put_io_u(td, io_u);
 872
 873         if (should_fsync(td))
 874                 fsync(td->fd);
 875 }
 876
 877 static int io_u_queue(struct thread_data *td, struct io_u *io_u)
 878 {
 879         struct iocb *iocb = &io_u->iocb;
 880         int ret;
 881
 882         do {
 883                 ret = io_submit(td->aio_ctx, 1, &iocb);
 884                 if (ret == 1)
 885                         return 0;
 886                 else if (ret == EAGAIN)
 887                         usleep(100);
 888                 else if (ret == EINTR)
 889                         continue;
 890                 else
 891                         break;
 892         } while (1);
 893
 894         return ret;
 895 }
 896
 897 #define iocb_time(iocb) ((unsigned long) (iocb)->data)
 898 #define ev_to_iou(ev)   (struct io_u *) ((unsigned long) (ev)->obj)
 899
 900 static int ios_completed(struct thread_data *td, int nr)
 901 {
 902         unsigned long msec;
 903         struct io_u *io_u;
 904         struct timeval e;
 905         int i, bytes_done;
 906
 907         gettimeofday(&e, NULL);
 908
 909         for (i = 0, bytes_done = 0; i < nr; i++) {
 910                 io_u = ev_to_iou(td->aio_events + i);
 911
 912                 td->io_blocks++;
 913                 td->io_kb += io_u->buflen >> 10;
 914                 td->this_io_kb += io_u->buflen >> 10;
 915
 916                 msec = mtime_since(&io_u->issue_time, &e);
 917
 918                 add_clat_sample(td, msec);
 919                 add_bw_sample(td, msec);
 920
 921                 bytes_done += io_u->buflen;
 922                 put_io_u(td, io_u);
 923         }
 924
 925         return bytes_done;
 926 }
 927
 928 static void cleanup_pending_aio(struct thread_data *td)
 929 {
 930         struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
 931         struct list_head *entry, *n;
 932         struct io_u *io_u;
 933         int r;
 934
 935         /*
 936          * get immediately available events, if any
 937          */
 938         r = io_getevents(td->aio_ctx, 0, td->cur_depth, td->aio_events, &ts);
 939         if (r > 0)
 940                 ios_completed(td, r);
 941
 942         /*
 943          * now cancel remaining active events
 944          */
 945         list_for_each_safe(entry, n, &td->io_u_busylist) {
 946                 io_u = list_entry(entry, struct io_u, list);
 947
 948                 r = io_cancel(td->aio_ctx, &io_u->iocb, td->aio_events);
 949                 if (!r)
 950                         put_io_u(td, io_u);
 951         }
 952
 953         if (td->cur_depth) {
 954                 r = io_getevents(td->aio_ctx, td->cur_depth, td->cur_depth, td->aio_events, NULL);
 955                 if (r > 0)
 956                         ios_completed(td, r);
 957         }
 958 }
 959
 960 static int async_do_verify(struct thread_data *td, struct io_u **io_u)
 961 {
 962         struct io_u *v_io_u = *io_u;
 963         int ret = 0;
 964
 965         if (v_io_u) {
 966                 ret = verify_io_us(td, v_io_u, NULL);
 967                 put_io_u(td, v_io_u);
 968                 *io_u = NULL;
 969         }
 970
 971         return ret;
 972 }
 973
 974 static int do_async_verify(struct thread_data *td)
 975 {
 976         struct timeval t;
 977         struct io_u *io_u, *v_io_u = NULL;
 978         struct verify_header *hdr;
 979         int ret, back;
 980         char *p;
 981
 982         td_set_runstate(td, TD_VERIFYING);
 983
 984         do {
 985                 if (td->terminate)
 986                         break;
 987
 988                 gettimeofday(&t, NULL);
 989                 if (runtime_exceeded(td, &t))
 990                         break;
 991
 992                 io_u = __get_io_u(td);
 993                 if (!io_u)
 994                         break;
 995
 996                 io_u->offset = td->cur_off;
 997                 io_u->buflen = td->max_bs;
 998
 999                 if (io_u->offset + io_u->buflen > (td->kb << 10)) {
1000                         io_u->buflen = (td->kb << 10) - io_u->offset;
1001                         if (!io_u->buflen) {
1002                                 put_io_u(td, io_u);
1003                                 break;
1004                         }
1005                 }
1006
1007                 io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
1008                 ret = io_u_queue(td, io_u);
1009                 if (ret) {
1010                         put_io_u(td, io_u);
1011                         td->error = ret;
1012                         break;
1013                 }
1014
1015                 /*
1016                  * we have one pending to verify, do that while the next
1017                  * we are doing io on the next one
1018                  */
1019                 if (async_do_verify(td, &v_io_u))
1020                         break;
1021
1022                 ret = io_getevents(td->aio_ctx, 1, 1, td->aio_events, NULL);
1023                 if (ret != 1) {
1024                         if (ret < 0)
1025                                 td->error = ret;
1026                         break;
1027                 }
1028
1029                 /*
1030                  * got our io_u to verify, find back offset so we can
1031                  * submit the next one before verifying this one
1032                  */
1033                 v_io_u = ev_to_iou(td->aio_events);
1034                 p = v_io_u->buf;
1035                 back = v_io_u->buflen;
1036                 do {
1037                         hdr = (struct verify_header *) p;
1038
1039                         if (hdr->len > back)
1040                                 break;
1041
1042                         back -= hdr->len;
1043                         p += hdr->len;
1044                 } while (back);
1045
1046                 td->cur_off += (v_io_u->buflen - back);
1047
1048                 /*
1049                  * if we can't submit more io, we need to verify now
1050                  */
1051                 if (queue_full(td) && async_do_verify(td, &v_io_u))
1052                         break;
1053
1054         } while (1);
1055
1056         async_do_verify(td, &v_io_u);
1057
1058         if (td->cur_depth)
1059                 cleanup_pending_aio(td);
1060
1061         td_set_runstate(td, TD_RUNNING);
1062         return td->error == 0;
1063 }
1064
1065 static void do_async_io(struct thread_data *td)
1066 {
1067         struct timeval s, e;
1068         unsigned long usec;
1069
1070         while (td->this_io_kb < td->kb) {
1071                 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
1072                 struct timespec *timeout;
1073                 int ret, min_evts = 0;
1074                 struct io_u *io_u;
1075                 unsigned int bytes_done;
1076
1077                 if (td->terminate)
1078                         break;
1079
1080                 io_u = get_io_u(td);
1081                 if (!io_u)
1082                         break;
1083
1084                 memcpy(&s, &io_u->start_time, sizeof(s));
1085
1086                 ret = io_u_queue(td, io_u);
1087                 if (ret) {
1088                         put_io_u(td, io_u);
1089                         td->error = ret;
1090                         break;
1091                 }
1092
1093                 gettimeofday(&io_u->issue_time, NULL);
1094                 add_slat_sample(td, mtime_since(&io_u->start_time, &io_u->issue_time));
1095                 if (td->cur_depth < td->aio_depth) {
1096                         timeout = &ts;
1097                         min_evts = 0;
1098                 } else {
1099                         timeout = NULL;
1100                         min_evts = 1;
1101                 }
1102
1103                 ret = io_getevents(td->aio_ctx, min_evts, td->cur_depth, td->aio_events, timeout);
1104                 if (ret < 0) {
1105                         td->error = errno;
1106                         break;
1107                 } else if (!ret)
1108                         continue;
1109
1110                 bytes_done = ios_completed(td, ret);
1111
1112                 /*
1113                  * the rate is batched for now, it should work for batches
1114                  * of completions except the very first one which may look
1115                  * a little bursty
1116                  */
1117                 gettimeofday(&e, NULL);
1118                 usec = utime_since(&s, &e);
1119
1120                 rate_throttle(td, usec, bytes_done);
1121
1122                 if (check_min_rate(td, &e)) {
1123                         td->error = ENODATA;
1124                         break;
1125                 }
1126
1127                 if (runtime_exceeded(td, &e))
1128                         break;
1129
1130                 if (td->thinktime)
1131                         usec_sleep(td->thinktime);
1132
1133                 if (should_fsync(td) && td->fsync_blocks &&
1134                     (td->io_blocks % td->fsync_blocks) == 0)
1135                         fsync(td->fd);
1136         }
1137
1138         if (td->cur_depth)
1139                 cleanup_pending_aio(td);
1140
1141         if (should_fsync(td))
1142                 fsync(td->fd);
1143 }
1144
1145 static void cleanup_aio(struct thread_data *td)
1146 {
1147         io_destroy(td->aio_ctx);
1148
1149         if (td->aio_events)
1150                 free(td->aio_events);
1151 }
1152
1153 static int init_aio(struct thread_data *td)
1154 {
1155         if (io_queue_init(td->aio_depth, &td->aio_ctx)) {
1156                 td->error = errno;
1157                 return 1;
1158         }
1159
1160         td->aio_events = malloc(td->aio_depth * sizeof(struct io_event));
1161         return 0;
1162 }
1163
1164 static void cleanup_io_u(struct thread_data *td)
1165 {
1166         struct list_head *entry, *n;
1167         struct io_u *io_u;
1168
1169         list_for_each_safe(entry, n, &td->io_u_freelist) {
1170                 io_u = list_entry(entry, struct io_u, list);
1171
1172                 list_del(&io_u->list);
1173                 free(io_u);
1174         }
1175
1176         if (td->mem_type == MEM_MALLOC)
1177                 free(td->orig_buffer);
1178         else if (td->mem_type == MEM_SHM) {
1179                 struct shmid_ds sbuf;
1180
1181                 shmdt(td->orig_buffer);
1182                 shmctl(td->shm_id, IPC_RMID, &sbuf);
1183         }
1184 }
1185
1186 static int init_io_u(struct thread_data *td)
1187 {
1188         struct io_u *io_u;
1189         int i, max_units, mem_size;
1190         char *p;
1191
1192         if (!td->use_aio)
1193                 max_units = 1;
1194         else
1195                 max_units = td->aio_depth;
1196
1197         mem_size = td->max_bs * max_units + MASK;
1198
1199         if (td->mem_type == MEM_MALLOC)
1200                 td->orig_buffer = malloc(mem_size);
1201         else if (td->mem_type == MEM_SHM) {
1202                 td->shm_id = shmget(IPC_PRIVATE, mem_size, IPC_CREAT | 0600);
1203                 if (td->shm_id < 0) {
1204                         td->error = errno;
1205                         perror("shmget");
1206                         return 1;
1207                 }
1208
1209                 td->orig_buffer = shmat(td->shm_id, NULL, 0);
1210                 if (td->orig_buffer == (void *) -1) {
1211                         td->error = errno;
1212                         perror("shmat");
1213                         return 1;
1214                 }
1215         }
1216
1217         INIT_LIST_HEAD(&td->io_u_freelist);
1218         INIT_LIST_HEAD(&td->io_u_busylist);
1219
1220         p = ALIGN(td->orig_buffer);
1221         for (i = 0; i < max_units; i++) {
1222                 io_u = malloc(sizeof(*io_u));
1223                 memset(io_u, 0, sizeof(*io_u));
1224                 INIT_LIST_HEAD(&io_u->list);
1225
1226                 io_u->buf = p + td->max_bs * i;
1227                 list_add(&io_u->list, &td->io_u_freelist);
1228         }
1229
1230         return 0;
1231 }
1232
1233 static void setup_log(struct io_log **log)
1234 {
1235         struct io_log *l = malloc(sizeof(*l));
1236
1237         l->nr_samples = 0;
1238         l->max_samples = 1024;
1239         l->log = malloc(l->max_samples * sizeof(struct io_sample));
1240         *log = l;
1241 }
1242
1243 static void finish_log(struct thread_data *td, struct io_log *log, char *name)
1244 {
1245         char file_name[128];
1246         FILE *f;
1247         int i;
1248
1249         sprintf(file_name, "client%d_%s.log", td->thread_number, name);
1250         f = fopen(file_name, "w");
1251         if (!f) {
1252                 perror("fopen log");
1253                 return;
1254         }
1255
1256         for (i = 0; i < log->nr_samples; i++)
1257                 fprintf(f, "%lu, %lu\n", log->log[i].time, log->log[i].val);
1258
1259         fclose(f);
1260         free(log->log);
1261         free(log);
1262 }
1263
1264 static int create_file(struct thread_data *td)
1265 {
1266         unsigned long long left;
1267         char *b;
1268         int r, bs;
1269
1270         /*
1271          * unless specifically asked for overwrite, let normal io extend it
1272          */
1273         if (!td_read(td) && !td->overwrite)
1274                 return 0;
1275
1276         if (!td->file_size) {
1277                 fprintf(stderr, "Need size for create\n");
1278                 td->error = EINVAL;
1279                 return 1;
1280         }
1281
1282         printf("Client%d: Laying out IO file\n", td->thread_number);
1283
1284         td->fd = open(td->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
1285         if (td->fd < 0) {
1286                 td->error = errno;
1287                 return 1;
1288         }
1289
1290         td->kb = td->file_size >> 10;
1291         b = malloc(td->max_bs);
1292         memset(b, 0, td->max_bs);
1293
1294         left = td->file_size;
1295         while (left) {
1296                 bs = td->max_bs;
1297                 if (bs > left)
1298                         bs = left;
1299
1300                 r = write(td->fd, b, bs);
1301
1302                 if (r == bs) {
1303                         left -= bs;
1304                         continue;
1305                 } else {
1306                         if (r < 0)
1307                                 td->error = errno;
1308                         else
1309                                 td->error = EIO;
1310
1311                         break;
1312                 }
1313         }
1314
1315         if (td->create_fsync)
1316                 fsync(td->fd);
1317
1318         close(td->fd);
1319         td->fd = -1;
1320         free(b);
1321         return 0;
1322 }
1323
1324 static int file_exists(struct thread_data *td)
1325 {
1326         struct stat st;
1327
1328         if (stat(td->file_name, &st) != -1)
1329                 return 1;
1330
1331         return errno != ENOENT;
1332 }
1333
1334 static int setup_file(struct thread_data *td)
1335 {
1336         struct stat st;
1337         int flags = 0;
1338
1339         if (!file_exists(td)) {
1340                 if (!td->create_file) {
1341                         td->error = ENOENT;
1342                         return 1;
1343                 }
1344                 if (create_file(td))
1345                         return 1;
1346         }
1347
1348         if (td->odirect)
1349                 flags |= O_DIRECT;
1350
1351         if (td_read(td))
1352                 td->fd = open(td->file_name, flags | O_RDONLY);
1353         else {
1354                 if (!td->overwrite)
1355                         flags |= O_TRUNC;
1356                 if (td->sync_io)
1357                         flags |= O_SYNC;
1358                 if (td->verify)
1359                         flags |= O_RDWR;
1360                 else
1361                         flags |= O_WRONLY;
1362
1363                 td->fd = open(td->file_name, flags | O_CREAT, 0600);
1364         }
1365
1366         if (td->fd == -1) {
1367                 td->error = errno;
1368                 return 1;
1369         }
1370
1371         if (td_read(td)) {
1372                 if (fstat(td->fd, &st) == -1) {
1373                         td->error = errno;
1374                         return 1;
1375                 }
1376
1377                 if (td->file_size > st.st_size)
1378                         st.st_size = td->file_size;
1379         } else {
1380                 if (!td->file_size)
1381                         td->file_size = 1024 * 1024 * 1024;
1382
1383                 st.st_size = td->file_size;
1384         }
1385
1386         td->kb = (st.st_size - td->file_offset) / 1024;
1387         if (!td->kb) {
1388                 fprintf(stderr, "Client%d: no io blocks\n", td->thread_number);
1389                 td->error = EINVAL;
1390                 return 1;
1391         }
1392
1393         if (td->invalidate_cache) {
1394                 if (fadvise(td->fd, 0, st.st_size, POSIX_FADV_DONTNEED) < 0) {
1395                         td->error = errno;
1396                         return 1;
1397                 }
1398         }
1399
1400         return 0;
1401 }
1402
1403 static void clear_io_state(struct thread_data *td)
1404 {
1405         td->cur_off = 0;
1406         td->last_kb = 0;
1407         td->stat_io_kb = 0;
1408         td->this_io_kb = 0;
1409 }
1410
1411 static void *thread_main(int shm_id, int offset, char *argv[])
1412 {
1413         struct thread_data *td;
1414         int ret = 1;
1415         void *data;
1416
1417         setsid();
1418
1419         data = shmat(shm_id, NULL, 0);
1420         if (data == (void *) -1) {
1421                 perror("shmat");
1422                 return NULL;
1423         }
1424
1425         td = data + offset * sizeof(struct thread_data);
1426         td->pid = getpid();
1427
1428         if (init_io_u(td))
1429                 goto err;
1430
1431         if (sched_setaffinity(td->pid, sizeof(td->cpumask), &td->cpumask) == -1) {
1432                 td->error = errno;
1433                 goto err;
1434         }
1435
1436         sprintf(argv[0], "fio%d", offset);
1437
1438         if (td->use_aio && init_aio(td))
1439                 goto err;
1440
1441         if (init_random_state(td))
1442                 goto err;
1443
1444         if (td->ioprio) {
1445                 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
1446                         td->error = errno;
1447                         goto err;
1448                 }
1449         }
1450
1451         sem_post(&startup_sem);
1452         sem_wait(&td->mutex);
1453
1454         if (!td->create_serialize && setup_file(td))
1455                 goto err;
1456
1457         gettimeofday(&td->start, NULL);
1458
1459         while (td->loops--) {
1460                 gettimeofday(&td->stat_sample_time, NULL);
1461
1462                 if (td->ratemin)
1463                         memcpy(&td->lastrate, &td->stat_sample_time, sizeof(td->lastrate));
1464
1465                 clear_io_state(td);
1466
1467                 if (!td->use_aio) {
1468                         do_sync_io(td);
1469
1470                         if (!td->verify)
1471                                 continue;
1472
1473                         clear_io_state(td);
1474                         if (!do_sync_verify(td))
1475                                 break;
1476                 } else {
1477                         do_async_io(td);
1478
1479                         if (!td->verify)
1480                                 continue;
1481
1482                         clear_io_state(td);
1483                         if (!do_async_verify(td))
1484                                 break;
1485                 }
1486         }
1487
1488         td->runtime = mtime_since_now(&td->start);
1489         ret = 0;
1490
1491         if (td->bw_log)
1492                 finish_log(td, td->bw_log, "bw");
1493         if (td->lat_log)
1494                 finish_log(td, td->lat_log, "lat");
1495
1496         if (exitall_on_terminate)
1497                 sig_handler(0);
1498
1499 err:
1500         if (td->fd != -1) {
1501                 close(td->fd);
1502                 td->fd = -1;
1503         }
1504         if (td->use_aio)
1505                 cleanup_aio(td);
1506         cleanup_io_u(td);
1507         if (ret) {
1508                 sem_post(&startup_sem);
1509                 sem_wait(&td->mutex);
1510         }
1511         td_set_runstate(td, TD_EXITED);
1512         shmdt(data);
1513         return NULL;
1514 }
1515
1516 static void free_shm(void)
1517 {
1518         struct shmid_ds sbuf;
1519
1520         if (threads) {
1521                 shmdt(threads);
1522                 threads = NULL;
1523                 shmctl(shm_id, IPC_RMID, &sbuf);
1524         }
1525 }
1526
1527 static int calc_lat(struct io_stat *is, unsigned long *min, unsigned long *max,
1528                     double *mean, double *dev)
1529 {
1530         double n;
1531
1532         if (is->samples == 0)
1533                 return 0;
1534
1535         *min = is->min_val;
1536         *max = is->max_val;
1537
1538         n = (double) is->samples;
1539         *mean = (double) is->val / n;
1540         *dev = sqrt(((double) is->val_sq - (*mean * *mean) / n) / (n - 1));
1541         return 1;
1542 }
1543
1544 static void show_thread_status(struct thread_data *td)
1545 {
1546         int prio, prio_class;
1547         unsigned long min, max, bw = 0;
1548         double mean, dev;
1549
1550         if (!td->io_kb && !td->error)
1551                 return;
1552
1553         if (td->runtime)
1554                 bw = td->io_kb * 1024 / td->runtime;
1555
1556         prio = td->ioprio & 0xff;
1557         prio_class = td->ioprio >> IOPRIO_CLASS_SHIFT;
1558
1559         printf("Client%d: err=%2d, io=%6luMiB, bw=%6luKiB/s, runt=%6lumsec\n", td->thread_number, td->error, td->io_kb >> 10, bw, td->runtime);
1560
1561         if (calc_lat(&td->slat_stat, &min, &max, &mean, &dev))
1562                 printf("  slat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1563         if (calc_lat(&td->clat_stat, &min, &max, &mean, &dev))
1564                 printf("  clat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1565         if (calc_lat(&td->bw_stat, &min, &max, &mean, &dev))
1566                 printf("  bw (KiB/s) : min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1567 }
1568
1569 static int setup_rate(struct thread_data *td)
1570 {
1571         int nr_reads_per_sec;
1572
1573         if (!td->rate)
1574                 return 0;
1575
1576         if (td->rate < td->ratemin) {
1577                 fprintf(stderr, "min rate larger than nominal rate\n");
1578                 return -1;
1579         }
1580
1581         nr_reads_per_sec = td->rate * 1024 / td->min_bs;
1582         td->rate_usec_cycle = 1000000 / nr_reads_per_sec;
1583         td->rate_pending_usleep = 0;
1584         return 0;
1585 }
1586
1587 static struct thread_data *get_new_job(int global)
1588 {
1589         struct thread_data *td;
1590
1591         if (global)
1592                 return &def_thread;
1593         if (thread_number >= max_jobs)
1594                 return NULL;
1595
1596         td = &threads[thread_number++];
1597         memset(td, 0, sizeof(*td));
1598
1599         td->fd = -1;
1600         td->thread_number = thread_number;
1601
1602         td->ddir = def_thread.ddir;
1603         td->ioprio = def_thread.ioprio;
1604         td->sequential = def_thread.sequential;
1605         td->bs = def_thread.bs;
1606         td->min_bs = def_thread.min_bs;
1607         td->max_bs = def_thread.max_bs;
1608         td->odirect = def_thread.odirect;
1609         td->thinktime = def_thread.thinktime;
1610         td->fsync_blocks = def_thread.fsync_blocks;
1611         td->start_delay = def_thread.start_delay;
1612         td->timeout = def_thread.timeout;
1613         td->use_aio = def_thread.use_aio;
1614         td->create_file = def_thread.create_file;
1615         td->overwrite = def_thread.overwrite;
1616         td->invalidate_cache = def_thread.invalidate_cache;
1617         td->file_size = def_thread.file_size;
1618         td->file_offset = def_thread.file_offset;
1619         td->rate = def_thread.rate;
1620         td->ratemin = def_thread.ratemin;
1621         td->ratecycle = def_thread.ratecycle;
1622         td->aio_depth = def_thread.aio_depth;
1623         td->sync_io = def_thread.sync_io;
1624         td->mem_type = def_thread.mem_type;
1625         td->bw_avg_time = def_thread.bw_avg_time;
1626         td->create_serialize = def_thread.create_serialize;
1627         td->create_fsync = def_thread.create_fsync;
1628         td->loops = def_thread.loops;
1629         td->verify = def_thread.verify;
1630         memcpy(&td->cpumask, &def_thread.cpumask, sizeof(td->cpumask));
1631
1632         return td;
1633 }
1634
1635 static void put_job(struct thread_data *td)
1636 {
1637         memset(&threads[td->thread_number - 1], 0, sizeof(*td));
1638         thread_number--;
1639 }
1640
1641 static int add_job(struct thread_data *td, const char *filename, int prioclass,
1642                    int prio)
1643 {
1644         if (td == &def_thread)
1645                 return 0;
1646
1647         strcpy(td->file_name, filename);
1648         sem_init(&td->mutex, 1, 0);
1649         td->ioprio = (prioclass << IOPRIO_CLASS_SHIFT) | prio;
1650
1651         td->clat_stat.min_val = ULONG_MAX;
1652         td->slat_stat.min_val = ULONG_MAX;
1653         td->bw_stat.min_val = ULONG_MAX;
1654
1655         run_str[td->thread_number - 1] = 'P';
1656
1657         if (td->use_aio && !td->aio_depth)
1658                 td->aio_depth = 1;
1659
1660         if (td->min_bs == -1)
1661                 td->min_bs = td->bs;
1662         if (td->max_bs == -1)
1663                 td->max_bs = td->bs;
1664         if (td_read(td) || !td->sequential)
1665                 td->verify = 0;
1666
1667         if (setup_rate(td))
1668                 return -1;
1669
1670         if (write_lat_log)
1671                 setup_log(&td->lat_log);
1672         if (write_bw_log)
1673                 setup_log(&td->bw_log);
1674
1675         printf("Client%d: file=%s, rw=%d, prio=%d/%d, seq=%d, odir=%d, bs=%d-%d, rate=%d, aio=%d, aio_depth=%d\n", td->thread_number, filename, td->ddir, prioclass, prio, td->sequential, td->odirect, td->min_bs, td->max_bs, td->rate, td->use_aio, td->aio_depth);
1676         return 0;
1677 }
1678
1679 static void fill_cpu_mask(cpu_set_t cpumask, int cpu)
1680 {
1681         unsigned int i;
1682
1683         CPU_ZERO(&cpumask);
1684
1685         for (i = 0; i < sizeof(int) * 8; i++) {
1686                 if ((1 << i) & cpu)
1687                         CPU_SET(i, &cpumask);
1688         }
1689 }
1690
1691 unsigned long get_mult(char c)
1692 {
1693         switch (c) {
1694                 case 'k':
1695                 case 'K':
1696                         return 1024;
1697                 case 'm':
1698                 case 'M':
1699                         return 1024 * 1024;
1700                 case 'g':
1701                 case 'G':
1702                         return 1024 * 1024 * 1024;
1703                 default:
1704                         return 1;
1705         }
1706 }
1707
1708 /*
1709  * convert string after '=' into decimal value, noting any size suffix
1710  */
1711 static int str_cnv(char *p, unsigned long long *val)
1712 {
1713         char *str;
1714         int len;
1715
1716         str = strstr(p, "=");
1717         if (!str)
1718                 return 1;
1719
1720         str++;
1721         len = strlen(str);
1722
1723         *val = strtoul(str, NULL, 10);
1724         if (*val == ULONG_MAX && errno == ERANGE)
1725                 return 1;
1726
1727         *val *= get_mult(str[len - 2]);
1728         return 0;
1729 }
1730
1731 static int check_strcnv(char *p, char *name, unsigned long long *val)
1732 {
1733         if (!strstr(p, name))
1734                 return 1;
1735
1736         return str_cnv(p, val);
1737 }
1738
1739 static int check_str(char *p, char *name, char *option)
1740 {
1741         char *s = strstr(p, name);
1742
1743         if (!s)
1744                 return 1;
1745
1746         s += strlen(name);
1747         if (strstr(s, option))
1748                 return 0;
1749
1750         return 1;
1751 }
1752
1753 static int check_range(char *p, char *name, unsigned long *s, unsigned long *e)
1754 {
1755         char str[128];
1756         char s1, s2;
1757
1758         sprintf(str, "%s=%%lu%%c-%%lu%%c", name);
1759         if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1760                 *s *= get_mult(s1);
1761                 *e *= get_mult(s2);
1762                 return 0;
1763         }
1764
1765         sprintf(str, "%s = %%lu%%c-%%lu%%c", name);
1766         if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1767                 *s *= get_mult(s1);
1768                 *e *= get_mult(s2);
1769                 return 0;
1770         }
1771
1772         sprintf(str, "%s=%%lu-%%lu", name);
1773         if (sscanf(p, str, s, e) == 2)
1774                 return 0;
1775
1776         sprintf(str, "%s = %%lu-%%lu", name);
1777         if (sscanf(p, str, s, e) == 2)
1778                 return 0;
1779
1780         return 1;
1781
1782 }
1783
1784 static int check_int(char *p, char *name, unsigned int *val)
1785 {
1786         char str[128];
1787
1788         sprintf(str, "%s=%%d", name);
1789         if (sscanf(p, str, val) == 1)
1790                 return 0;
1791
1792         sprintf(str, "%s = %%d", name);
1793         if (sscanf(p, str, val) == 1)
1794                 return 0;
1795
1796         return 1;
1797 }
1798
1799 static int is_empty_or_comment(char *line)
1800 {
1801         unsigned int i;
1802
1803         for (i = 0; i < strlen(line); i++) {
1804                 if (line[i] == ';')
1805                         return 1;
1806                 if (!isspace(line[i]) && !iscntrl(line[i]))
1807                         return 0;
1808         }
1809
1810         return 1;
1811 }
1812
1813 static int parse_jobs_ini(char *file)
1814 {
1815         unsigned int prioclass, prio, cpu, global;
1816         unsigned long long ull;
1817         unsigned long ul1, ul2;
1818         struct thread_data *td;
1819         char *string, *name;
1820         fpos_t off;
1821         FILE *f;
1822         char *p;
1823
1824         f = fopen(file, "r");
1825         if (!f) {
1826                 perror("fopen");
1827                 return 1;
1828         }
1829
1830         string = malloc(4096);
1831         name = malloc(256);
1832
1833         while ((p = fgets(string, 4096, f)) != NULL) {
1834                 if (is_empty_or_comment(p))
1835                         continue;
1836                 if (sscanf(p, "[%s]", name) != 1)
1837                         continue;
1838
1839                 global = !strncmp(name, "global", 6);
1840
1841                 name[strlen(name) - 1] = '\0';
1842
1843                 td = get_new_job(global);
1844                 if (!td)
1845                         break;
1846
1847                 prioclass = 2;
1848                 prio = 4;
1849
1850                 fgetpos(f, &off);
1851                 while ((p = fgets(string, 4096, f)) != NULL) {
1852                         if (is_empty_or_comment(p))
1853                                 continue;
1854                         if (strstr(p, "["))
1855                                 break;
1856                         if (!check_int(p, "rw", &td->ddir)) {
1857                                 fgetpos(f, &off);
1858                                 continue;
1859                         }
1860                         if (!check_int(p, "prio", &prio)) {
1861                                 fgetpos(f, &off);
1862                                 continue;
1863                         }
1864                         if (!check_int(p, "prioclass", &prioclass)) {
1865                                 fgetpos(f, &off);
1866                                 continue;
1867                         }
1868                         if (!check_int(p, "direct", &td->odirect)) {
1869                                 fgetpos(f, &off);
1870                                 continue;
1871                         }
1872                         if (!check_int(p, "rate", &td->rate)) {
1873                                 fgetpos(f, &off);
1874                                 continue;
1875                         }
1876                         if (!check_int(p, "ratemin", &td->ratemin)) {
1877                                 fgetpos(f, &off);
1878                                 continue;
1879                         }
1880                         if (!check_int(p, "ratecycle", &td->ratecycle)) {
1881                                 fgetpos(f, &off);
1882                                 continue;
1883                         }
1884                         if (!check_int(p, "thinktime", &td->thinktime)) {
1885                                 fgetpos(f, &off);
1886                                 continue;
1887                         }
1888                         if (!check_int(p, "cpumask", &cpu)) {
1889                                 fill_cpu_mask(td->cpumask, cpu);
1890                                 fgetpos(f, &off);
1891                                 continue;
1892                         }
1893                         if (!check_int(p, "fsync", &td->fsync_blocks)) {
1894                                 fgetpos(f, &off);
1895                                 continue;
1896                         }
1897                         if (!check_int(p, "startdelay", &td->start_delay)) {
1898                                 fgetpos(f, &off);
1899                                 continue;
1900                         }
1901                         if (!check_int(p, "timeout", &td->timeout)) {
1902                                 fgetpos(f, &off);
1903                                 continue;
1904                         }
1905                         if (!check_int(p, "invalidate",&td->invalidate_cache)) {
1906                                 fgetpos(f, &off);
1907                                 continue;
1908                         }
1909                         if (!check_int(p, "aio_depth", &td->aio_depth)) {
1910                                 fgetpos(f, &off);
1911                                 continue;
1912                         }
1913                         if (!check_int(p, "sync", &td->sync_io)) {
1914                                 fgetpos(f, &off);
1915                                 continue;
1916                         }
1917                         if (!check_int(p, "bwavgtime", &td->bw_avg_time)) {
1918                                 fgetpos(f, &off);
1919                                 continue;
1920                         }
1921                         if (!check_int(p, "create_serialize", &td->create_serialize)) {
1922                                 fgetpos(f, &off);
1923                                 continue;
1924                         }
1925                         if (!check_int(p, "create_fsync", &td->create_fsync)) {
1926                                 fgetpos(f, &off);
1927                                 continue;
1928                         }
1929                         if (!check_int(p, "loops", &td->loops)) {
1930                                 fgetpos(f, &off);
1931                                 continue;
1932                         }
1933                         if (!check_int(p, "verify", &td->verify)) {
1934                                 fgetpos(f, &off);
1935                                 continue;
1936                         }
1937                         if (!check_range(p, "bsrange", &ul1, &ul2)) {
1938                                 td->min_bs = ul1;
1939                                 td->max_bs = ul2;
1940                                 fgetpos(f, &off);
1941                                 continue;
1942                         }
1943                         if (!check_strcnv(p, "bs", &ull)) {
1944                                 td->bs = ull;
1945                                 fgetpos(f, &off);
1946                                 continue;
1947                         }
1948                         if (!check_strcnv(p, "size", &td->file_size)) {
1949                                 fgetpos(f, &off);
1950                                 continue;
1951                         }
1952                         if (!check_strcnv(p, "offset", &td->file_offset)) {
1953                                 fgetpos(f, &off);
1954                                 continue;
1955                         }
1956                         if (!check_str(p, "mem", "malloc")) {
1957                                 td->mem_type = MEM_MALLOC;
1958                                 fgetpos(f, &off);
1959                                 continue;
1960                         }
1961                         if (!check_str(p, "mem", "shm")) {
1962                                 td->mem_type = MEM_SHM;
1963                                 fgetpos(f, &off);
1964                                 continue;
1965                         }
1966                         if (!strncmp(p, "sequential", 10)) {
1967                                 td->sequential = 1;
1968                                 fgetpos(f, &off);
1969                                 continue;
1970                         }
1971                         if (!strncmp(p, "random", 6)) {
1972                                 td->sequential = 0;
1973                                 fgetpos(f, &off);
1974                                 continue;
1975                         }
1976                         if (!strncmp(p, "aio", 3)) {
1977                                 td->use_aio = 1;
1978                                 fgetpos(f, &off);
1979                                 continue;
1980                         }
1981                         if (!strncmp(p, "create", 6)) {
1982                                 td->create_file = 1;
1983                                 fgetpos(f, &off);
1984                                 continue;
1985                         }
1986                         if (!strncmp(p, "overwrite", 9)) {
1987                                 td->overwrite = 1;
1988                                 fgetpos(f, &off);
1989                                 continue;
1990                         }
1991                         if (!strncmp(p, "exitall", 7)) {
1992                                 exitall_on_terminate = 1;
1993                                 fgetpos(f, &off);
1994                                 continue;
1995                         }
1996                         printf("Client%d: bad option %s\n",td->thread_number,p);
1997                 }
1998                 fsetpos(f, &off);
1999
2000                 if (add_job(td, name, prioclass, prio))
2001                         put_job(td);
2002         }
2003
2004         free(string);
2005         free(name);
2006         fclose(f);
2007         return 0;
2008 }
2009
2010 static int parse_options(int argc, char *argv[])
2011 {
2012         int i;
2013
2014         for (i = 1; i < argc; i++) {
2015                 char *parm = argv[i];
2016
2017                 if (parm[0] != '-')
2018                         break;
2019
2020                 parm++;
2021                 switch (*parm) {
2022                         case 's':
2023                                 parm++;
2024                                 def_thread.sequential = !!atoi(parm);
2025                                 break;
2026                         case 'b':
2027                                 parm++;
2028                                 def_thread.bs = atoi(parm);
2029                                 def_thread.bs <<= 10;
2030                                 if (!def_thread.bs) {
2031                                         printf("bad block size\n");
2032                                         def_thread.bs = DEF_BS;
2033                                 }
2034                                 break;
2035                         case 't':
2036                                 parm++;
2037                                 def_thread.timeout = atoi(parm);
2038                                 break;
2039                         case 'r':
2040                                 parm++;
2041                                 repeatable = !!atoi(parm);
2042                                 break;
2043                         case 'R':
2044                                 parm++;
2045                                 rate_quit = !!atoi(parm);
2046                                 break;
2047                         case 'o':
2048                                 parm++;
2049                                 def_thread.odirect = !!atoi(parm);
2050                                 break;
2051                         case 'f':
2052                                 if (i + 1 >= argc) {
2053                                         printf("-f needs file as arg\n");
2054                                         break;
2055                                 }
2056                                 ini_file = strdup(argv[i+1]);
2057                                 i++;
2058                                 break;
2059                         case 'l':
2060                                 write_lat_log = 1;
2061                                 break;
2062                         case 'w':
2063                                 write_bw_log = 1;
2064                                 break;
2065                         default:
2066                                 printf("bad option %s\n", argv[i]);
2067                                 break;
2068                 }
2069         }
2070
2071         return i;
2072 }
2073
2074 static void print_thread_status(struct thread_data *td, int nr_running,
2075                                 int t_rate, int m_rate)
2076 {
2077         printf("Threads now running: %d", nr_running);
2078         if (m_rate || t_rate)
2079                 printf(", commitrate %d/%dKiB/sec", t_rate, m_rate);
2080         printf(" : [%s]\r", run_str);
2081         fflush(stdout);
2082 }
2083
2084 static void check_str_update(struct thread_data *td, int n, int t, int m)
2085 {
2086         char c = run_str[td->thread_number - 1];
2087
2088         if (td->runstate == td->old_runstate)
2089                 return;
2090
2091         switch (td->runstate) {
2092                 case TD_REAPED:
2093                         c = '_';
2094                         break;
2095                 case TD_EXITED:
2096                         c = 'E';
2097                         break;
2098                 case TD_RUNNING:
2099                         c = '+';
2100                         break;
2101                 case TD_VERIFYING:
2102                         c = 'V';
2103                         break;
2104                 case TD_CREATED:
2105                         c = 'C';
2106                         break;
2107                 case TD_NOT_CREATED:
2108                         c = 'P';
2109                         break;
2110                 default:
2111                         printf("state %d\n", td->runstate);
2112         }
2113
2114         run_str[td->thread_number - 1] = c;
2115         print_thread_status(td, n, t, m);
2116         td->old_runstate = td->runstate;
2117 }
2118
2119 static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
2120 {
2121         int i;
2122
2123         /*
2124          * reap exited threads (TD_EXITED -> TD_REAPED)
2125          */
2126         for (i = 0; i < thread_number; i++) {
2127                 struct thread_data *td = &threads[i];
2128
2129                 check_str_update(td, *nr_running, *t_rate, *m_rate);
2130
2131                 if (td->runstate != TD_EXITED)
2132                         continue;
2133
2134                 td_set_runstate(td, TD_REAPED);
2135                 waitpid(td->pid, NULL, 0);
2136                 (*nr_running)--;
2137                 (*m_rate) -= td->ratemin;
2138                 (*t_rate) -= td->rate;
2139                 check_str_update(td, *nr_running, *t_rate, *m_rate);
2140
2141                 if (td->terminate)
2142                         continue;
2143         }
2144 }
2145
2146 static void run_threads(char *argv[])
2147 {
2148         struct timeval genesis;
2149         struct thread_data *td;
2150         unsigned long spent;
2151         int i, todo, nr_running, m_rate, t_rate;
2152
2153         printf("Starting %d threads\n", thread_number);
2154         fflush(stdout);
2155
2156         signal(SIGINT, sig_handler);
2157
2158         todo = thread_number;
2159         nr_running = 0;
2160         m_rate = t_rate = 0;
2161
2162         for (i = 0; i < thread_number; i++) {
2163                 td = &threads[i];
2164
2165                 if (!td->create_serialize)
2166                         continue;
2167
2168                 /*
2169                  * do file setup here so it happens sequentially,
2170                  * we don't want X number of threads getting their
2171                  * client data interspersed on disk
2172                  */
2173                 if (setup_file(td)) {
2174                         td_set_runstate(td, TD_REAPED);
2175                         todo--;
2176                 }
2177         }
2178
2179         gettimeofday(&genesis, NULL);
2180
2181         while (todo) {
2182                 /*
2183                  * create threads (TD_NOT_CREATED -> TD_CREATED)
2184                  */
2185                 for (i = 0; i < thread_number; i++) {
2186                         td = &threads[i];
2187
2188                         if (td->runstate != TD_NOT_CREATED)
2189                                 continue;
2190
2191                         /*
2192                          * never got a chance to start, killed by other
2193                          * thread for some reason
2194                          */
2195                         if (td->terminate) {
2196                                 todo--;
2197                                 continue;
2198                         }
2199
2200                         if (td->start_delay) {
2201                                 spent = mtime_since_now(&genesis);
2202
2203                                 if (td->start_delay * 1000 > spent)
2204                                         continue;
2205                         }
2206
2207                         td_set_runstate(td, TD_CREATED);
2208                         check_str_update(td, nr_running, t_rate, m_rate);
2209                         sem_init(&startup_sem, 1, 1);
2210                         todo--;
2211
2212                         if (fork())
2213                                 sem_wait(&startup_sem);
2214                         else {
2215                                 thread_main(shm_id, i, argv);
2216                                 exit(0);
2217                         }
2218                 }
2219
2220                 /*
2221                  * start created threads (TD_CREATED -> TD_RUNNING)
2222                  */
2223                 for (i = 0; i < thread_number; i++) {
2224                         struct thread_data *td = &threads[i];
2225
2226                         if (td->runstate != TD_CREATED)
2227                                 continue;
2228
2229                         td_set_runstate(td, TD_RUNNING);
2230                         nr_running++;
2231                         m_rate += td->ratemin;
2232                         t_rate += td->rate;
2233                         check_str_update(td, nr_running, t_rate, m_rate);
2234                         sem_post(&td->mutex);
2235                 }
2236
2237                 for (i = 0; i < thread_number; i++) {
2238                         struct thread_data *td = &threads[i];
2239
2240                         if (td->runstate == TD_RUNNING)
2241                                 run_str[td->thread_number - 1] = '+';
2242                         else if (td->runstate == TD_VERIFYING)
2243                                 run_str[td->thread_number - 1] = 'V';
2244                         else
2245                                 continue;
2246
2247                         check_str_update(td, nr_running, t_rate, m_rate);
2248                 }
2249
2250                 reap_threads(&nr_running, &t_rate, &m_rate);
2251
2252                 if (todo)
2253                         usleep(100000);
2254         }
2255
2256         while (nr_running) {
2257                 reap_threads(&nr_running, &t_rate, &m_rate);
2258                 usleep(10000);
2259         }
2260 }
2261
2262 int setup_thread_area(void)
2263 {
2264         /*
2265          * 1024 is too much on some machines, scale max_jobs if
2266          * we get a failure that looks like too large a shm segment
2267          */
2268         do {
2269                 int s = max_jobs * sizeof(struct thread_data);
2270
2271                 shm_id = shmget(0, s, IPC_CREAT | 0600);
2272                 if (shm_id != -1)
2273                         break;
2274                 if (errno != EINVAL) {
2275                         perror("shmget");
2276                         break;
2277                 }
2278
2279                 max_jobs >>= 1;
2280         } while (max_jobs);
2281
2282         if (shm_id == -1)
2283                 return 1;
2284
2285         threads = shmat(shm_id, NULL, 0);
2286         if (threads == (void *) -1) {
2287                 perror("shmat");
2288                 return 1;
2289         }
2290
2291         atexit(free_shm);
2292         return 0;
2293 }
2294
2295 int main(int argc, char *argv[])
2296 {
2297         static unsigned long max_run[2], min_run[2];
2298         static unsigned long max_bw[2], min_bw[2];
2299         static unsigned long io_mb[2], agg[2];
2300         int i;
2301
2302         if (setup_thread_area())
2303                 return 1;
2304
2305         if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &def_thread.cpumask) == -1) {
2306                 perror("sched_getaffinity");
2307                 return 1;
2308         }
2309
2310         /*
2311          * fill globals
2312          */
2313         def_thread.ddir = DDIR_READ;
2314         def_thread.bs = DEF_BS;
2315         def_thread.min_bs = -1;
2316         def_thread.max_bs = -1;
2317         def_thread.odirect = DEF_ODIRECT;
2318         def_thread.ratecycle = DEF_RATE_CYCLE;
2319         def_thread.sequential = DEF_SEQUENTIAL;
2320         def_thread.timeout = DEF_TIMEOUT;
2321         def_thread.create_file = DEF_CREATE;
2322         def_thread.overwrite = DEF_OVERWRITE;
2323         def_thread.invalidate_cache = DEF_INVALIDATE;
2324         def_thread.sync_io = DEF_SYNCIO;
2325         def_thread.mem_type = MEM_MALLOC;
2326         def_thread.bw_avg_time = DEF_BWAVGTIME;
2327         def_thread.create_serialize = DEF_CREATE_SER;
2328         def_thread.create_fsync = DEF_CREATE_FSYNC;
2329         def_thread.loops = DEF_LOOPS;
2330         def_thread.verify = DEF_VERIFY;
2331
2332         i = parse_options(argc, argv);
2333
2334         if (!ini_file) {
2335                 printf("Need job file\n");
2336                 return 1;
2337         }
2338
2339         if (parse_jobs_ini(ini_file))
2340                 return 1;
2341
2342         if (!thread_number) {
2343                 printf("Nothing to do\n");
2344                 return 1;
2345         }
2346
2347         run_threads(argv);
2348
2349         min_bw[0] = min_run[0] = ~0UL;
2350         min_bw[1] = min_run[1] = ~0UL;
2351         io_mb[0] = io_mb[1] = 0;
2352         agg[0] = agg[1] = 0;
2353         for (i = 0; i < thread_number; i++) {
2354                 struct thread_data *td = &threads[i];
2355                 unsigned long bw = 0;
2356
2357                 if (!td->error) {
2358                         if (td->runtime < min_run[td->ddir])
2359                                 min_run[td->ddir] = td->runtime;
2360                         if (td->runtime > max_run[td->ddir])
2361                                 max_run[td->ddir] = td->runtime;
2362
2363                         if (td->runtime)
2364                                 bw = td->io_kb * 1024 / td->runtime;
2365                         if (bw < min_bw[td->ddir])
2366                                 min_bw[td->ddir] = bw;
2367                         if (bw > max_bw[td->ddir])
2368                                 max_bw[td->ddir] = bw;
2369
2370                         io_mb[td->ddir] += td->io_kb >> 10;
2371                 }
2372
2373                 show_thread_status(td);
2374         }
2375
2376         if (max_run[0])
2377                 agg[0] = io_mb[0] * 1024 * 1000 / max_run[0];
2378         if (max_run[1])
2379                 agg[1] = io_mb[1] * 1024 * 1000 / max_run[1];
2380
2381         printf("\nRun status:\n");
2382         if (max_run[DDIR_READ])
2383                 printf("   READ: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[0], agg[0], min_bw[0], max_bw[0], min_run[0], max_run[0]);
2384         if (max_run[DDIR_WRITE])
2385                 printf("  WRITE: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[1], agg[1], min_bw[1], max_bw[1], min_run[1], max_run[1]);
2386
2387         return 0;
2388 }