fio.c

   1 /*
   2  * fio - the flexible io tester
   3  *
   4  * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 2 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <fcntl.h>
  25 #include <string.h>
  26 #include <errno.h>
  27 #include <signal.h>
  28 #include <time.h>
  29 #include <ctype.h>
  30 #include <sched.h>
  31 #include <libaio.h>
  32 #include <math.h>
  33 #include <limits.h>
  34 #include <sys/time.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <sys/wait.h>
  38 #include <semaphore.h>
  39 #include <sys/ipc.h>
  40 #include <sys/shm.h>
  41 #include <asm/unistd.h>
  42
  43 #define MAX_JOBS        (1024)
  44
  45 /*
  46  * assume we don't have _get either, if _set isn't defined
  47  */
  48 #ifndef __NR_ioprio_set
  49 #if defined(__i386__)
  50 #define __NR_ioprio_set         289
  51 #define __NR_ioprio_get         290
  52 #elif defined(__powerpc__) || defined(__powerpc64__)
  53 #define __NR_ioprio_set         273
  54 #define __NR_ioprio_get         274
  55 #elif defined(__x86_64__)
  56 #define __NR_ioprio_set         251
  57 #define __NR_ioprio_get         252
  58 #elif defined(__ia64__)
  59 #define __NR_ioprio_set         1274
  60 #define __NR_ioprio_get         1275
  61 #elif defined(__alpha__)
  62 #define __NR_ioprio_set         442
  63 #define __NR_ioprio_get         443
  64 #elif defined(__s390x__) || defined(__s390__)
  65 #define __NR_ioprio_set         282
  66 #define __NR_ioprio_get         283
  67 #else
  68 #error "Unsupported arch"
  69 #endif
  70 #endif
  71
  72 #ifndef __NR_fadvise64
  73 #if defined(__i386__)
  74 #define __NR_fadvise64          250
  75 #elif defined(__powerpc__) || defined(__powerpc64__)
  76 #define __NR_fadvise64          233
  77 #elif defined(__x86_64__)
  78 #define __NR_fadvise64          221
  79 #elif defined(__ia64__)
  80 #define __NR_fadvise64          1234
  81 #elif defined(__alpha__)
  82 #define __NR_fadvise64          413
  83 #elif defined(__s390x__) || defined(__s390__)
  84 #define __NR_fadvise64          253
  85 #else
  86 #error "Unsupported arch"
  87 #endif
  88 #endif
  89
  90 static int ioprio_set(int which, int who, int ioprio)
  91 {
  92         return syscall(__NR_ioprio_set, which, who, ioprio);
  93 }
  94
  95 /*
  96  * we want fadvise64 really, but it's so tangled... later
  97  */
  98 static int fadvise(int fd, loff_t offset, size_t len, int advice)
  99 {
 100 #if 0
 101         return syscall(__NR_fadvise64, fd, offset, offset >> 32, len, advice);
 102 #else
 103         return posix_fadvise(fd, (off_t) offset, len, advice);
 104 #endif
 105 }
 106
 107 enum {
 108         IOPRIO_WHO_PROCESS = 1,
 109         IOPRIO_WHO_PGRP,
 110         IOPRIO_WHO_USER,
 111 };
 112
 113 #define IOPRIO_CLASS_SHIFT      13
 114
 115 #define MASK    (4095)
 116
 117 #define DEF_BS          (4096)
 118 #define DEF_TIMEOUT     (30)
 119 #define DEF_RATE_CYCLE  (1000)
 120 #define DEF_ODIRECT     (1)
 121 #define DEF_SEQUENTIAL  (1)
 122 #define DEF_RAND_REPEAT (1)
 123 #define DEF_OVERWRITE   (0)
 124 #define DEF_CREATE      (1)
 125 #define DEF_INVALIDATE  (1)
 126
 127 #define ALIGN(buf)      (char *) (((unsigned long) (buf) + MASK) & ~(MASK))
 128
 129 static int repeatable = DEF_RAND_REPEAT;
 130 static int rate_quit = 1;
 131
 132 static int thread_number;
 133 static char *ini_file;
 134
 135 static int max_jobs = MAX_JOBS;
 136
 137 static int shm_id;
 138
 139 enum {
 140         DDIR_READ = 0,
 141         DDIR_WRITE,
 142 };
 143
 144 /*
 145  * thread life cycle
 146  */
 147 enum {
 148         TD_NOT_CREATED = 0,
 149         TD_CREATED,
 150         TD_STARTED,
 151         TD_EXITED,
 152         TD_REAPED,
 153 };
 154
 155 #define td_read(td)             ((td)->ddir == DDIR_READ)
 156 #define should_fsync(td)        (!td_read(td) && !(td)->odirect)
 157
 158 struct thread_data {
 159         char file_name[256];
 160         int thread_number;
 161         int error;
 162         int fd;
 163         pid_t pid;
 164         char *buf;
 165         volatile int terminate;
 166         volatile int runstate;
 167         unsigned int ddir;
 168         unsigned int ioprio;
 169         unsigned int sequential;
 170         unsigned int bs;
 171         unsigned int odirect;
 172         unsigned int delay_sleep;
 173         unsigned int fsync_blocks;
 174         unsigned int start_delay;
 175         unsigned int timeout;
 176         unsigned int use_aio;
 177         unsigned int create_file;
 178         unsigned int overwrite;
 179         unsigned int invalidate_cache;
 180         unsigned long long file_size;
 181         unsigned long long file_offset;
 182         cpu_set_t cpumask;
 183
 184         io_context_t *aio_ctx;
 185         struct iocb *aio_iocbs;
 186         unsigned int aio_depth;
 187         unsigned int aio_cur_depth;
 188         struct io_event *aio_events;
 189         char *aio_iocbs_status;
 190
 191         unsigned int rate;
 192         unsigned int ratemin;
 193         unsigned int ratecycle;
 194         unsigned long rate_usec_cycle;
 195         long rate_pending_usleep;
 196         unsigned long rate_blocks;
 197         struct timeval lastrate;
 198
 199         unsigned long max_latency;      /* msec */
 200         unsigned long min_latency;      /* msec */
 201         unsigned long runtime;          /* sec */
 202         unsigned long blocks;
 203         unsigned long io_blocks;
 204         unsigned long last_block;
 205         sem_t mutex;
 206         struct drand48_data random_state;
 207
 208         /*
 209          * bandwidth and latency stats
 210          */
 211         unsigned long stat_time;
 212         unsigned long stat_time_sq;
 213         unsigned long stat_time_samples;
 214         unsigned long stat_io_blocks;
 215         unsigned long stat_bw;
 216         unsigned long stat_bw_sq;
 217         unsigned long stat_bw_samples;
 218         struct timeval stat_sample_time;
 219
 220         struct timeval start;
 221 };
 222
 223 static struct thread_data *threads;
 224 static struct thread_data def_thread;
 225
 226 static sem_t startup_sem;
 227
 228 static void sig_handler(int sig)
 229 {
 230         int i;
 231
 232         for (i = 0; i < thread_number; i++) {
 233                 struct thread_data *td = &threads[i];
 234
 235                 td->terminate = 1;
 236                 td->start_delay = 0;
 237         }
 238 }
 239
 240 static int init_random_state(struct thread_data *td)
 241 {
 242         unsigned long seed = 123;
 243
 244         if (td->sequential)
 245                 return 0;
 246
 247         if (!repeatable) {
 248                 int fd = open("/dev/random", O_RDONLY);
 249
 250                 if (fd == -1) {
 251                         td->error = errno;
 252                         return 1;
 253                 }
 254
 255                 if (read(fd, &seed, sizeof(seed)) < (int) sizeof(seed)) {
 256                         td->error = EIO;
 257                         close(fd);
 258                         return 1;
 259                 }
 260
 261                 close(fd);
 262         }
 263
 264         srand48_r(seed, &td->random_state);
 265         return 0;
 266 }
 267
 268 static unsigned long utime_since(struct timeval *s, struct timeval *e)
 269 {
 270         double sec, usec;
 271
 272         sec = e->tv_sec - s->tv_sec;
 273         usec = e->tv_usec - s->tv_usec;
 274         if (sec > 0 && usec < 0) {
 275                 sec--;
 276                 usec += 1000000;
 277         }
 278
 279         sec *= (double) 1000000;
 280
 281         return sec + usec;
 282 }
 283
 284 static unsigned long mtime_since(struct timeval *s, struct timeval *e)
 285 {
 286         double sec, usec;
 287
 288         sec = e->tv_sec - s->tv_sec;
 289         usec = e->tv_usec - s->tv_usec;
 290         if (sec > 0 && usec < 0) {
 291                 sec--;
 292                 usec += 1000000;
 293         }
 294
 295         sec *= (double) 1000;
 296         usec /= (double) 1000;
 297
 298         return sec + usec;
 299 }
 300
 301 static unsigned long mtime_since_now(struct timeval *s)
 302 {
 303         struct timeval t;
 304
 305         gettimeofday(&t, NULL);
 306         return mtime_since(s, &t);
 307 }
 308
 309 static inline unsigned long msec_now(struct timeval *s)
 310 {
 311         return s->tv_sec * 1000 + s->tv_usec / 1000;
 312 }
 313
 314 static unsigned long get_next_offset(struct thread_data *td)
 315 {
 316         unsigned long b;
 317         long r;
 318
 319         if (!td->sequential) {
 320                 lrand48_r(&td->random_state, &r);
 321                 b = (1+(double) (td->blocks-1) * r / (RAND_MAX+1.0));
 322         } else {
 323                 b = td->last_block;
 324                 td->last_block++;
 325         }
 326
 327         return b * td->bs + td->file_offset;
 328 }
 329
 330 static void add_stat_sample(struct thread_data *td, unsigned long msec)
 331 {
 332         unsigned long spent;
 333
 334         td->stat_time += msec;
 335         td->stat_time_sq += msec * msec;
 336         td->stat_time_samples++;
 337
 338         spent = mtime_since_now(&td->stat_sample_time);
 339         if (spent >= 500) {
 340                 unsigned long rate = ((td->io_blocks - td->stat_io_blocks) * td->bs) / spent;
 341
 342                 td->stat_bw += rate;
 343                 td->stat_bw_sq += rate * rate;
 344                 gettimeofday(&td->stat_sample_time, NULL);
 345                 td->stat_io_blocks = td->io_blocks;
 346                 td->stat_bw_samples++;
 347         }
 348 }
 349
 350 static void usec_sleep(int usec)
 351 {
 352         struct timespec req = { .tv_sec = 0, .tv_nsec = usec * 1000 };
 353         struct timespec rem;
 354
 355         do {
 356                 rem.tv_sec = rem.tv_nsec = 0;
 357                 nanosleep(&req, &rem);
 358                 if (!rem.tv_nsec)
 359                         break;
 360
 361                 req.tv_nsec = rem.tv_nsec;
 362         } while (1);
 363 }
 364
 365 static void rate_throttle(struct thread_data *td, unsigned long time_spent)
 366 {
 367         if (!td->rate)
 368                 return;
 369
 370         if (time_spent < td->rate_usec_cycle) {
 371                 unsigned long s = td->rate_usec_cycle - time_spent;
 372
 373                 td->rate_pending_usleep += s;
 374                 if (td->rate_pending_usleep >= 100000) {
 375                         usec_sleep(td->rate_pending_usleep);
 376                         td->rate_pending_usleep = 0;
 377                 }
 378         } else {
 379                 long overtime = time_spent - td->rate_usec_cycle;
 380
 381                 td->rate_pending_usleep -= overtime;
 382         }
 383 }
 384
 385 static int check_min_rate(struct thread_data *td, struct timeval *now)
 386 {
 387         unsigned long spent;
 388         unsigned long rate;
 389
 390         /*
 391          * allow a 2 second settle period in the beginning
 392          */
 393         if (mtime_since(&td->start, now) < 2000)
 394                 return 0;
 395
 396         /*
 397          * if rate blocks is set, sample is running
 398          */
 399         if (td->rate_blocks) {
 400                 spent = mtime_since(&td->lastrate, now);
 401                 if (spent < td->ratecycle)
 402                         return 0;
 403
 404                 rate = ((td->io_blocks - td->rate_blocks) * td->bs) / spent;
 405                 if (rate < td->ratemin) {
 406                         printf("Client%d: min rate %d not met, got %ldKiB/sec\n", td->thread_number, td->ratemin, rate);
 407                         if (rate_quit)
 408                                 sig_handler(0);
 409                         return 1;
 410                 }
 411         }
 412
 413         td->rate_blocks = td->io_blocks;
 414         memcpy(&td->lastrate, now, sizeof(*now));
 415         return 0;
 416 }
 417
 418 static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
 419 {
 420         if (mtime_since(&td->start, t) >= td->timeout * 1000)
 421                 return 1;
 422
 423         return 0;
 424 }
 425
 426 static void do_sync_io(struct thread_data *td)
 427 {
 428         struct timeval s, e;
 429         unsigned long blocks, msec, usec;
 430
 431         for (blocks = 0; blocks < td->blocks; blocks++) {
 432                 off_t offset = get_next_offset(td);
 433                 int ret;
 434
 435                 if (td->terminate)
 436                         break;
 437
 438                 if (lseek(td->fd, offset, SEEK_SET) == -1) {
 439                         td->error = errno;
 440                         break;
 441                 }
 442
 443                 if (td->delay_sleep)
 444                         usec_sleep(td->delay_sleep);
 445
 446                 gettimeofday(&s, NULL);
 447
 448                 if (td_read(td))
 449                         ret = read(td->fd, td->buf, td->bs);
 450                 else
 451                         ret = write(td->fd, td->buf, td->bs);
 452
 453                 if (ret < (int) td->bs) {
 454                         if (ret == -1)
 455                                 td->error = errno;
 456                         break;
 457                 }
 458
 459                 td->io_blocks++;
 460
 461                 if (should_fsync(td) && td->fsync_blocks &&
 462                     (td->io_blocks % td->fsync_blocks) == 0)
 463                         fsync(td->fd);
 464
 465                 gettimeofday(&e, NULL);
 466
 467                 usec = utime_since(&s, &e);
 468
 469                 rate_throttle(td, usec);
 470
 471                 if (check_min_rate(td, &e)) {
 472                         td->error = ENODATA;
 473                         break;
 474                 }
 475
 476                 msec = usec / 1000;
 477                 add_stat_sample(td, msec);
 478
 479                 if (msec < td->min_latency)
 480                         td->min_latency = msec;
 481                 if (msec > td->max_latency)
 482                         td->max_latency = msec;
 483
 484                 if (runtime_exceeded(td, &e))
 485                         break;
 486         }
 487
 488         if (should_fsync(td))
 489                 fsync(td->fd);
 490 }
 491
 492 static void aio_put_iocb(struct thread_data *td, struct iocb *iocb)
 493 {
 494         long offset = ((long) iocb - (long) td->aio_iocbs)/ sizeof(struct iocb);
 495
 496         td->aio_iocbs_status[offset] = 0;
 497         td->aio_cur_depth--;
 498 }
 499
 500 static struct iocb *aio_get_iocb(struct thread_data *td, struct timeval *t)
 501 {
 502         struct iocb *iocb = NULL;
 503         unsigned int i;
 504
 505         for (i = 0; i < td->aio_depth; i++) {
 506                 if (td->aio_iocbs_status[i] == 0) {
 507                         td->aio_iocbs_status[i] = 1;
 508                         iocb = &td->aio_iocbs[i];
 509                         break;
 510                 }
 511         }
 512
 513         if (iocb) {
 514                 off_t off = get_next_offset(td);
 515                 char *p = td->buf + i * td->bs;
 516
 517                 if (td_read(td))
 518                         io_prep_pread(iocb, td->fd, p, td->bs, off);
 519                 else
 520                         io_prep_pwrite(iocb, td->fd, p, td->bs, off);
 521
 522                 io_set_callback(iocb, (io_callback_t) msec_now(t));
 523         }
 524
 525         return iocb;
 526 }
 527
 528 static int aio_submit(struct thread_data *td, struct iocb *iocb)
 529 {
 530         int ret;
 531
 532         do {
 533                 ret = io_submit(*td->aio_ctx, 1, &iocb);
 534                 if (ret == 1)
 535                         return 0;
 536
 537                 if (errno == EINTR)
 538                         continue;
 539                 else if (errno == EAGAIN)
 540                         usleep(100);
 541                 else
 542                         break;
 543         } while (1);
 544
 545         return 1;
 546 }
 547
 548 #define iocb_time(iocb) ((unsigned long) (iocb)->data)
 549
 550 static void do_async_io(struct thread_data *td)
 551 {
 552         struct timeval s, e;
 553         unsigned long blocks, msec, usec;
 554
 555         for (blocks = 0; blocks < td->blocks; blocks++) {
 556                 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
 557                 struct timespec *timeout;
 558                 int ret, i, min_evts = 0;
 559                 struct iocb *iocb;
 560
 561                 if (td->terminate)
 562                         break;
 563
 564                 if (td->delay_sleep)
 565                         usec_sleep(td->delay_sleep);
 566
 567                 gettimeofday(&s, NULL);
 568
 569                 iocb = aio_get_iocb(td, &s);
 570
 571                 ret = aio_submit(td, iocb);
 572                 if (ret) {
 573                         td->error = errno;
 574                         break;
 575                 }
 576
 577                 td->aio_cur_depth++;
 578
 579                 if (td->aio_cur_depth < td->aio_depth) {
 580                         timeout = &ts;
 581                         min_evts = 0;
 582                 } else {
 583                         timeout = NULL;
 584                         min_evts = 1;
 585                 }
 586
 587                 ret = io_getevents(*td->aio_ctx, min_evts, td->aio_cur_depth, td->aio_events, timeout);
 588                 if (ret < 0) {
 589                         td->error = errno;
 590                         break;
 591                 } else if (!ret)
 592                         continue;
 593
 594                 gettimeofday(&e, NULL);
 595
 596                 for (i = 0; i < ret; i++) {
 597                         struct io_event *ev = td->aio_events + i;
 598
 599                         td->io_blocks++;
 600
 601                         iocb = ev->obj;
 602
 603                         msec = msec_now(&e) - iocb_time(iocb);
 604                         add_stat_sample(td, msec);
 605
 606                         if (msec < td->min_latency)
 607                                 td->min_latency = msec;
 608                         if (msec > td->max_latency)
 609                                 td->max_latency = msec;
 610
 611                         aio_put_iocb(td, iocb);
 612                 }
 613
 614                 /*
 615                  * the rate is batched for now, it should work for batches
 616                  * of completions except the very first one which may look
 617                  * a little bursty
 618                  */
 619                 usec = utime_since(&s, &e);
 620
 621                 rate_throttle(td, usec);
 622
 623                 if (check_min_rate(td, &e)) {
 624                         td->error = ENODATA;
 625                         break;
 626                 }
 627
 628                 if (runtime_exceeded(td, &e))
 629                         break;
 630         }
 631 }
 632
 633 static void cleanup_pending_aio(struct thread_data *td)
 634 {
 635         struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
 636         unsigned int i;
 637         int r;
 638
 639         /*
 640          * get immediately available events, if any
 641          */
 642         r = io_getevents(*td->aio_ctx, 0, td->aio_cur_depth, td->aio_events, &ts);
 643         if (r > 0) {
 644                 for (i = 0; i < r; i++)
 645                         aio_put_iocb(td, &td->aio_iocbs[i]);
 646         }
 647
 648         /*
 649          * now cancel remaining active events
 650          */
 651         for (i = 0; i < td->aio_depth; i++) {
 652                 if (td->aio_iocbs_status[i] == 0)
 653                         continue;
 654
 655                 r = io_cancel(*td->aio_ctx, &td->aio_iocbs[i], td->aio_events);
 656                 if (!r)
 657                         aio_put_iocb(td, &td->aio_iocbs[i]);
 658         }
 659
 660         if (td->aio_cur_depth)
 661                 io_getevents(*td->aio_ctx, td->aio_cur_depth, td->aio_cur_depth, td->aio_events, NULL);
 662 }
 663
 664 static void cleanup_aio(struct thread_data *td)
 665 {
 666         if (td->aio_cur_depth)
 667                 cleanup_pending_aio(td);
 668
 669         if (td->aio_ctx) {
 670                 io_destroy(*td->aio_ctx);
 671                 free(td->aio_ctx);
 672         }
 673         if (td->aio_iocbs)
 674                 free(td->aio_iocbs);
 675         if (td->aio_events)
 676                 free(td->aio_events);
 677         if (td->aio_iocbs_status)
 678                 free(td->aio_iocbs_status);
 679 }
 680
 681 static int init_aio(struct thread_data *td)
 682 {
 683         td->aio_ctx = malloc(sizeof(*td->aio_ctx));
 684
 685         if (io_queue_init(td->aio_depth, td->aio_ctx)) {
 686                 td->error = errno;
 687                 return 1;
 688         }
 689
 690         td->aio_iocbs = malloc(td->aio_depth * sizeof(struct iocb));
 691         td->aio_events = malloc(td->aio_depth * sizeof(struct io_event));
 692         td->aio_iocbs_status = malloc(td->aio_depth * sizeof(char));
 693         return 0;
 694 }
 695
 696 static int create_file(struct thread_data *td)
 697 {
 698         unsigned int i;
 699         char *b;
 700
 701         if (!td->file_size) {
 702                 fprintf(stderr, "Need size for create\n");
 703                 td->error = EINVAL;
 704                 return 1;
 705         }
 706
 707         /*
 708          * unless specifically asked for overwrite, let normal io extend it
 709          */
 710         if (!td_read(td) && !td->overwrite)
 711                 return 0;
 712
 713         td->fd = open(td->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
 714         if (td->fd < 0) {
 715                 td->error = errno;
 716                 return 1;
 717         }
 718
 719         td->blocks = td->file_size / td->bs;
 720         b = malloc(td->bs);
 721         memset(b, 0, td->bs);
 722
 723         for (i = 0; i < td->blocks; i++) {
 724                 int r = write(td->fd, b, td->bs);
 725
 726                 if (r == td->bs)
 727                         continue;
 728                 else {
 729                         if (r < 0)
 730                                 td->error = errno;
 731                         else
 732                                 td->error = EIO;
 733
 734                         break;
 735                 }
 736         }
 737
 738         fsync(td->fd);
 739         close(td->fd);
 740         td->fd = -1;
 741         free(b);
 742         return 0;
 743 }
 744
 745 static int file_exists(struct thread_data *td)
 746 {
 747         struct stat st;
 748
 749         if (stat(td->file_name, &st) != -1)
 750                 return 1;
 751
 752         return errno != ENOENT;
 753 }
 754
 755 static int setup_file(struct thread_data *td)
 756 {
 757         struct stat st;
 758         int flags = 0;
 759
 760         if (!file_exists(td)) {
 761                 if (!td->create_file) {
 762                         td->error = ENOENT;
 763                         return 1;
 764                 }
 765                 if (create_file(td))
 766                         return 1;
 767         }
 768
 769         if (td->odirect)
 770                 flags |= O_DIRECT;
 771
 772         if (td_read(td))
 773                 td->fd = open(td->file_name, flags | O_RDONLY);
 774         else {
 775                 if (!td->overwrite)
 776                         flags |= O_TRUNC;
 777
 778                 td->fd = open(td->file_name, flags | O_WRONLY | O_CREAT, 0600);
 779         }
 780
 781         if (td->fd == -1) {
 782                 td->error = errno;
 783                 return 1;
 784         }
 785
 786         if (td_read(td)) {
 787                 if (fstat(td->fd, &st) == -1) {
 788                         td->error = errno;
 789                         return 1;
 790                 }
 791
 792                 if (td->file_size > st.st_size)
 793                         st.st_size = td->file_size;
 794         } else {
 795                 if (!td->file_size)
 796                         td->file_size = 1024 * 1024 * 1024;
 797
 798                 st.st_size = td->file_size;
 799         }
 800
 801         td->blocks = (st.st_size - td->file_offset) / td->bs;
 802         if (!td->blocks) {
 803                 fprintf(stderr, "Client%d: no io blocks\n", td->thread_number);
 804                 td->error = EINVAL;
 805                 return 1;
 806         }
 807
 808         if (td->invalidate_cache) {
 809                 if (fadvise(td->fd, 0, st.st_size, POSIX_FADV_DONTNEED) < 0) {
 810                         td->error = errno;
 811                         return 1;
 812                 }
 813         }
 814
 815         return 0;
 816 }
 817
 818 static void *thread_main(int shm_id, int offset, char *argv[])
 819 {
 820         struct thread_data *td;
 821         void *data, *ptr = NULL;
 822         int ret = 1;
 823
 824         setsid();
 825
 826         data = shmat(shm_id, NULL, 0);
 827         td = data + offset * sizeof(struct thread_data);
 828         td->pid = getpid();
 829
 830         if (sched_setaffinity(td->pid, sizeof(td->cpumask), &td->cpumask) == -1) {
 831                 td->error = errno;
 832                 goto err;
 833         }
 834
 835         printf("Client%d (pid=%u) started\n", td->thread_number, td->pid);
 836
 837         sprintf(argv[0], "fio%d", offset);
 838
 839         if (td->use_aio && init_aio(td))
 840                 goto err;
 841
 842         if (init_random_state(td))
 843                 goto err;
 844
 845         if (td->ioprio) {
 846                 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
 847                         td->error = errno;
 848                         goto err;
 849                 }
 850         }
 851
 852         if (setup_file(td))
 853                 goto err;
 854
 855         if (td->use_aio)
 856                 ptr = malloc(td->bs * td->aio_depth + MASK);
 857         else
 858                 ptr = malloc(td->bs + MASK);
 859
 860         td->buf = ALIGN(ptr);
 861
 862         sem_post(&startup_sem);
 863         sem_wait(&td->mutex);
 864
 865         gettimeofday(&td->start, NULL);
 866
 867         if (td->ratemin)
 868                 memcpy(&td->lastrate, &td->start, sizeof(td->start));
 869
 870         memcpy(&td->stat_sample_time, &td->start, sizeof(td->start));
 871
 872         if (!td->use_aio)
 873                 do_sync_io(td);
 874         else
 875                 do_async_io(td);
 876
 877         td->runtime = mtime_since_now(&td->start);
 878         ret = 0;
 879 err:
 880         if (td->use_aio)
 881                 cleanup_aio(td);
 882         if (td->fd != -1) {
 883                 close(td->fd);
 884                 td->fd = -1;
 885         }
 886         if (ret) {
 887                 sem_post(&startup_sem);
 888                 sem_wait(&td->mutex);
 889         }
 890         if (ptr)
 891                 free(ptr);
 892         td->runstate = TD_EXITED;
 893         shmdt(data);
 894         return NULL;
 895 }
 896
 897 static void free_shm(void)
 898 {
 899         shmdt(threads);
 900 }
 901
 902 static void show_thread_status(struct thread_data *td)
 903 {
 904         int prio, prio_class;
 905         unsigned long bw = 0;
 906         double n_lat, n_bw, m_lat, m_bw, dev_lat, dev_bw;
 907
 908         if (!td->io_blocks && !td->error)
 909                 return;
 910
 911         if (td->runtime)
 912                 bw = (td->io_blocks * td->bs) / td->runtime;
 913
 914         prio = td->ioprio & 0xff;
 915         prio_class = td->ioprio >> IOPRIO_CLASS_SHIFT;
 916
 917         n_lat = (double) td->stat_time_samples;
 918         n_bw = (double) td->stat_bw_samples;
 919
 920         m_lat = (double) td->stat_time / n_lat;
 921         dev_lat = sqrt(((double) td->stat_time_sq - (m_lat * m_lat) / n_lat) / (n_lat - 1));
 922         m_bw = (double) td->stat_bw / n_bw;
 923         dev_bw = sqrt(((double) td->stat_bw_sq - (m_bw * m_bw) / n_bw) / (n_bw - 1));
 924
 925         printf("Client%d: err=%2d, io=%6luMiB, bw=%6luKiB/sec, latmax=%5lumsec, latavg=%5.02fmsec, latdev=%5.02fmsec, bwavg=%5.02fKiB/sec, bwdev=%5.02fKiB/sec\n", td->thread_number, td->error, td->io_blocks * td->bs >> 20, bw, td->max_latency, m_lat, dev_lat, m_bw, dev_bw);
 926 }
 927
 928 static int setup_rate(struct thread_data *td)
 929 {
 930         int nr_reads_per_sec;
 931
 932         if (!td->rate)
 933                 return 0;
 934
 935         if (td->rate < td->ratemin) {
 936                 fprintf(stderr, "min rate larger than nominal rate\n");
 937                 return -1;
 938         }
 939
 940         nr_reads_per_sec = td->rate * 1024 / td->bs;
 941         td->rate_usec_cycle = 1000000 / nr_reads_per_sec;
 942         td->rate_pending_usleep = 0;
 943         return 0;
 944 }
 945
 946 static struct thread_data *get_new_job(int global)
 947 {
 948         struct thread_data *td;
 949
 950         if (global)
 951                 return &def_thread;
 952         if (thread_number >= max_jobs)
 953                 return NULL;
 954
 955         td = &threads[thread_number++];
 956         memset(td, 0, sizeof(*td));
 957
 958         td->fd = -1;
 959         td->thread_number = thread_number;
 960         td->ddir = def_thread.ddir;
 961         td->bs = def_thread.bs;
 962         td->odirect = def_thread.odirect;
 963         td->ratecycle = def_thread.ratecycle;
 964         td->sequential = def_thread.sequential;
 965         td->timeout = def_thread.timeout;
 966         td->create_file = def_thread.create_file;
 967         td->overwrite = def_thread.overwrite;
 968         td->invalidate_cache = def_thread.invalidate_cache;
 969         memcpy(&td->cpumask, &def_thread.cpumask, sizeof(td->cpumask));
 970
 971         return td;
 972 }
 973
 974 static void put_job(struct thread_data *td)
 975 {
 976         memset(&threads[td->thread_number - 1], 0, sizeof(*td));
 977         thread_number--;
 978 }
 979
 980 static int add_job(struct thread_data *td, const char *filename, int prioclass,
 981                    int prio)
 982 {
 983         if (td == &def_thread)
 984                 return 0;
 985
 986         strcpy(td->file_name, filename);
 987         sem_init(&td->mutex, 1, 0);
 988         td->min_latency = 10000000;
 989         td->ioprio = (prioclass << IOPRIO_CLASS_SHIFT) | prio;
 990
 991         if (td->use_aio && !td->aio_depth)
 992                 td->aio_depth = 1;
 993
 994         if (setup_rate(td))
 995                 return -1;
 996
 997         printf("Client%d: file=%s, rw=%d, prio=%d/%d, seq=%d, odir=%d, bs=%d, rate=%d, aio=%d, aio_depth=%d\n", td->thread_number, filename, td->ddir, prioclass, prio, td->sequential, td->odirect, td->bs, td->rate, td->use_aio, td->aio_depth);
 998         return 0;
 999 }
1000
1001 static void fill_cpu_mask(cpu_set_t cpumask, int cpu)
1002 {
1003         unsigned int i;
1004
1005         CPU_ZERO(&cpumask);
1006
1007         for (i = 0; i < sizeof(int) * 8; i++) {
1008                 if ((1 << i) & cpu)
1009                         CPU_SET(i, &cpumask);
1010         }
1011 }
1012
1013 static void fill_option(const char *input, char *output)
1014 {
1015         int i;
1016
1017         i = 0;
1018         while (input[i] != ',' && input[i] != '}' && input[i] != '\0') {
1019                 output[i] = input[i];
1020                 i++;
1021         }
1022
1023         output[i] = '\0';
1024 }
1025
1026 /*
1027  * convert string after '=' into decimal value, noting any size suffix
1028  */
1029 static int str_cnv(char *p, unsigned long long *val)
1030 {
1031         unsigned long mult;
1032         char *str;
1033         int len;
1034
1035         str = strstr(p, "=");
1036         if (!str)
1037                 return 1;
1038
1039         str++;
1040         len = strlen(str);
1041         mult = 1;
1042
1043         switch (str[len - 2]) {
1044                 case 'k':
1045                 case 'K':
1046                         mult = 1024;
1047                         break;
1048                 case 'm':
1049                 case 'M':
1050                         mult = 1024 * 1024;
1051                         break;
1052                 case 'g':
1053                 case 'G':
1054                         mult = 1024 * 1024 * 1024;
1055                         break;
1056         }
1057
1058         *val = strtoul(str, NULL, 10);
1059         if (*val == ULONG_MAX && errno == ERANGE)
1060                 return 1;
1061
1062         *val *= mult;
1063         return 0;
1064
1065 }
1066
1067 /*
1068  * job key words:
1069  *
1070  * file=
1071  * bs=
1072  * rw=
1073  * direct=
1074  */
1075 static void parse_jobs_cmd(int argc, char *argv[], int index)
1076 {
1077         struct thread_data *td;
1078         unsigned int prio, prioclass, cpu;
1079         char *string, *filename, *p, *c;
1080         int i;
1081
1082         string = malloc(256);
1083         filename = malloc(256);
1084
1085         for (i = index; i < argc; i++) {
1086                 p = argv[i];
1087
1088                 c = strpbrk(p, "{");
1089                 if (!c)
1090                         break;
1091
1092                 filename[0] = 0;
1093
1094                 td = get_new_job(0);
1095                 if (!td)
1096                         break;
1097
1098                 prioclass = 2;
1099                 prio = 4;
1100
1101                 c = strstr(p, "rw=");
1102                 if (c) {
1103                         c += 3;
1104                         if (*c == '0')
1105                                 td->ddir = DDIR_READ;
1106                         else
1107                                 td->ddir = DDIR_WRITE;
1108                 }
1109
1110                 c = strstr(p, "prio=");
1111                 if (c) {
1112                         c += 5;
1113                         prio = *c - '0';
1114                 }
1115
1116                 c = strstr(p, "prioclass=");
1117                 if (c) {
1118                         c += 10;
1119                         prioclass = *c - '0';
1120                 }
1121
1122                 c = strstr(p, "file=");
1123                 if (c) {
1124                         c += 5;
1125                         fill_option(c, filename);
1126                 }
1127
1128                 c = strstr(p, "bs=");
1129                 if (c) {
1130                         c += 3;
1131                         fill_option(c, string);
1132                         td->bs = strtoul(string, NULL, 10);
1133                         td->bs <<= 10;
1134                 }
1135
1136                 c = strstr(p, "direct=");
1137                 if (c) {
1138                         c += 7;
1139                         if (*c != '0')
1140                                 td->odirect = 1;
1141                         else
1142                                 td->odirect = 0;
1143                 }
1144
1145                 c = strstr(p, "delay=");
1146                 if (c) {
1147                         c += 6;
1148                         fill_option(c, string);
1149                         td->delay_sleep = strtoul(string, NULL, 10);
1150                 }
1151
1152                 c = strstr(p, "rate=");
1153                 if (c) {
1154                         c += 5;
1155                         fill_option(c, string);
1156                         td->rate = strtoul(string, NULL, 10);
1157                 }
1158
1159                 c = strstr(p, "ratemin=");
1160                 if (c) {
1161                         c += 8;
1162                         fill_option(c, string);
1163                         td->ratemin = strtoul(string, NULL, 10);
1164                 }
1165
1166                 c = strstr(p, "ratecycle=");
1167                 if (c) {
1168                         c += 10;
1169                         fill_option(c, string);
1170                         td->ratecycle = strtoul(string, NULL, 10);
1171                 }
1172
1173                 c = strstr(p, "cpumask=");
1174                 if (c) {
1175                         c += 8;
1176                         fill_option(c, string);
1177                         cpu = strtoul(string, NULL, 10);
1178                         fill_cpu_mask(td->cpumask, cpu);
1179                 }
1180
1181                 c = strstr(p, "fsync=");
1182                 if (c) {
1183                         c += 6;
1184                         fill_option(c, string);
1185                         td->fsync_blocks = strtoul(string, NULL, 10);
1186                 }
1187
1188                 c = strstr(p, "startdelay=");
1189                 if (c) {
1190                         c += 11;
1191                         fill_option(c, string);
1192                         td->start_delay = strtoul(string, NULL, 10);
1193                 }
1194
1195                 c = strstr(p, "timeout=");
1196                 if (c) {
1197                         c += 8;
1198                         fill_option(c, string);
1199                         td->timeout = strtoul(string, NULL, 10);
1200                 }
1201
1202                 c = strstr(p, "invalidate=");
1203                 if (c) {
1204                         c += 11;
1205                         if (*c != '0')
1206                                 td->invalidate_cache = 1;
1207                         else
1208                                 td->invalidate_cache = 0;
1209                 }
1210
1211                 c = strstr(p, "size=");
1212                 if (c) {
1213                         c += 5;
1214                         str_cnv(c, &td->file_size);
1215                 }
1216
1217                 c = strstr(p, "offset=");
1218                 if (c) {
1219                         c += 7;
1220                         str_cnv(c, &td->file_offset);
1221                 }
1222
1223                 c = strstr(p, "aio_depth=");
1224                 if (c) {
1225                         c += 10;
1226                         fill_option(c, string);
1227                         td->aio_depth = strtoul(string, NULL, 10);
1228                 }
1229
1230                 c = strstr(p, "aio");
1231                 if (c)
1232                         td->use_aio = 1;
1233
1234                 c = strstr(p, "create");
1235                 if (c)
1236                         td->create_file = 1;
1237
1238                 c = strstr(p, "overwrite");
1239                 if (c)
1240                         td->overwrite = 1;
1241
1242                 c = strstr(p, "random");
1243                 if (c)
1244                         td->sequential = 0;
1245                 c = strstr(p, "sequential");
1246                 if (c)
1247                         td->sequential = 1;
1248
1249                 if (add_job(td, filename, prioclass, prio))
1250                         put_job(td);
1251         }
1252
1253         free(string);
1254         free(filename);
1255 }
1256
1257 static int check_strcnv(char *p, char *name, unsigned long long *val)
1258 {
1259         if (!strstr(p, name))
1260                 return 1;
1261
1262         return str_cnv(p, val);
1263 }
1264
1265 static int check_int(char *p, char *name, unsigned int *val)
1266 {
1267         char str[128];
1268
1269         sprintf(str, "%s=%%d", name);
1270         if (sscanf(p, str, val) == 1)
1271                 return 0;
1272
1273         sprintf(str, "%s = %%d", name);
1274         if (sscanf(p, str, val) == 1)
1275                 return 0;
1276
1277         return 1;
1278 }
1279
1280 static int is_empty_or_comment(char *line)
1281 {
1282         unsigned int i;
1283
1284         for (i = 0; i < strlen(line); i++) {
1285                 if (line[i] == ';')
1286                         return 1;
1287                 if (!isspace(line[i]) && !iscntrl(line[i]))
1288                         return 0;
1289         }
1290
1291         return 1;
1292 }
1293
1294 static int parse_jobs_ini(char *file)
1295 {
1296         unsigned int prioclass, prio, cpu, global;
1297         struct thread_data *td;
1298         char *string, *name;
1299         fpos_t off;
1300         FILE *f;
1301         char *p;
1302
1303         f = fopen(file, "r");
1304         if (!f) {
1305                 perror("fopen");
1306                 return 1;
1307         }
1308
1309         string = malloc(4096);
1310         name = malloc(256);
1311
1312         while ((p = fgets(string, 4096, f)) != NULL) {
1313                 if (is_empty_or_comment(p))
1314                         continue;
1315                 if (sscanf(p, "[%s]", name) != 1)
1316                         continue;
1317
1318                 global = !strncmp(name, "global", 6);
1319
1320                 name[strlen(name) - 1] = '\0';
1321
1322                 td = get_new_job(global);
1323                 if (!td)
1324                         break;
1325
1326                 prioclass = 2;
1327                 prio = 4;
1328
1329                 fgetpos(f, &off);
1330                 while ((p = fgets(string, 4096, f)) != NULL) {
1331                         if (is_empty_or_comment(p))
1332                                 continue;
1333                         if (strstr(p, "["))
1334                                 break;
1335                         if (!check_int(p, "bs", &td->bs)) {
1336                                 td->bs <<= 10;
1337                                 fgetpos(f, &off);
1338                                 continue;
1339                         }
1340                         if (!check_int(p, "rw", &td->ddir)) {
1341                                 fgetpos(f, &off);
1342                                 continue;
1343                         }
1344                         if (!check_int(p, "prio", &prio)) {
1345                                 fgetpos(f, &off);
1346                                 continue;
1347                         }
1348                         if (!check_int(p, "prioclass", &prioclass)) {
1349                                 fgetpos(f, &off);
1350                                 continue;
1351                         }
1352                         if (!check_int(p, "direct", &td->odirect)) {
1353                                 fgetpos(f, &off);
1354                                 continue;
1355                         }
1356                         if (!check_int(p, "rate", &td->rate)) {
1357                                 fgetpos(f, &off);
1358                                 continue;
1359                         }
1360                         if (!check_int(p, "ratemin", &td->ratemin)) {
1361                                 fgetpos(f, &off);
1362                                 continue;
1363                         }
1364                         if (!check_int(p, "ratecycle", &td->ratecycle)) {
1365                                 fgetpos(f, &off);
1366                                 continue;
1367                         }
1368                         if (!check_int(p, "delay", &td->delay_sleep)) {
1369                                 fgetpos(f, &off);
1370                                 continue;
1371                         }
1372                         if (!check_int(p, "cpumask", &cpu)) {
1373                                 fill_cpu_mask(td->cpumask, cpu);
1374                                 fgetpos(f, &off);
1375                                 continue;
1376                         }
1377                         if (!check_int(p, "fsync", &td->fsync_blocks)) {
1378                                 fgetpos(f, &off);
1379                                 continue;
1380                         }
1381                         if (!check_int(p, "startdelay", &td->start_delay)) {
1382                                 fgetpos(f, &off);
1383                                 continue;
1384                         }
1385                         if (!check_int(p, "timeout", &td->timeout)) {
1386                                 fgetpos(f, &off);
1387                                 continue;
1388                         }
1389                         if (!check_int(p, "invalidate",&td->invalidate_cache)) {
1390                                 fgetpos(f, &off);
1391                                 continue;
1392                         }
1393                         if (!check_int(p, "aio_depth", &td->aio_depth)) {
1394                                 fgetpos(f, &off);
1395                                 continue;
1396                         }
1397                         if (!check_strcnv(p, "size", &td->file_size)) {
1398                                 fgetpos(f, &off);
1399                                 continue;
1400                         }
1401                         if (!check_strcnv(p, "offset", &td->file_offset)) {
1402                                 fgetpos(f, &off);
1403                                 continue;
1404                         }
1405                         if (!strncmp(p, "sequential", 10)) {
1406                                 td->sequential = 1;
1407                                 fgetpos(f, &off);
1408                                 continue;
1409                         }
1410                         if (!strncmp(p, "random", 6)) {
1411                                 td->sequential = 0;
1412                                 fgetpos(f, &off);
1413                                 continue;
1414                         }
1415                         if (!strncmp(p, "aio", 3)) {
1416                                 td->use_aio = 1;
1417                                 fgetpos(f, &off);
1418                                 continue;
1419                         }
1420                         if (!strncmp(p, "create", 6)) {
1421                                 td->create_file = 1;
1422                                 fgetpos(f, &off);
1423                                 continue;
1424                         }
1425                         if (!strncmp(p, "overwrite", 9)) {
1426                                 td->overwrite = 1;
1427                                 fgetpos(f, &off);
1428                                 continue;
1429                         }
1430                         printf("Client%d: bad option %s\n",td->thread_number,p);
1431                 }
1432                 fsetpos(f, &off);
1433
1434                 if (add_job(td, name, prioclass, prio))
1435                         put_job(td);
1436         }
1437
1438         free(string);
1439         free(name);
1440         fclose(f);
1441         return 0;
1442 }
1443
1444 static int parse_options(int argc, char *argv[])
1445 {
1446         int i;
1447
1448         for (i = 1; i < argc; i++) {
1449                 char *parm = argv[i];
1450
1451                 if (parm[0] != '-')
1452                         break;
1453
1454                 parm++;
1455                 switch (*parm) {
1456                         case 's':
1457                                 parm++;
1458                                 def_thread.sequential = !!atoi(parm);
1459                                 break;
1460                         case 'b':
1461                                 parm++;
1462                                 def_thread.bs = atoi(parm);
1463                                 def_thread.bs <<= 10;
1464                                 if (!def_thread.bs) {
1465                                         printf("bad block size\n");
1466                                         def_thread.bs = DEF_BS;
1467                                 }
1468                                 break;
1469                         case 't':
1470                                 parm++;
1471                                 def_thread.timeout = atoi(parm);
1472                                 break;
1473                         case 'r':
1474                                 parm++;
1475                                 repeatable = !!atoi(parm);
1476                                 break;
1477                         case 'R':
1478                                 parm++;
1479                                 rate_quit = !!atoi(parm);
1480                                 break;
1481                         case 'o':
1482                                 parm++;
1483                                 def_thread.odirect = !!atoi(parm);
1484                                 break;
1485                         case 'f':
1486                                 if (i + 1 >= argc) {
1487                                         printf("-f needs file as arg\n");
1488                                         break;
1489                                 }
1490                                 ini_file = strdup(argv[i+1]);
1491                                 i++;
1492                                 break;
1493                         default:
1494                                 printf("bad option %s\n", argv[i]);
1495                                 break;
1496                 }
1497         }
1498
1499         return i;
1500 }
1501
1502 static void print_thread_status(struct thread_data *td, int nr_running,
1503                                 int t_rate, int m_rate, int die)
1504 {
1505         printf("Client%d: %s\n", td->thread_number, die ? "exited" : "spawned");
1506
1507         printf("Threads now running: %d", nr_running);
1508         if (m_rate || t_rate)
1509                 printf(", commitrate %d/%dKiB/sec", t_rate, m_rate);
1510         printf("\n");
1511 }
1512
1513 static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
1514 {
1515         int i;
1516
1517         /*
1518          * reap exited threads (TD_EXITED -> TD_REAPED)
1519          */
1520         for (i = 0; i < thread_number; i++) {
1521                 struct thread_data *td = &threads[i];
1522
1523                 if (td->runstate != TD_EXITED)
1524                         continue;
1525
1526                 td->runstate = TD_REAPED;
1527                 waitpid(td->pid, NULL, 0);
1528                 (*nr_running)--;
1529                 (*m_rate) -= td->ratemin;
1530                 (*t_rate) -= td->rate;
1531
1532                 if (td->terminate)
1533                         continue;
1534
1535                 print_thread_status(td, *nr_running, *t_rate, *m_rate, 1);
1536         }
1537 }
1538
1539 static void run_threads(char *argv[])
1540 {
1541         struct timeval genesis;
1542         struct thread_data *td;
1543         unsigned long spent;
1544         int i, todo, nr_running, m_rate, t_rate;
1545
1546         gettimeofday(&genesis, NULL);
1547
1548         printf("Starting %d threads\n", thread_number);
1549         fflush(stdout);
1550
1551         signal(SIGINT, sig_handler);
1552
1553         todo = thread_number;
1554         nr_running = 0;
1555         m_rate = t_rate = 0;
1556
1557         while (todo) {
1558                 /*
1559                  * create threads (TD_NOT_CREATED -> TD_CREATED)
1560                  */
1561                 for (i = 0; i < thread_number; i++) {
1562                         td = &threads[i];
1563
1564                         if (td->runstate != TD_NOT_CREATED)
1565                                 continue;
1566
1567                         /*
1568                          * never got a chance to start, killed by other
1569                          * thread for some reason
1570                          */
1571                         if (td->terminate) {
1572                                 todo--;
1573                                 continue;
1574                         }
1575
1576                         if (td->start_delay) {
1577                                 spent = mtime_since_now(&genesis);
1578
1579                                 if (td->start_delay * 1000 > spent)
1580                                         continue;
1581                         }
1582
1583                         td->runstate = TD_CREATED;
1584                         sem_init(&startup_sem, 1, 1);
1585                         todo--;
1586
1587                         if (fork())
1588                                 sem_wait(&startup_sem);
1589                         else {
1590                                 thread_main(shm_id, i, argv);
1591                                 exit(0);
1592                         }
1593                 }
1594
1595                 /*
1596                  * start created threads (TD_CREATED -> TD_STARTED)
1597                  */
1598                 for (i = 0; i < thread_number; i++) {
1599                         struct thread_data *td = &threads[i];
1600
1601                         if (td->runstate != TD_CREATED)
1602                                 continue;
1603
1604                         td->runstate = TD_STARTED;
1605                         nr_running++;
1606                         m_rate += td->ratemin;
1607                         t_rate += td->rate;
1608                         sem_post(&td->mutex);
1609
1610                         print_thread_status(td, nr_running, t_rate, m_rate, 0);
1611                 }
1612
1613                 reap_threads(&nr_running, &t_rate, &m_rate);
1614
1615                 if (todo)
1616                         usleep(100000);
1617         }
1618
1619         while (nr_running) {
1620                 reap_threads(&nr_running, &t_rate, &m_rate);
1621                 usleep(10000);
1622         }
1623 }
1624
1625 int setup_thread_area(void)
1626 {
1627         /*
1628          * 1024 is too much on some machines, scale max_jobs if
1629          * we get a failure that looks like too large a shm segment
1630          */
1631         do {
1632                 int s = max_jobs * sizeof(struct thread_data);
1633
1634                 shm_id = shmget(0, s, IPC_CREAT | 0600);
1635                 if (shm_id != -1)
1636                         break;
1637                 if (errno != EINVAL) {
1638                         perror("shmget");
1639                         break;
1640                 }
1641
1642                 max_jobs >>= 1;
1643         } while (max_jobs);
1644
1645         if (shm_id == -1)
1646                 return 1;
1647
1648         threads = shmat(shm_id, NULL, 0);
1649         if (threads == (void *) -1) {
1650                 perror("shmat");
1651                 return 1;
1652         }
1653
1654         atexit(free_shm);
1655         return 0;
1656 }
1657
1658 int main(int argc, char *argv[])
1659 {
1660         static unsigned long max_run[2], min_run[2], total_blocks[2];
1661         static unsigned long max_bw[2], min_bw[2], maxl[2], minl[2];
1662         static unsigned long read_mb, write_mb, read_agg, write_agg;
1663         int i;
1664
1665         if (setup_thread_area())
1666                 return 1;
1667
1668         if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &def_thread.cpumask) == -1) {
1669                 perror("sched_getaffinity");
1670                 return 1;
1671         }
1672
1673         /*
1674          * fill globals
1675          */
1676         def_thread.ddir = DDIR_READ;
1677         def_thread.bs = DEF_BS;
1678         def_thread.odirect = DEF_ODIRECT;
1679         def_thread.ratecycle = DEF_RATE_CYCLE;
1680         def_thread.sequential = DEF_SEQUENTIAL;
1681         def_thread.timeout = DEF_TIMEOUT;
1682         def_thread.create_file = DEF_CREATE;
1683         def_thread.overwrite = DEF_OVERWRITE;
1684         def_thread.invalidate_cache = DEF_INVALIDATE;
1685
1686         i = parse_options(argc, argv);
1687
1688         if (ini_file) {
1689                 if (parse_jobs_ini(ini_file))
1690                         return 1;
1691         } else
1692                 parse_jobs_cmd(argc, argv, i);
1693
1694         if (!thread_number) {
1695                 printf("Nothing to do\n");
1696                 return 1;
1697         }
1698
1699         run_threads(argv);
1700
1701         min_bw[0] = min_run[0] = ~0UL;
1702         min_bw[1] = min_run[1] = ~0UL;
1703         minl[0] = minl[1] = ~0UL;
1704         for (i = 0; i < thread_number; i++) {
1705                 struct thread_data *td = &threads[i];
1706                 unsigned long bw = 0;
1707
1708                 if (td->error)
1709                         goto show_stat;
1710
1711                 if (td->runtime < min_run[td->ddir])
1712                         min_run[td->ddir] = td->runtime;
1713                 if (td->runtime > max_run[td->ddir])
1714                         max_run[td->ddir] = td->runtime;
1715
1716                 if (td->runtime)
1717                         bw = (td->io_blocks * td->bs) / td->runtime;
1718                 if (bw < min_bw[td->ddir])
1719                         min_bw[td->ddir] = bw;
1720                 if (bw > max_bw[td->ddir])
1721                         max_bw[td->ddir] = bw;
1722                 if (td->max_latency < minl[td->ddir])
1723                         minl[td->ddir] = td->max_latency;
1724                 if (td->max_latency > maxl[td->ddir])
1725                         maxl[td->ddir] = td->max_latency;
1726
1727                 total_blocks[td->ddir] += td->io_blocks;
1728
1729                 if (td_read(td)) {
1730                         read_mb += (td->bs * td->io_blocks) >> 20;
1731                         if (td->runtime)
1732                                 read_agg += (td->io_blocks * td->bs) / td->runtime;
1733                 } else {
1734                         write_mb += (td->bs * td->io_blocks) >> 20;
1735                         if (td->runtime)
1736                                 write_agg += (td->io_blocks * td->bs) / td->runtime;
1737                 }
1738
1739 show_stat:
1740                 show_thread_status(td);
1741         }
1742
1743         printf("Run status:\n");
1744         if (max_run[DDIR_READ])
1745                 printf("   READ: io=%luMiB, aggrb=%lu, minl=%lu, maxl=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", read_mb, read_agg, minl[0], maxl[0], min_bw[0], max_bw[0], min_run[0], max_run[0]);
1746         if (max_run[DDIR_WRITE])
1747                 printf("  WRITE: io=%luMiB, aggrb=%lu, minl=%lu, maxl=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", write_mb, write_agg, minl[1], maxl[1], min_bw[1], max_bw[1], min_run[1], max_run[1]);
1748
1749         return 0;
1750 }