[PATCH] fio: Implement async verify
[disktools.git] / fio.c
CommitLineData
abe4da87
JA
1/*
2 * fio - the flexible io tester
3 *
4 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 *
20 */
892199bd
JA
21#include <stdio.h>
22#include <stdlib.h>
23#include <unistd.h>
24#include <fcntl.h>
25#include <string.h>
26#include <errno.h>
27#include <signal.h>
28#include <time.h>
7dd1389e 29#include <ctype.h>
18e0b78c 30#include <sched.h>
43000118 31#include <libaio.h>
e128065d 32#include <math.h>
02983297 33#include <limits.h>
892199bd
JA
34#include <sys/time.h>
35#include <sys/types.h>
36#include <sys/stat.h>
37#include <sys/wait.h>
38#include <semaphore.h>
39#include <sys/ipc.h>
40#include <sys/shm.h>
41#include <asm/unistd.h>
946d8870 42#include <asm/types.h>
892199bd 43
2c83567e 44#include "list.h"
e8457004 45#include "md5.h"
2c83567e 46
4240cfa1
JA
47#define MAX_JOBS (1024)
48
892199bd
JA
49/*
50 * assume we don't have _get either, if _set isn't defined
51 */
52#ifndef __NR_ioprio_set
892199bd
JA
53#if defined(__i386__)
54#define __NR_ioprio_set 289
55#define __NR_ioprio_get 290
56#elif defined(__powerpc__) || defined(__powerpc64__)
57#define __NR_ioprio_set 273
58#define __NR_ioprio_get 274
59#elif defined(__x86_64__)
60#define __NR_ioprio_set 251
61#define __NR_ioprio_get 252
62#elif defined(__ia64__)
63#define __NR_ioprio_set 1274
64#define __NR_ioprio_get 1275
65#elif defined(__alpha__)
66#define __NR_ioprio_set 442
67#define __NR_ioprio_get 443
68#elif defined(__s390x__) || defined(__s390__)
69#define __NR_ioprio_set 282
70#define __NR_ioprio_get 283
71#else
72#error "Unsupported arch"
73#endif
b95799ca 74#endif
892199bd 75
b95799ca
JA
76#ifndef __NR_fadvise64
77#if defined(__i386__)
78#define __NR_fadvise64 250
79#elif defined(__powerpc__) || defined(__powerpc64__)
80#define __NR_fadvise64 233
81#elif defined(__x86_64__)
82#define __NR_fadvise64 221
83#elif defined(__ia64__)
84#define __NR_fadvise64 1234
85#elif defined(__alpha__)
86#define __NR_fadvise64 413
87#elif defined(__s390x__) || defined(__s390__)
88#define __NR_fadvise64 253
89#else
90#error "Unsupported arch"
91#endif
892199bd
JA
92#endif
93
94static int ioprio_set(int which, int who, int ioprio)
95{
96 return syscall(__NR_ioprio_set, which, who, ioprio);
97}
98
b95799ca
JA
99/*
100 * we want fadvise64 really, but it's so tangled... later
101 */
102static int fadvise(int fd, loff_t offset, size_t len, int advice)
103{
104#if 0
105 return syscall(__NR_fadvise64, fd, offset, offset >> 32, len, advice);
106#else
107 return posix_fadvise(fd, (off_t) offset, len, advice);
108#endif
109}
110
892199bd
JA
111enum {
112 IOPRIO_WHO_PROCESS = 1,
113 IOPRIO_WHO_PGRP,
114 IOPRIO_WHO_USER,
115};
116
117#define IOPRIO_CLASS_SHIFT 13
118
892199bd
JA
119#define MASK (4095)
120
4240cfa1
JA
121#define DEF_BS (4096)
122#define DEF_TIMEOUT (30)
123#define DEF_RATE_CYCLE (1000)
124#define DEF_ODIRECT (1)
125#define DEF_SEQUENTIAL (1)
4240cfa1 126#define DEF_RAND_REPEAT (1)
02983297
JA
127#define DEF_OVERWRITE (0)
128#define DEF_CREATE (1)
b95799ca 129#define DEF_INVALIDATE (1)
99c6704f 130#define DEF_SYNCIO (0)
eb39963f 131#define DEF_RANDSEED (0xb1899bedUL)
1d035750 132#define DEF_BWAVGTIME (500)
fc097bfe
JA
133#define DEF_CREATE_SER (1)
134#define DEF_CREATE_FSYNC (1)
b6794fbf 135#define DEF_LOOPS (1)
cfc702bd 136#define DEF_VERIFY (0)
4240cfa1
JA
137
138#define ALIGN(buf) (char *) (((unsigned long) (buf) + MASK) & ~(MASK))
892199bd 139
4240cfa1 140static int repeatable = DEF_RAND_REPEAT;
02bdd9ba 141static int rate_quit = 1;
a0a9b35b
JA
142static int write_lat_log;
143static int write_bw_log;
98dd52d6 144static int exitall_on_terminate;
892199bd 145
892199bd 146static int thread_number;
7dd1389e 147static char *ini_file;
892199bd 148
8867c0a8
JA
149static int max_jobs = MAX_JOBS;
150
8dbff0b1
JA
151static char run_str[MAX_JOBS + 1];
152
892199bd
JA
153static int shm_id;
154
4240cfa1
JA
155enum {
156 DDIR_READ = 0,
157 DDIR_WRITE,
158};
892199bd 159
02bdd9ba
JA
160/*
161 * thread life cycle
162 */
163enum {
164 TD_NOT_CREATED = 0,
165 TD_CREATED,
e8457004
JA
166 TD_RUNNING,
167 TD_VERIFYING,
02bdd9ba
JA
168 TD_EXITED,
169 TD_REAPED,
170};
171
99c6704f
JA
172enum {
173 MEM_MALLOC,
174 MEM_SHM,
175};
176
2c83567e
JA
177/*
178 * The io unit
179 */
180struct io_u {
181 struct iocb iocb;
57d753e3 182 struct timeval start_time;
2c83567e
JA
183 struct timeval issue_time;
184
2c83567e
JA
185 char *buf;
186 unsigned int buflen;
4ac89145 187 unsigned long long offset;
2c83567e
JA
188
189 struct list_head list;
190};
191
57d753e3
JA
192struct io_stat {
193 unsigned long val;
194 unsigned long val_sq;
195 unsigned long max_val;
196 unsigned long min_val;
197 unsigned long samples;
198};
199
a0a9b35b
JA
200struct io_sample {
201 unsigned long time;
202 unsigned long val;
203};
204
205struct io_log {
206 unsigned long nr_samples;
207 unsigned long max_samples;
208 struct io_sample *log;
209};
210
e8457004
JA
211#define FIO_HDR_MAGIC 0xf00baaef
212
213struct verify_header {
214 unsigned int fio_magic;
215 unsigned int len;
216 char md5_digest[MD5_HASH_WORDS * 4];
217};
218
02983297
JA
219#define td_read(td) ((td)->ddir == DDIR_READ)
220#define should_fsync(td) (!td_read(td) && !(td)->odirect)
221
892199bd
JA
222struct thread_data {
223 char file_name[256];
224 int thread_number;
225 int error;
226 int fd;
892199bd 227 pid_t pid;
6b71c826 228 char *orig_buffer;
4240cfa1 229 volatile int terminate;
02bdd9ba 230 volatile int runstate;
40ef7f64 231 volatile int old_runstate;
f737299d
JA
232 unsigned int ddir;
233 unsigned int ioprio;
234 unsigned int sequential;
235 unsigned int bs;
8c033f93
JA
236 unsigned int min_bs;
237 unsigned int max_bs;
f737299d 238 unsigned int odirect;
e97712ed 239 unsigned int thinktime;
4240cfa1 240 unsigned int fsync_blocks;
fc24389f 241 unsigned int start_delay;
47d45203 242 unsigned int timeout;
43000118 243 unsigned int use_aio;
02983297
JA
244 unsigned int create_file;
245 unsigned int overwrite;
b95799ca 246 unsigned int invalidate_cache;
1d035750 247 unsigned int bw_avg_time;
fc097bfe
JA
248 unsigned int create_serialize;
249 unsigned int create_fsync;
b6794fbf 250 unsigned int loops;
02983297
JA
251 unsigned long long file_size;
252 unsigned long long file_offset;
74b4b5fb 253 unsigned int sync_io;
99c6704f 254 unsigned int mem_type;
cfc702bd 255 unsigned int verify;
18e0b78c 256 cpu_set_t cpumask;
86184d14 257
7889f07b 258 struct drand48_data bsrange_state;
e8457004 259 struct drand48_data verify_state;
7889f07b 260
99c6704f
JA
261 int shm_id;
262
63a09e51
JA
263 off_t cur_off;
264
254605cd 265 io_context_t aio_ctx;
43000118 266 unsigned int aio_depth;
43000118 267 struct io_event *aio_events;
2c83567e
JA
268
269 unsigned int cur_depth;
270 struct list_head io_u_freelist;
271 struct list_head io_u_busylist;
43000118 272
7dd1389e 273 unsigned int rate;
4240cfa1
JA
274 unsigned int ratemin;
275 unsigned int ratecycle;
276 unsigned long rate_usec_cycle;
277 long rate_pending_usleep;
8c033f93 278 unsigned long rate_kb;
4240cfa1 279 struct timeval lastrate;
86184d14 280
892199bd 281 unsigned long runtime; /* sec */
7889f07b 282 unsigned long kb;
4240cfa1 283 unsigned long io_blocks;
8c033f93 284 unsigned long io_kb;
f8e3d840 285 unsigned long this_io_kb;
7889f07b 286 unsigned long last_kb;
892199bd 287 sem_t mutex;
892199bd
JA
288 struct drand48_data random_state;
289
290 /*
e128065d 291 * bandwidth and latency stats
892199bd 292 */
57d753e3
JA
293 struct io_stat clat_stat; /* completion latency */
294 struct io_stat slat_stat; /* submission latency */
295
296 struct io_stat bw_stat; /* bandwidth stats */
8c033f93 297 unsigned long stat_io_kb;
fd1ae4c9 298 struct timeval stat_sample_time;
4240cfa1 299
a0a9b35b
JA
300 struct io_log *lat_log;
301 struct io_log *bw_log;
302
4240cfa1 303 struct timeval start;
892199bd
JA
304};
305
306static struct thread_data *threads;
47d45203 307static struct thread_data def_thread;
892199bd
JA
308
309static sem_t startup_sem;
310
5c24b2c4 311static void sig_handler(int sig)
892199bd
JA
312{
313 int i;
314
213b446c
JA
315 for (i = 0; i < thread_number; i++) {
316 struct thread_data *td = &threads[i];
317
318 td->terminate = 1;
319 td->start_delay = 0;
320 }
02bdd9ba
JA
321}
322
e8457004 323static int init_random_state(struct thread_data *td)
946d8870 324{
e8457004
JA
325 unsigned long seed;
326 int fd;
946d8870 327
e8457004
JA
328 fd = open("/dev/random", O_RDONLY);
329 if (fd == -1) {
330 td->error = errno;
331 return 1;
946d8870
JA
332 }
333
e8457004
JA
334 if (read(fd, &seed, sizeof(seed)) < (int) sizeof(seed)) {
335 td->error = EIO;
336 close(fd);
337 return 1;
946d8870
JA
338 }
339
e8457004 340 close(fd);
7889f07b
JA
341
342 srand48_r(seed, &td->bsrange_state);
e8457004 343 srand48_r(seed, &td->verify_state);
892199bd
JA
344
345 if (td->sequential)
346 return 0;
347
e8457004
JA
348 if (repeatable)
349 seed = DEF_RANDSEED;
892199bd
JA
350
351 srand48_r(seed, &td->random_state);
352 return 0;
353}
354
5c24b2c4 355static unsigned long utime_since(struct timeval *s, struct timeval *e)
892199bd
JA
356{
357 double sec, usec;
358
359 sec = e->tv_sec - s->tv_sec;
360 usec = e->tv_usec - s->tv_usec;
361 if (sec > 0 && usec < 0) {
362 sec--;
363 usec += 1000000;
364 }
365
366 sec *= (double) 1000000;
367
368 return sec + usec;
369}
370
5c24b2c4 371static unsigned long mtime_since(struct timeval *s, struct timeval *e)
892199bd
JA
372{
373 double sec, usec;
374
375 sec = e->tv_sec - s->tv_sec;
376 usec = e->tv_usec - s->tv_usec;
377 if (sec > 0 && usec < 0) {
378 sec--;
379 usec += 1000000;
380 }
381
382 sec *= (double) 1000;
383 usec /= (double) 1000;
384
385 return sec + usec;
386}
387
be33abe4
JA
388static unsigned long mtime_since_now(struct timeval *s)
389{
390 struct timeval t;
391
392 gettimeofday(&t, NULL);
393 return mtime_since(s, &t);
394}
395
98168d55
JA
396static inline unsigned long msec_now(struct timeval *s)
397{
398 return s->tv_sec * 1000 + s->tv_usec / 1000;
399}
400
4ac89145 401static unsigned long long get_next_offset(struct thread_data *td)
892199bd 402{
f31fd9c8 403 unsigned long long kb;
40ef7f64 404 double r;
892199bd
JA
405
406 if (!td->sequential) {
40ef7f64 407 drand48_r(&td->random_state, &r);
f31fd9c8 408 kb = (1+(double) (td->kb-1) * r / (RAND_MAX+1.0));
7889f07b 409 } else
f31fd9c8 410 kb = td->last_kb;
7889f07b 411
f31fd9c8 412 return (kb << 10) + td->file_offset;
7889f07b
JA
413}
414
415static unsigned int get_next_buflen(struct thread_data *td)
416{
417 unsigned int buflen;
40ef7f64 418 double r;
7889f07b
JA
419
420 if (td->min_bs == td->max_bs)
421 buflen = td->min_bs;
422 else {
40ef7f64 423 drand48_r(&td->bsrange_state, &r);
7889f07b
JA
424 buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0));
425 buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1);
892199bd
JA
426 }
427
f8e3d840
JA
428 if (buflen > ((td->kb - td->this_io_kb) << 10))
429 buflen = (td->kb - td->this_io_kb) << 10;
7889f07b
JA
430
431 td->last_kb += buflen >> 10;
432 return buflen;
892199bd
JA
433}
434
57d753e3
JA
435static inline void add_stat_sample(struct thread_data *td, struct io_stat *is,
436 unsigned long val)
892199bd 437{
57d753e3
JA
438 if (val > is->max_val)
439 is->max_val = val;
440 if (val < is->min_val)
441 is->min_val = val;
442
443 is->val += val;
444 is->val_sq += val * val;
445 is->samples++;
446}
fd1ae4c9 447
a0a9b35b
JA
448static void add_log_sample(struct thread_data *td, struct io_log *log,
449 unsigned long val)
450{
451 if (log->nr_samples == log->max_samples) {
452 int new_size = sizeof(struct io_sample) * log->max_samples * 2;
453
454 log->log = realloc(log->log, new_size);
455 log->max_samples <<= 1;
456 }
457
458 log->log[log->nr_samples].val = val;
459 log->log[log->nr_samples].time = mtime_since_now(&td->start);
460 log->nr_samples++;
461}
462
57d753e3
JA
463static void add_clat_sample(struct thread_data *td, unsigned long msec)
464{
465 add_stat_sample(td, &td->clat_stat, msec);
a0a9b35b
JA
466
467 if (td->lat_log)
468 add_log_sample(td, td->lat_log, msec);
57d753e3 469}
fd1ae4c9 470
57d753e3
JA
471static void add_slat_sample(struct thread_data *td, unsigned long msec)
472{
473 add_stat_sample(td, &td->slat_stat, msec);
474}
fd1ae4c9 475
57d753e3
JA
476static void add_bw_sample(struct thread_data *td, unsigned long msec)
477{
478 unsigned long spent = mtime_since_now(&td->stat_sample_time);
479 unsigned long rate;
480
1d035750 481 if (spent < td->bw_avg_time)
57d753e3
JA
482 return;
483
f8e3d840 484 rate = ((td->this_io_kb - td->stat_io_kb) * 1024) / spent;
57d753e3
JA
485 add_stat_sample(td, &td->bw_stat, rate);
486
a0a9b35b
JA
487 if (td->bw_log)
488 add_log_sample(td, td->bw_log, rate);
489
57d753e3 490 gettimeofday(&td->stat_sample_time, NULL);
f8e3d840 491 td->stat_io_kb = td->this_io_kb;
892199bd
JA
492}
493
5c24b2c4 494static void usec_sleep(int usec)
892199bd 495{
86184d14
JA
496 struct timespec req = { .tv_sec = 0, .tv_nsec = usec * 1000 };
497 struct timespec rem;
892199bd
JA
498
499 do {
86184d14
JA
500 rem.tv_sec = rem.tv_nsec = 0;
501 nanosleep(&req, &rem);
502 if (!rem.tv_nsec)
892199bd 503 break;
86184d14
JA
504
505 req.tv_nsec = rem.tv_nsec;
892199bd
JA
506 } while (1);
507}
508
9e850933
JA
509static void rate_throttle(struct thread_data *td, unsigned long time_spent,
510 unsigned int bytes)
86184d14 511{
9e850933
JA
512 unsigned long usec_cycle;
513
4240cfa1
JA
514 if (!td->rate)
515 return;
516
9e850933
JA
517 usec_cycle = td->rate_usec_cycle * (bytes / td->min_bs);
518
519 if (time_spent < usec_cycle) {
520 unsigned long s = usec_cycle - time_spent;
86184d14
JA
521
522 td->rate_pending_usleep += s;
fad86e6a 523 if (td->rate_pending_usleep >= 100000) {
86184d14
JA
524 usec_sleep(td->rate_pending_usleep);
525 td->rate_pending_usleep = 0;
526 }
4240cfa1 527 } else {
9e850933 528 long overtime = time_spent - usec_cycle;
42b2b9fe 529
4240cfa1
JA
530 td->rate_pending_usleep -= overtime;
531 }
532}
533
5c24b2c4 534static int check_min_rate(struct thread_data *td, struct timeval *now)
4240cfa1 535{
7607bc6b 536 unsigned long spent;
4240cfa1
JA
537 unsigned long rate;
538
539 /*
540 * allow a 2 second settle period in the beginning
541 */
7607bc6b 542 if (mtime_since(&td->start, now) < 2000)
4240cfa1
JA
543 return 0;
544
545 /*
546 * if rate blocks is set, sample is running
547 */
8c033f93 548 if (td->rate_kb) {
4240cfa1
JA
549 spent = mtime_since(&td->lastrate, now);
550 if (spent < td->ratecycle)
551 return 0;
552
f8e3d840 553 rate = ((td->this_io_kb - td->rate_kb) * 1024) / spent;
4240cfa1
JA
554 if (rate < td->ratemin) {
555 printf("Client%d: min rate %d not met, got %ldKiB/sec\n", td->thread_number, td->ratemin, rate);
02bdd9ba 556 if (rate_quit)
e6402082 557 sig_handler(0);
4240cfa1
JA
558 return 1;
559 }
86184d14 560 }
4240cfa1 561
f8e3d840 562 td->rate_kb = td->this_io_kb;
4240cfa1
JA
563 memcpy(&td->lastrate, now, sizeof(*now));
564 return 0;
86184d14
JA
565}
566
67903a2e
JA
567static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
568{
569 if (mtime_since(&td->start, t) >= td->timeout * 1000)
570 return 1;
571
572 return 0;
573}
574
e8457004
JA
575static void fill_random_bytes(struct thread_data *td,
576 unsigned char *p, unsigned int len)
577{
40ef7f64 578 double r;
e8457004 579 int todo;
e8457004
JA
580
581 while (len) {
40ef7f64 582 drand48_r(&td->verify_state, &r);
e8457004 583
40ef7f64
JA
584 /*
585 * lrand48_r seems to be broken and only fill the bottom
586 * 32-bits, even on 64-bit archs with 64-bit longs
587 */
588 todo = sizeof(r);
e8457004
JA
589 if (todo > len)
590 todo = len;
591
592 memcpy(p, &r, todo);
593
594 len -= todo;
595 p += todo;
596 }
597}
598
599static int verify_io_u(struct thread_data *td, struct io_u *io_u)
600{
601 struct verify_header *hdr = (struct verify_header *) io_u->buf;
602 unsigned char *p = (unsigned char *) io_u->buf;
603 struct md5_ctx md5_ctx;
604
605 if (hdr->fio_magic != FIO_HDR_MAGIC)
606 return 1;
607
608 memset(&md5_ctx, 0, sizeof(md5_ctx));
609 p += sizeof(*hdr);
610 md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
611
612 return memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
613}
614
615static int verify_io_us(struct thread_data *td, struct io_u *io_u, int *back)
616{
617 struct verify_header *hdr;
618 unsigned int left;
619 struct io_u i;
620 char *buf;
621 off_t off;
622
623 *back = 0;
624
625 left = io_u->buflen;
626 buf = io_u->buf;
627 off = io_u->offset;
628 while (left) {
629 hdr = (struct verify_header *) buf;
630 i.buf = buf;
631 i.buflen = hdr->len;
632
633 if (hdr->len > left) {
634 *back = left;
635 return 0;
636 }
637
638 if (verify_io_u(td, &i)) {
639 printf("failed verify at offset %lu\n", (unsigned long) off);
640 return 1;
641 }
642
643 buf += hdr->len;
644 left -= hdr->len;
645 off += hdr->len;
646 }
647
648 return 0;
649}
650
cfc702bd
JA
651/*
652 * fill body of io_u->buf with random data and add a header with the
653 * (eg) sha1sum of that data.
654 */
e8457004 655static void populate_io_u(struct thread_data *td, struct io_u *io_u)
cfc702bd 656{
e8457004
JA
657 struct md5_ctx md5_ctx;
658 struct verify_header hdr;
659 unsigned char *p = (unsigned char *) io_u->buf;
660
661 hdr.fio_magic = FIO_HDR_MAGIC;
662 hdr.len = io_u->buflen;
663 p += sizeof(hdr);
664 fill_random_bytes(td, p, io_u->buflen - sizeof(hdr));
665
666 memset(&md5_ctx, 0, sizeof(md5_ctx));
667 md5_update(&md5_ctx, p, io_u->buflen - sizeof(hdr));
668 memcpy(hdr.md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
669 memcpy(io_u->buf, &hdr, sizeof(hdr));
cfc702bd
JA
670}
671
2c83567e
JA
672static void put_io_u(struct thread_data *td, struct io_u *io_u)
673{
674 list_del(&io_u->list);
675 list_add(&io_u->list, &td->io_u_freelist);
676 td->cur_depth--;
677}
678
e8457004
JA
679static struct io_u *__get_io_u(struct thread_data *td)
680{
681 struct io_u *io_u;
682
683 if (list_empty(&td->io_u_freelist))
684 return NULL;
685
686 io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
687 list_del(&io_u->list);
688 list_add(&io_u->list, &td->io_u_busylist);
f4bb2243 689 td->cur_depth++;
e8457004
JA
690 return io_u;
691}
692
2c83567e
JA
693static struct io_u *get_io_u(struct thread_data *td)
694{
695 struct io_u *io_u;
696
e8457004
JA
697 io_u = __get_io_u(td);
698 if (!io_u)
2c83567e
JA
699 return NULL;
700
b2a369fb
JA
701 io_u->offset = get_next_offset(td);
702 io_u->buflen = get_next_buflen(td);
703 if (!io_u->buflen) {
e8457004 704 put_io_u(td, io_u);
7889f07b 705 return NULL;
e8457004 706 }
2c83567e 707
cfc702bd 708 if (!td_read(td) && td->verify)
e8457004 709 populate_io_u(td, io_u);
cfc702bd 710
2c83567e
JA
711 if (td->use_aio) {
712 if (td_read(td))
713 io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
714 else
715 io_prep_pwrite(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
716 }
717
57d753e3 718 gettimeofday(&io_u->start_time, NULL);
2c83567e
JA
719 return io_u;
720}
721
40ef7f64
JA
722static inline void td_set_runstate(struct thread_data *td, int runstate)
723{
724 td->old_runstate = td->runstate;
725 td->runstate = runstate;
726}
727
cfc702bd
JA
728static int do_sync_verify(struct thread_data *td)
729{
40ef7f64 730 struct timeval t;
e8457004
JA
731 struct io_u *io_u = NULL;
732 loff_t off = 0;
733 int back, ret;
734
40ef7f64 735 td_set_runstate(td, TD_VERIFYING);
e8457004
JA
736
737 io_u = __get_io_u(td);
738
739 if (lseek(td->fd, 0, SEEK_SET) < 0) {
740 td->error = errno;
741 goto out;
742 }
743
40ef7f64
JA
744 if (!td->odirect) {
745 unsigned long size = td->kb << 10;
746
747 if (fadvise(td->fd, 0, size, POSIX_FADV_DONTNEED) < 0) {
748 td->error = errno;
749 goto out;
750 }
751 }
752
e8457004
JA
753 do {
754 if (td->terminate)
755 break;
40ef7f64
JA
756
757 gettimeofday(&t, NULL);
758 if (runtime_exceeded(td, &t))
759 break;
760
e8457004
JA
761 io_u->offset = off;
762 io_u->buflen = td->max_bs;
763
764 ret = read(td->fd, io_u->buf, io_u->buflen);
765 if (ret < (int) io_u->buflen) {
766 if (ret == -1) {
767 td->error = errno;
768 break;
769 } else if (!ret)
770 break;
771 else
772 io_u->buflen = ret;
773 }
774
775 if (verify_io_us(td, io_u, &back))
776 break;
777
778 if (back) {
f8e3d840 779 ret -= back;
e8457004
JA
780 if (lseek(td->fd, -back, SEEK_CUR) < 0) {
781 td->error = errno;
782 break;
783 }
784 }
785
786 off += ret;
787 } while (1);
788
789out:
40ef7f64 790 td_set_runstate(td, TD_RUNNING);
e8457004
JA
791 put_io_u(td, io_u);
792
793 return td->error == 0;
cfc702bd
JA
794}
795
43000118 796static void do_sync_io(struct thread_data *td)
892199bd 797{
7889f07b 798 unsigned long msec, usec;
e8457004 799 struct io_u *io_u = NULL;
2c83567e 800 struct timeval e;
892199bd 801
63a09e51 802 td->cur_off = 0;
e8457004 803 td->last_kb = 0;
63a09e51 804
f8e3d840 805 for (td->this_io_kb = 0; td->this_io_kb < td->kb;) {
892199bd
JA
806 int ret;
807
808 if (td->terminate)
809 break;
810
2c83567e 811 io_u = get_io_u(td);
7889f07b
JA
812 if (!io_u)
813 break;
2c83567e 814
63a09e51
JA
815 if (td->cur_off != io_u->offset) {
816 if (lseek(td->fd, io_u->offset, SEEK_SET) == -1) {
817 td->error = errno;
818 break;
819 }
892199bd
JA
820 }
821
02983297 822 if (td_read(td))
2c83567e 823 ret = read(td->fd, io_u->buf, io_u->buflen);
892199bd 824 else
2c83567e 825 ret = write(td->fd, io_u->buf, io_u->buflen);
892199bd 826
2c83567e 827 if (ret < (int) io_u->buflen) {
892199bd
JA
828 if (ret == -1)
829 td->error = errno;
830 break;
831 }
832
4240cfa1 833 td->io_blocks++;
8c033f93 834 td->io_kb += io_u->buflen >> 10;
f8e3d840 835 td->this_io_kb += io_u->buflen >> 10;
63a09e51 836 td->cur_off = io_u->offset + io_u->buflen;
4240cfa1 837
86184d14
JA
838 gettimeofday(&e, NULL);
839
57d753e3 840 usec = utime_since(&io_u->start_time, &e);
86184d14 841
9e850933 842 rate_throttle(td, usec, io_u->buflen);
892199bd 843
4240cfa1
JA
844 if (check_min_rate(td, &e)) {
845 td->error = ENODATA;
846 break;
847 }
892199bd 848
4240cfa1 849 msec = usec / 1000;
57d753e3
JA
850 add_clat_sample(td, msec);
851 add_bw_sample(td, msec);
67903a2e
JA
852
853 if (runtime_exceeded(td, &e))
854 break;
2c83567e 855
cdf92433 856 put_io_u(td, io_u);
e8457004 857 io_u = NULL;
cdf92433 858
e97712ed
JA
859 if (td->thinktime)
860 usec_sleep(td->thinktime);
861
cdf92433
JA
862 if (should_fsync(td) && td->fsync_blocks &&
863 (td->io_blocks % td->fsync_blocks) == 0)
864 fsync(td->fd);
892199bd
JA
865 }
866
e8457004
JA
867 if (io_u)
868 put_io_u(td, io_u);
869
4240cfa1 870 if (should_fsync(td))
892199bd 871 fsync(td->fd);
892199bd 872}
43000118 873
2c83567e 874static int io_u_queue(struct thread_data *td, struct io_u *io_u)
56b0eff0 875{
2c83567e 876 struct iocb *iocb = &io_u->iocb;
56b0eff0
JA
877 int ret;
878
879 do {
254605cd 880 ret = io_submit(td->aio_ctx, 1, &iocb);
56b0eff0
JA
881 if (ret == 1)
882 return 0;
a592bd33 883 else if (ret == EAGAIN)
56b0eff0 884 usleep(100);
a592bd33
JA
885 else if (ret == EINTR)
886 continue;
56b0eff0
JA
887 else
888 break;
889 } while (1);
890
a592bd33 891 return ret;
56b0eff0
JA
892}
893
98168d55 894#define iocb_time(iocb) ((unsigned long) (iocb)->data)
2c83567e
JA
895#define ev_to_iou(ev) (struct io_u *) ((unsigned long) (ev)->obj)
896
f4bb2243 897static int ios_completed(struct thread_data *td, int nr, int stats)
2c83567e
JA
898{
899 unsigned long msec;
900 struct io_u *io_u;
901 struct timeval e;
9e850933 902 int i, bytes_done;
2c83567e 903
f4bb2243
JA
904 if (stats)
905 gettimeofday(&e, NULL);
2c83567e 906
9e850933 907 for (i = 0, bytes_done = 0; i < nr; i++) {
2c83567e
JA
908 io_u = ev_to_iou(td->aio_events + i);
909
f4bb2243
JA
910 if (stats) {
911 td->io_blocks++;
912 td->io_kb += io_u->buflen >> 10;
913 td->this_io_kb += io_u->buflen >> 10;
8c033f93 914
f4bb2243 915 msec = mtime_since(&io_u->issue_time, &e);
2c83567e 916
f4bb2243
JA
917 add_clat_sample(td, msec);
918 add_bw_sample(td, msec);
919 }
2c83567e 920
f4bb2243 921 bytes_done += io_u->buflen;
2c83567e
JA
922 put_io_u(td, io_u);
923 }
9e850933
JA
924
925 return bytes_done;
2c83567e
JA
926}
927
928static void cleanup_pending_aio(struct thread_data *td)
929{
930 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
931 struct list_head *entry, *n;
932 struct io_u *io_u;
933 int r;
934
935 /*
936 * get immediately available events, if any
937 */
938 r = io_getevents(td->aio_ctx, 0, td->cur_depth, td->aio_events, &ts);
939 if (r > 0)
f4bb2243 940 ios_completed(td, r, 1);
2c83567e
JA
941
942 /*
943 * now cancel remaining active events
944 */
945 list_for_each_safe(entry, n, &td->io_u_busylist) {
946 io_u = list_entry(entry, struct io_u, list);
947
948 r = io_cancel(td->aio_ctx, &io_u->iocb, td->aio_events);
949 if (!r)
950 put_io_u(td, io_u);
951 }
952
953 if (td->cur_depth) {
954 r = io_getevents(td->aio_ctx, td->cur_depth, td->cur_depth, td->aio_events, NULL);
955 if (r > 0)
f4bb2243 956 ios_completed(td, r, 1);
2c83567e
JA
957 }
958}
98168d55 959
cfc702bd
JA
960static int do_async_verify(struct thread_data *td)
961{
f4bb2243
JA
962 struct timeval t;
963 struct io_u *io_u;
964 int ret, back;
965
966 td_set_runstate(td, TD_VERIFYING);
967
968 td->cur_off = 0;
969 td->last_kb = 0;
970
971 do {
972 if (td->terminate)
973 break;
974
975 gettimeofday(&t, NULL);
976 if (runtime_exceeded(td, &t))
977 break;
978
979 io_u = __get_io_u(td);
980 if (!io_u)
981 break;
982
983 io_u->offset = td->cur_off;
984 io_u->buflen = td->max_bs;
985
986 if (io_u->offset + io_u->buflen > (td->kb << 10)) {
987 io_u->buflen = (td->kb << 10) - io_u->offset;
988 if (!io_u->buflen) {
989 put_io_u(td, io_u);
990 break;
991 }
992 }
993
994 io_prep_pread(&io_u->iocb, td->fd, io_u->buf, io_u->buflen, io_u->offset);
995 ret = io_u_queue(td, io_u);
996 if (ret) {
997 put_io_u(td, io_u);
998 td->error = ret;
999 break;
1000 }
1001
1002 ret = io_getevents(td->aio_ctx, 1, td->cur_depth, td->aio_events, NULL);
1003 if (ret < 0) {
1004 td->error = errno;
1005 break;
1006 }
1007
1008 td->cur_off += ios_completed(td, ret, 0);
1009
1010 if (verify_io_us(td, io_u, &back))
1011 break;
1012
1013 td->cur_off -= back;
1014 } while (1);
1015
1016 if (td->cur_depth)
1017 cleanup_pending_aio(td);
1018
1019 td_set_runstate(td, TD_RUNNING);
1020 return td->error == 0;
cfc702bd
JA
1021}
1022
43000118
JA
1023static void do_async_io(struct thread_data *td)
1024{
1025 struct timeval s, e;
7889f07b 1026 unsigned long usec;
43000118 1027
e8457004
JA
1028 td->cur_off = 0;
1029 td->last_kb = 0;
1030
f8e3d840 1031 for (td->this_io_kb = 0; td->this_io_kb < td->kb;) {
43000118
JA
1032 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
1033 struct timespec *timeout;
2c83567e
JA
1034 int ret, min_evts = 0;
1035 struct io_u *io_u;
9e850933 1036 unsigned int bytes_done;
43000118
JA
1037
1038 if (td->terminate)
1039 break;
1040
2c83567e 1041 io_u = get_io_u(td);
7889f07b
JA
1042 if (!io_u)
1043 break;
43000118 1044
57d753e3 1045 memcpy(&s, &io_u->start_time, sizeof(s));
8baf1bcc 1046
2c83567e 1047 ret = io_u_queue(td, io_u);
56b0eff0 1048 if (ret) {
a3fdb993 1049 put_io_u(td, io_u);
a592bd33 1050 td->error = ret;
43000118
JA
1051 break;
1052 }
1053
57d753e3
JA
1054 gettimeofday(&io_u->issue_time, NULL);
1055 add_slat_sample(td, mtime_since(&io_u->start_time, &io_u->issue_time));
2c83567e 1056 if (td->cur_depth < td->aio_depth) {
43000118
JA
1057 timeout = &ts;
1058 min_evts = 0;
1059 } else {
1060 timeout = NULL;
1061 min_evts = 1;
1062 }
1063
2c83567e 1064 ret = io_getevents(td->aio_ctx, min_evts, td->cur_depth, td->aio_events, timeout);
43000118
JA
1065 if (ret < 0) {
1066 td->error = errno;
1067 break;
1068 } else if (!ret)
1069 continue;
1070
f4bb2243 1071 bytes_done = ios_completed(td, ret, 1);
43000118 1072
98168d55
JA
1073 /*
1074 * the rate is batched for now, it should work for batches
1075 * of completions except the very first one which may look
1076 * a little bursty
1077 */
2c83567e 1078 gettimeofday(&e, NULL);
43000118
JA
1079 usec = utime_since(&s, &e);
1080
9e850933 1081 rate_throttle(td, usec, bytes_done);
43000118
JA
1082
1083 if (check_min_rate(td, &e)) {
1084 td->error = ENODATA;
1085 break;
1086 }
67903a2e
JA
1087
1088 if (runtime_exceeded(td, &e))
1089 break;
765d9223
JA
1090
1091 if (td->thinktime)
1092 usec_sleep(td->thinktime);
cdf92433
JA
1093
1094 if (should_fsync(td) && td->fsync_blocks &&
1095 (td->io_blocks % td->fsync_blocks) == 0)
1096 fsync(td->fd);
43000118 1097 }
56b0eff0 1098
2c83567e
JA
1099 if (td->cur_depth)
1100 cleanup_pending_aio(td);
4ac89145
JA
1101
1102 if (should_fsync(td))
1103 fsync(td->fd);
56b0eff0
JA
1104}
1105
1106static void cleanup_aio(struct thread_data *td)
1107{
254605cd
JA
1108 io_destroy(td->aio_ctx);
1109
43000118
JA
1110 if (td->aio_events)
1111 free(td->aio_events);
43000118
JA
1112}
1113
1114static int init_aio(struct thread_data *td)
1115{
254605cd 1116 if (io_queue_init(td->aio_depth, &td->aio_ctx)) {
43000118
JA
1117 td->error = errno;
1118 return 1;
1119 }
1120
43000118 1121 td->aio_events = malloc(td->aio_depth * sizeof(struct io_event));
43000118
JA
1122 return 0;
1123}
1124
2c83567e
JA
1125static void cleanup_io_u(struct thread_data *td)
1126{
1127 struct list_head *entry, *n;
1128 struct io_u *io_u;
1129
1130 list_for_each_safe(entry, n, &td->io_u_freelist) {
1131 io_u = list_entry(entry, struct io_u, list);
1132
1133 list_del(&io_u->list);
2c83567e
JA
1134 free(io_u);
1135 }
6b71c826 1136
99c6704f
JA
1137 if (td->mem_type == MEM_MALLOC)
1138 free(td->orig_buffer);
1139 else if (td->mem_type == MEM_SHM) {
1140 struct shmid_ds sbuf;
1141
1142 shmdt(td->orig_buffer);
1143 shmctl(td->shm_id, IPC_RMID, &sbuf);
1144 }
2c83567e
JA
1145}
1146
99c6704f 1147static int init_io_u(struct thread_data *td)
2c83567e
JA
1148{
1149 struct io_u *io_u;
99c6704f 1150 int i, max_units, mem_size;
6b71c826 1151 char *p;
2c83567e
JA
1152
1153 if (!td->use_aio)
1154 max_units = 1;
1155 else
1156 max_units = td->aio_depth;
1157
7889f07b 1158 mem_size = td->max_bs * max_units + MASK;
99c6704f
JA
1159
1160 if (td->mem_type == MEM_MALLOC)
1161 td->orig_buffer = malloc(mem_size);
1162 else if (td->mem_type == MEM_SHM) {
1163 td->shm_id = shmget(IPC_PRIVATE, mem_size, IPC_CREAT | 0600);
1164 if (td->shm_id < 0) {
1165 td->error = errno;
1166 perror("shmget");
1167 return 1;
1168 }
1169
1170 td->orig_buffer = shmat(td->shm_id, NULL, 0);
1171 if (td->orig_buffer == (void *) -1) {
1172 td->error = errno;
1173 perror("shmat");
1174 return 1;
1175 }
1176 }
6b71c826 1177
2c83567e
JA
1178 INIT_LIST_HEAD(&td->io_u_freelist);
1179 INIT_LIST_HEAD(&td->io_u_busylist);
1180
99c6704f 1181 p = ALIGN(td->orig_buffer);
2c83567e
JA
1182 for (i = 0; i < max_units; i++) {
1183 io_u = malloc(sizeof(*io_u));
1184 memset(io_u, 0, sizeof(*io_u));
1185 INIT_LIST_HEAD(&io_u->list);
1186
7889f07b 1187 io_u->buf = p + td->max_bs * i;
2c83567e
JA
1188 list_add(&io_u->list, &td->io_u_freelist);
1189 }
99c6704f
JA
1190
1191 return 0;
2c83567e
JA
1192}
1193
a0a9b35b
JA
1194static void setup_log(struct io_log **log)
1195{
1196 struct io_log *l = malloc(sizeof(*l));
1197
1198 l->nr_samples = 0;
1199 l->max_samples = 1024;
1200 l->log = malloc(l->max_samples * sizeof(struct io_sample));
1201 *log = l;
1202}
1203
1204static void finish_log(struct thread_data *td, struct io_log *log, char *name)
1205{
1206 char file_name[128];
1207 FILE *f;
1208 int i;
1209
1210 sprintf(file_name, "client%d_%s.log", td->thread_number, name);
1211 f = fopen(file_name, "w");
1212 if (!f) {
1213 perror("fopen log");
1214 return;
1215 }
1216
1217 for (i = 0; i < log->nr_samples; i++)
1218 fprintf(f, "%lu, %lu\n", log->log[i].time, log->log[i].val);
1219
1220 fclose(f);
1221 free(log->log);
1222 free(log);
1223}
1224
02983297
JA
1225static int create_file(struct thread_data *td)
1226{
7889f07b 1227 unsigned long long left;
02983297 1228 char *b;
7889f07b 1229 int r, bs;
02983297 1230
02983297
JA
1231 /*
1232 * unless specifically asked for overwrite, let normal io extend it
1233 */
1234 if (!td_read(td) && !td->overwrite)
1235 return 0;
1236
57d753e3
JA
1237 if (!td->file_size) {
1238 fprintf(stderr, "Need size for create\n");
1239 td->error = EINVAL;
1240 return 1;
1241 }
1242
42fd89a7
JA
1243 printf("Client%d: Laying out IO file\n", td->thread_number);
1244
02983297
JA
1245 td->fd = open(td->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
1246 if (td->fd < 0) {
1247 td->error = errno;
1248 return 1;
1249 }
1250
7889f07b
JA
1251 td->kb = td->file_size >> 10;
1252 b = malloc(td->max_bs);
1253 memset(b, 0, td->max_bs);
1254
1255 left = td->file_size;
1256 while (left) {
1257 bs = td->max_bs;
1258 if (bs > left)
1259 bs = left;
02983297 1260
7889f07b 1261 r = write(td->fd, b, bs);
02983297 1262
7889f07b
JA
1263 if (r == bs) {
1264 left -= bs;
02983297 1265 continue;
7889f07b 1266 } else {
02983297
JA
1267 if (r < 0)
1268 td->error = errno;
1269 else
1270 td->error = EIO;
1271
1272 break;
1273 }
1274 }
1275
fc097bfe
JA
1276 if (td->create_fsync)
1277 fsync(td->fd);
1278
02983297
JA
1279 close(td->fd);
1280 td->fd = -1;
1281 free(b);
1282 return 0;
1283}
1284
1285static int file_exists(struct thread_data *td)
1286{
1287 struct stat st;
1288
1289 if (stat(td->file_name, &st) != -1)
1290 return 1;
1291
1292 return errno != ENOENT;
1293}
1294
1295static int setup_file(struct thread_data *td)
1296{
1297 struct stat st;
1298 int flags = 0;
1299
1300 if (!file_exists(td)) {
1301 if (!td->create_file) {
1302 td->error = ENOENT;
1303 return 1;
1304 }
1305 if (create_file(td))
1306 return 1;
1307 }
1308
1309 if (td->odirect)
1310 flags |= O_DIRECT;
1311
1312 if (td_read(td))
1313 td->fd = open(td->file_name, flags | O_RDONLY);
1314 else {
1315 if (!td->overwrite)
1316 flags |= O_TRUNC;
74b4b5fb
JA
1317 if (td->sync_io)
1318 flags |= O_SYNC;
e8457004
JA
1319 if (td->verify)
1320 flags |= O_RDWR;
1321 else
1322 flags |= O_WRONLY;
02983297 1323
e8457004 1324 td->fd = open(td->file_name, flags | O_CREAT, 0600);
02983297
JA
1325 }
1326
1327 if (td->fd == -1) {
1328 td->error = errno;
1329 return 1;
1330 }
1331
1332 if (td_read(td)) {
1333 if (fstat(td->fd, &st) == -1) {
1334 td->error = errno;
1335 return 1;
1336 }
1337
1338 if (td->file_size > st.st_size)
1339 st.st_size = td->file_size;
1340 } else {
1341 if (!td->file_size)
1342 td->file_size = 1024 * 1024 * 1024;
1343
1344 st.st_size = td->file_size;
1345 }
1346
7889f07b
JA
1347 td->kb = (st.st_size - td->file_offset) / 1024;
1348 if (!td->kb) {
02983297
JA
1349 fprintf(stderr, "Client%d: no io blocks\n", td->thread_number);
1350 td->error = EINVAL;
1351 return 1;
1352 }
1353
b95799ca
JA
1354 if (td->invalidate_cache) {
1355 if (fadvise(td->fd, 0, st.st_size, POSIX_FADV_DONTNEED) < 0) {
1356 td->error = errno;
1357 return 1;
1358 }
1359 }
1360
02983297
JA
1361 return 0;
1362}
1363
5c24b2c4 1364static void *thread_main(int shm_id, int offset, char *argv[])
892199bd
JA
1365{
1366 struct thread_data *td;
02983297 1367 int ret = 1;
2c83567e 1368 void *data;
892199bd 1369
7292613b
JA
1370 setsid();
1371
892199bd 1372 data = shmat(shm_id, NULL, 0);
4ac89145
JA
1373 if (data == (void *) -1) {
1374 perror("shmat");
1375 return NULL;
1376 }
1377
892199bd
JA
1378 td = data + offset * sizeof(struct thread_data);
1379 td->pid = getpid();
1380
99c6704f
JA
1381 if (init_io_u(td))
1382 goto err;
2c83567e 1383
18e0b78c
JA
1384 if (sched_setaffinity(td->pid, sizeof(td->cpumask), &td->cpumask) == -1) {
1385 td->error = errno;
1386 goto err;
1387 }
1388
4240cfa1 1389 sprintf(argv[0], "fio%d", offset);
892199bd 1390
43000118
JA
1391 if (td->use_aio && init_aio(td))
1392 goto err;
1393
892199bd 1394 if (init_random_state(td))
599002b3 1395 goto err;
892199bd 1396
f737299d 1397 if (td->ioprio) {
892199bd
JA
1398 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
1399 td->error = errno;
599002b3 1400 goto err;
892199bd
JA
1401 }
1402 }
1403
1404 sem_post(&startup_sem);
1405 sem_wait(&td->mutex);
43000118 1406
fc097bfe
JA
1407 if (!td->create_serialize && setup_file(td))
1408 goto err;
1409
7292613b
JA
1410 gettimeofday(&td->start, NULL);
1411
1412 if (td->ratemin)
1413 memcpy(&td->lastrate, &td->start, sizeof(td->start));
1414
fd1ae4c9
JA
1415 memcpy(&td->stat_sample_time, &td->start, sizeof(td->start));
1416
b6794fbf 1417 while (td->loops--) {
cfc702bd 1418 if (!td->use_aio) {
b6794fbf 1419 do_sync_io(td);
cfc702bd
JA
1420
1421 if (td->verify) {
f8e3d840 1422 if (!do_sync_verify(td))
cfc702bd
JA
1423 break;
1424 }
1425 } else {
b6794fbf 1426 do_async_io(td);
cfc702bd
JA
1427
1428 if (td->verify) {
f8e3d840 1429 if (!do_async_verify(td))
cfc702bd
JA
1430 break;
1431 }
1432 }
b6794fbf 1433 }
7292613b 1434
be33abe4 1435 td->runtime = mtime_since_now(&td->start);
892199bd 1436 ret = 0;
a0a9b35b
JA
1437
1438 if (td->bw_log)
1439 finish_log(td, td->bw_log, "bw");
1440 if (td->lat_log)
1441 finish_log(td, td->lat_log, "lat");
4ac89145 1442
98dd52d6
JA
1443 if (exitall_on_terminate)
1444 sig_handler(0);
1445
892199bd 1446err:
7292613b
JA
1447 if (td->fd != -1) {
1448 close(td->fd);
1449 td->fd = -1;
1450 }
4ac89145
JA
1451 if (td->use_aio)
1452 cleanup_aio(td);
2c83567e 1453 cleanup_io_u(td);
599002b3 1454 if (ret) {
892199bd 1455 sem_post(&startup_sem);
599002b3
JA
1456 sem_wait(&td->mutex);
1457 }
40ef7f64 1458 td_set_runstate(td, TD_EXITED);
4240cfa1 1459 shmdt(data);
892199bd
JA
1460 return NULL;
1461}
1462
5c24b2c4 1463static void free_shm(void)
892199bd 1464{
c269123b
JA
1465 struct shmid_ds sbuf;
1466
1467 if (threads) {
1468 shmdt(threads);
1469 threads = NULL;
1470 shmctl(shm_id, IPC_RMID, &sbuf);
1471 }
892199bd
JA
1472}
1473
57d753e3
JA
1474static int calc_lat(struct io_stat *is, unsigned long *min, unsigned long *max,
1475 double *mean, double *dev)
1476{
1477 double n;
1478
1479 if (is->samples == 0)
1480 return 0;
1481
1482 *min = is->min_val;
1483 *max = is->max_val;
1484
1485 n = (double) is->samples;
1486 *mean = (double) is->val / n;
1487 *dev = sqrt(((double) is->val_sq - (*mean * *mean) / n) / (n - 1));
1488 return 1;
1489}
1490
5c24b2c4 1491static void show_thread_status(struct thread_data *td)
892199bd
JA
1492{
1493 int prio, prio_class;
57d753e3
JA
1494 unsigned long min, max, bw = 0;
1495 double mean, dev;
892199bd 1496
8c033f93 1497 if (!td->io_kb && !td->error)
213b446c
JA
1498 return;
1499
892199bd 1500 if (td->runtime)
7889f07b 1501 bw = td->io_kb * 1024 / td->runtime;
892199bd
JA
1502
1503 prio = td->ioprio & 0xff;
1504 prio_class = td->ioprio >> IOPRIO_CLASS_SHIFT;
1505
396fe7cd 1506 printf("Client%d: err=%2d, io=%6luMiB, bw=%6luKiB/s, runt=%6lumsec\n", td->thread_number, td->error, td->io_kb >> 10, bw, td->runtime);
fd1ae4c9 1507
57d753e3
JA
1508 if (calc_lat(&td->slat_stat, &min, &max, &mean, &dev))
1509 printf(" slat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1510 if (calc_lat(&td->clat_stat, &min, &max, &mean, &dev))
1511 printf(" clat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
1512 if (calc_lat(&td->bw_stat, &min, &max, &mean, &dev))
1513 printf(" bw (KiB/s) : min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev);
892199bd
JA
1514}
1515
5c24b2c4 1516static int setup_rate(struct thread_data *td)
86184d14 1517{
4240cfa1
JA
1518 int nr_reads_per_sec;
1519
1520 if (!td->rate)
1521 return 0;
1522
1523 if (td->rate < td->ratemin) {
1524 fprintf(stderr, "min rate larger than nominal rate\n");
1525 return -1;
1526 }
86184d14 1527
7889f07b 1528 nr_reads_per_sec = td->rate * 1024 / td->min_bs;
86184d14
JA
1529 td->rate_usec_cycle = 1000000 / nr_reads_per_sec;
1530 td->rate_pending_usleep = 0;
4240cfa1 1531 return 0;
86184d14
JA
1532}
1533
47d45203 1534static struct thread_data *get_new_job(int global)
892199bd 1535{
4240cfa1
JA
1536 struct thread_data *td;
1537
47d45203
JA
1538 if (global)
1539 return &def_thread;
8867c0a8 1540 if (thread_number >= max_jobs)
4240cfa1
JA
1541 return NULL;
1542
1543 td = &threads[thread_number++];
fc24389f 1544 memset(td, 0, sizeof(*td));
892199bd 1545
e4ed35c3 1546 td->fd = -1;
86184d14 1547 td->thread_number = thread_number;
76cb7b42 1548
47d45203 1549 td->ddir = def_thread.ddir;
76cb7b42
JA
1550 td->ioprio = def_thread.ioprio;
1551 td->sequential = def_thread.sequential;
47d45203 1552 td->bs = def_thread.bs;
8c033f93
JA
1553 td->min_bs = def_thread.min_bs;
1554 td->max_bs = def_thread.max_bs;
47d45203 1555 td->odirect = def_thread.odirect;
e97712ed 1556 td->thinktime = def_thread.thinktime;
76cb7b42
JA
1557 td->fsync_blocks = def_thread.fsync_blocks;
1558 td->start_delay = def_thread.start_delay;
67903a2e 1559 td->timeout = def_thread.timeout;
76cb7b42 1560 td->use_aio = def_thread.use_aio;
02983297
JA
1561 td->create_file = def_thread.create_file;
1562 td->overwrite = def_thread.overwrite;
b95799ca 1563 td->invalidate_cache = def_thread.invalidate_cache;
76cb7b42 1564 td->file_size = def_thread.file_size;
9b5cf6c0 1565 td->file_offset = def_thread.file_offset;
76cb7b42
JA
1566 td->rate = def_thread.rate;
1567 td->ratemin = def_thread.ratemin;
1568 td->ratecycle = def_thread.ratecycle;
1569 td->aio_depth = def_thread.aio_depth;
99c6704f
JA
1570 td->sync_io = def_thread.sync_io;
1571 td->mem_type = def_thread.mem_type;
1d035750 1572 td->bw_avg_time = def_thread.bw_avg_time;
fc097bfe
JA
1573 td->create_serialize = def_thread.create_serialize;
1574 td->create_fsync = def_thread.create_fsync;
b6794fbf 1575 td->loops = def_thread.loops;
cfc702bd 1576 td->verify = def_thread.verify;
47d45203 1577 memcpy(&td->cpumask, &def_thread.cpumask, sizeof(td->cpumask));
f737299d
JA
1578
1579 return td;
1580}
1581
4240cfa1
JA
1582static void put_job(struct thread_data *td)
1583{
1584 memset(&threads[td->thread_number - 1], 0, sizeof(*td));
1585 thread_number--;
1586}
1587
5c24b2c4
JA
1588static int add_job(struct thread_data *td, const char *filename, int prioclass,
1589 int prio)
f737299d 1590{
47d45203
JA
1591 if (td == &def_thread)
1592 return 0;
1593
f737299d 1594 strcpy(td->file_name, filename);
4240cfa1 1595 sem_init(&td->mutex, 1, 0);
f737299d
JA
1596 td->ioprio = (prioclass << IOPRIO_CLASS_SHIFT) | prio;
1597
57d753e3
JA
1598 td->clat_stat.min_val = ULONG_MAX;
1599 td->slat_stat.min_val = ULONG_MAX;
1600 td->bw_stat.min_val = ULONG_MAX;
1601
8dbff0b1
JA
1602 run_str[td->thread_number - 1] = 'P';
1603
4ac89145
JA
1604 if (td->use_aio && !td->aio_depth)
1605 td->aio_depth = 1;
43000118 1606
8bdcfab5
JA
1607 if (td->min_bs == -1)
1608 td->min_bs = td->bs;
1609 if (td->max_bs == -1)
1610 td->max_bs = td->bs;
1611
4240cfa1
JA
1612 if (setup_rate(td))
1613 return -1;
f737299d 1614
a0a9b35b
JA
1615 if (write_lat_log)
1616 setup_log(&td->lat_log);
1617 if (write_bw_log)
1618 setup_log(&td->bw_log);
1619
7889f07b 1620 printf("Client%d: file=%s, rw=%d, prio=%d/%d, seq=%d, odir=%d, bs=%d-%d, rate=%d, aio=%d, aio_depth=%d\n", td->thread_number, filename, td->ddir, prioclass, prio, td->sequential, td->odirect, td->min_bs, td->max_bs, td->rate, td->use_aio, td->aio_depth);
4240cfa1 1621 return 0;
892199bd
JA
1622}
1623
18e0b78c
JA
1624static void fill_cpu_mask(cpu_set_t cpumask, int cpu)
1625{
f737299d 1626 unsigned int i;
18e0b78c
JA
1627
1628 CPU_ZERO(&cpumask);
1629
1630 for (i = 0; i < sizeof(int) * 8; i++) {
1631 if ((1 << i) & cpu)
1632 CPU_SET(i, &cpumask);
1633 }
1634}
1635
7889f07b
JA
1636unsigned long get_mult(char c)
1637{
1638 switch (c) {
1639 case 'k':
1640 case 'K':
1641 return 1024;
1642 case 'm':
1643 case 'M':
1644 return 1024 * 1024;
1645 case 'g':
1646 case 'G':
1647 return 1024 * 1024 * 1024;
1648 default:
1649 return 1;
1650 }
1651}
1652
02983297
JA
1653/*
1654 * convert string after '=' into decimal value, noting any size suffix
1655 */
1656static int str_cnv(char *p, unsigned long long *val)
1657{
02983297
JA
1658 char *str;
1659 int len;
1660
1661 str = strstr(p, "=");
1662 if (!str)
1663 return 1;
1664
1665 str++;
1666 len = strlen(str);
02983297
JA
1667
1668 *val = strtoul(str, NULL, 10);
1669 if (*val == ULONG_MAX && errno == ERANGE)
1670 return 1;
1671
7889f07b 1672 *val *= get_mult(str[len - 2]);
02983297 1673 return 0;
02983297
JA
1674}
1675
02983297
JA
1676static int check_strcnv(char *p, char *name, unsigned long long *val)
1677{
1678 if (!strstr(p, name))
1679 return 1;
1680
1681 return str_cnv(p, val);
1682}
1683
99c6704f
JA
1684static int check_str(char *p, char *name, char *option)
1685{
1686 char *s = strstr(p, name);
1687
1688 if (!s)
1689 return 1;
1690
1691 s += strlen(name);
1692 if (strstr(s, option))
1693 return 0;
1694
1695 return 1;
1696}
1697
7889f07b
JA
1698static int check_range(char *p, char *name, unsigned long *s, unsigned long *e)
1699{
1700 char str[128];
1701 char s1, s2;
1702
1703 sprintf(str, "%s=%%lu%%c-%%lu%%c", name);
1704 if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1705 *s *= get_mult(s1);
1706 *e *= get_mult(s2);
1707 return 0;
1708 }
1709
1710 sprintf(str, "%s = %%lu%%c-%%lu%%c", name);
1711 if (sscanf(p, str, s, &s1, e, &s2) == 4) {
1712 *s *= get_mult(s1);
1713 *e *= get_mult(s2);
1714 return 0;
1715 }
1716
1717 sprintf(str, "%s=%%lu-%%lu", name);
1718 if (sscanf(p, str, s, e) == 2)
1719 return 0;
1720
1721 sprintf(str, "%s = %%lu-%%lu", name);
1722 if (sscanf(p, str, s, e) == 2)
1723 return 0;
1724
1725 return 1;
1726
1727}
1728
5c24b2c4 1729static int check_int(char *p, char *name, unsigned int *val)
7dd1389e
JA
1730{
1731 char str[128];
1732
1733 sprintf(str, "%s=%%d", name);
1734 if (sscanf(p, str, val) == 1)
1735 return 0;
1736
1737 sprintf(str, "%s = %%d", name);
1738 if (sscanf(p, str, val) == 1)
1739 return 0;
1740
1741 return 1;
1742}
1743
7292613b 1744static int is_empty_or_comment(char *line)
7dd1389e
JA
1745{
1746 unsigned int i;
1747
7292613b 1748 for (i = 0; i < strlen(line); i++) {
7292613b 1749 if (line[i] == ';')
47d45203
JA
1750 return 1;
1751 if (!isspace(line[i]) && !iscntrl(line[i]))
7292613b
JA
1752 return 0;
1753 }
7dd1389e
JA
1754
1755 return 1;
1756}
1757
5c24b2c4 1758static int parse_jobs_ini(char *file)
7dd1389e 1759{
47d45203 1760 unsigned int prioclass, prio, cpu, global;
8c033f93 1761 unsigned long long ull;
7889f07b 1762 unsigned long ul1, ul2;
f737299d 1763 struct thread_data *td;
7dd1389e
JA
1764 char *string, *name;
1765 fpos_t off;
1766 FILE *f;
1767 char *p;
1768
1769 f = fopen(file, "r");
1770 if (!f) {
1771 perror("fopen");
4240cfa1 1772 return 1;
7dd1389e
JA
1773 }
1774
1775 string = malloc(4096);
1776 name = malloc(256);
1777
7dd1389e 1778 while ((p = fgets(string, 4096, f)) != NULL) {
7292613b
JA
1779 if (is_empty_or_comment(p))
1780 continue;
7dd1389e
JA
1781 if (sscanf(p, "[%s]", name) != 1)
1782 continue;
1783
47d45203
JA
1784 global = !strncmp(name, "global", 6);
1785
7dd1389e
JA
1786 name[strlen(name) - 1] = '\0';
1787
47d45203 1788 td = get_new_job(global);
4240cfa1
JA
1789 if (!td)
1790 break;
f737299d 1791
7dd1389e 1792 prioclass = 2;
f737299d 1793 prio = 4;
7dd1389e
JA
1794
1795 fgetpos(f, &off);
1796 while ((p = fgets(string, 4096, f)) != NULL) {
7292613b 1797 if (is_empty_or_comment(p))
e6402082
JA
1798 continue;
1799 if (strstr(p, "["))
7dd1389e 1800 break;
f737299d 1801 if (!check_int(p, "rw", &td->ddir)) {
7dd1389e
JA
1802 fgetpos(f, &off);
1803 continue;
1804 }
1805 if (!check_int(p, "prio", &prio)) {
1806 fgetpos(f, &off);
1807 continue;
1808 }
1809 if (!check_int(p, "prioclass", &prioclass)) {
1810 fgetpos(f, &off);
1811 continue;
1812 }
f737299d 1813 if (!check_int(p, "direct", &td->odirect)) {
7dd1389e
JA
1814 fgetpos(f, &off);
1815 continue;
1816 }
f737299d 1817 if (!check_int(p, "rate", &td->rate)) {
7dd1389e
JA
1818 fgetpos(f, &off);
1819 continue;
1820 }
4240cfa1
JA
1821 if (!check_int(p, "ratemin", &td->ratemin)) {
1822 fgetpos(f, &off);
1823 continue;
1824 }
1825 if (!check_int(p, "ratecycle", &td->ratecycle)) {
1826 fgetpos(f, &off);
1827 continue;
1828 }
e97712ed 1829 if (!check_int(p, "thinktime", &td->thinktime)) {
7dd1389e
JA
1830 fgetpos(f, &off);
1831 continue;
1832 }
18e0b78c 1833 if (!check_int(p, "cpumask", &cpu)) {
f737299d 1834 fill_cpu_mask(td->cpumask, cpu);
18e0b78c
JA
1835 fgetpos(f, &off);
1836 continue;
1837 }
4240cfa1
JA
1838 if (!check_int(p, "fsync", &td->fsync_blocks)) {
1839 fgetpos(f, &off);
1840 continue;
1841 }
fc24389f
JA
1842 if (!check_int(p, "startdelay", &td->start_delay)) {
1843 fgetpos(f, &off);
1844 continue;
1845 }
67903a2e
JA
1846 if (!check_int(p, "timeout", &td->timeout)) {
1847 fgetpos(f, &off);
1848 continue;
1849 }
b95799ca
JA
1850 if (!check_int(p, "invalidate",&td->invalidate_cache)) {
1851 fgetpos(f, &off);
1852 continue;
1853 }
43000118
JA
1854 if (!check_int(p, "aio_depth", &td->aio_depth)) {
1855 fgetpos(f, &off);
1856 continue;
1857 }
74b4b5fb
JA
1858 if (!check_int(p, "sync", &td->sync_io)) {
1859 fgetpos(f, &off);
1860 continue;
1861 }
1d035750
JA
1862 if (!check_int(p, "bwavgtime", &td->bw_avg_time)) {
1863 fgetpos(f, &off);
1864 continue;
1865 }
fc097bfe
JA
1866 if (!check_int(p, "create_serialize", &td->create_serialize)) {
1867 fgetpos(f, &off);
1868 continue;
1869 }
1870 if (!check_int(p, "create_fsync", &td->create_fsync)) {
1871 fgetpos(f, &off);
1872 continue;
1873 }
b6794fbf
JA
1874 if (!check_int(p, "loops", &td->loops)) {
1875 fgetpos(f, &off);
1876 continue;
1877 }
e8457004
JA
1878 if (!check_int(p, "verify", &td->verify)) {
1879 fgetpos(f, &off);
1880 continue;
1881 }
7889f07b
JA
1882 if (!check_range(p, "bsrange", &ul1, &ul2)) {
1883 td->min_bs = ul1;
1884 td->max_bs = ul2;
1885 fgetpos(f, &off);
1886 continue;
1887 }
8c033f93
JA
1888 if (!check_strcnv(p, "bs", &ull)) {
1889 td->bs = ull;
1890 fgetpos(f, &off);
1891 continue;
1892 }
02983297
JA
1893 if (!check_strcnv(p, "size", &td->file_size)) {
1894 fgetpos(f, &off);
1895 continue;
1896 }
1897 if (!check_strcnv(p, "offset", &td->file_offset)) {
1898 fgetpos(f, &off);
1899 continue;
1900 }
99c6704f 1901 if (!check_str(p, "mem", "malloc")) {
99c6704f
JA
1902 td->mem_type = MEM_MALLOC;
1903 fgetpos(f, &off);
1904 continue;
1905 }
1906 if (!check_str(p, "mem", "shm")) {
99c6704f
JA
1907 td->mem_type = MEM_SHM;
1908 fgetpos(f, &off);
1909 continue;
1910 }
43000118 1911 if (!strncmp(p, "sequential", 10)) {
f737299d 1912 td->sequential = 1;
7dd1389e
JA
1913 fgetpos(f, &off);
1914 continue;
1915 }
43000118 1916 if (!strncmp(p, "random", 6)) {
f737299d 1917 td->sequential = 0;
7dd1389e
JA
1918 fgetpos(f, &off);
1919 continue;
1920 }
43000118
JA
1921 if (!strncmp(p, "aio", 3)) {
1922 td->use_aio = 1;
1923 fgetpos(f, &off);
1924 continue;
1925 }
02983297
JA
1926 if (!strncmp(p, "create", 6)) {
1927 td->create_file = 1;
1928 fgetpos(f, &off);
1929 continue;
1930 }
1931 if (!strncmp(p, "overwrite", 9)) {
1932 td->overwrite = 1;
1933 fgetpos(f, &off);
1934 continue;
1935 }
98dd52d6
JA
1936 if (!strncmp(p, "exitall", 7)) {
1937 exitall_on_terminate = 1;
1938 fgetpos(f, &off);
1939 continue;
1940 }
e6402082 1941 printf("Client%d: bad option %s\n",td->thread_number,p);
7dd1389e
JA
1942 }
1943 fsetpos(f, &off);
1944
4240cfa1
JA
1945 if (add_job(td, name, prioclass, prio))
1946 put_job(td);
7dd1389e
JA
1947 }
1948
1949 free(string);
1950 free(name);
fc7d63df 1951 fclose(f);
4240cfa1 1952 return 0;
7dd1389e
JA
1953}
1954
5c24b2c4 1955static int parse_options(int argc, char *argv[])
892199bd 1956{
01c4d8de 1957 int i;
892199bd
JA
1958
1959 for (i = 1; i < argc; i++) {
1960 char *parm = argv[i];
1961
1962 if (parm[0] != '-')
1963 break;
1964
1965 parm++;
1966 switch (*parm) {
1967 case 's':
1968 parm++;
47d45203 1969 def_thread.sequential = !!atoi(parm);
892199bd
JA
1970 break;
1971 case 'b':
1972 parm++;
47d45203
JA
1973 def_thread.bs = atoi(parm);
1974 def_thread.bs <<= 10;
1975 if (!def_thread.bs) {
4240cfa1 1976 printf("bad block size\n");
47d45203 1977 def_thread.bs = DEF_BS;
4240cfa1 1978 }
892199bd
JA
1979 break;
1980 case 't':
1981 parm++;
47d45203 1982 def_thread.timeout = atoi(parm);
892199bd 1983 break;
892199bd
JA
1984 case 'r':
1985 parm++;
1986 repeatable = !!atoi(parm);
1987 break;
02bdd9ba
JA
1988 case 'R':
1989 parm++;
1990 rate_quit = !!atoi(parm);
1991 break;
892199bd
JA
1992 case 'o':
1993 parm++;
47d45203 1994 def_thread.odirect = !!atoi(parm);
892199bd 1995 break;
7dd1389e
JA
1996 case 'f':
1997 if (i + 1 >= argc) {
1998 printf("-f needs file as arg\n");
1999 break;
2000 }
2001 ini_file = strdup(argv[i+1]);
a642279f 2002 i++;
7dd1389e 2003 break;
a0a9b35b
JA
2004 case 'l':
2005 write_lat_log = 1;
2006 break;
2007 case 'w':
2008 write_bw_log = 1;
2009 break;
892199bd 2010 default:
7dd1389e 2011 printf("bad option %s\n", argv[i]);
892199bd
JA
2012 break;
2013 }
2014 }
2015
892199bd
JA
2016 return i;
2017}
2018
3f39453a 2019static void print_thread_status(struct thread_data *td, int nr_running,
8dbff0b1 2020 int t_rate, int m_rate)
3f39453a 2021{
3f39453a
JA
2022 printf("Threads now running: %d", nr_running);
2023 if (m_rate || t_rate)
2024 printf(", commitrate %d/%dKiB/sec", t_rate, m_rate);
8dbff0b1
JA
2025 printf(" : [%s]\r", run_str);
2026 fflush(stdout);
3f39453a
JA
2027}
2028
40ef7f64
JA
2029static void check_str_update(struct thread_data *td, int n, int t, int m)
2030{
2031 char c = run_str[td->thread_number - 1];
2032
2033 if (td->runstate == td->old_runstate)
2034 return;
2035
2036 switch (td->runstate) {
2037 case TD_REAPED:
2038 c = '_';
2039 break;
f4bb2243
JA
2040 case TD_EXITED:
2041 c = 'E';
2042 break;
40ef7f64
JA
2043 case TD_RUNNING:
2044 c = '+';
2045 break;
2046 case TD_VERIFYING:
2047 c = 'V';
2048 break;
2049 case TD_CREATED:
2050 c = 'C';
2051 break;
2052 case TD_NOT_CREATED:
2053 c = 'P';
2054 break;
2055 default:
2056 printf("state %d\n", td->runstate);
2057 }
2058
2059 run_str[td->thread_number - 1] = c;
2060 print_thread_status(td, n, t, m);
2061 td->old_runstate = td->runstate;
2062}
2063
213b446c 2064static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
02bdd9ba 2065{
213b446c 2066 int i;
02bdd9ba 2067
3f39453a
JA
2068 /*
2069 * reap exited threads (TD_EXITED -> TD_REAPED)
2070 */
02bdd9ba
JA
2071 for (i = 0; i < thread_number; i++) {
2072 struct thread_data *td = &threads[i];
2073
40ef7f64
JA
2074 check_str_update(td, *nr_running, *t_rate, *m_rate);
2075
213b446c
JA
2076 if (td->runstate != TD_EXITED)
2077 continue;
02bdd9ba 2078
40ef7f64 2079 td_set_runstate(td, TD_REAPED);
213b446c
JA
2080 waitpid(td->pid, NULL, 0);
2081 (*nr_running)--;
2082 (*m_rate) -= td->ratemin;
2083 (*t_rate) -= td->rate;
40ef7f64 2084 check_str_update(td, *nr_running, *t_rate, *m_rate);
e6402082
JA
2085
2086 if (td->terminate)
2087 continue;
213b446c 2088 }
02bdd9ba
JA
2089}
2090
fc24389f
JA
2091static void run_threads(char *argv[])
2092{
be33abe4 2093 struct timeval genesis;
fc24389f
JA
2094 struct thread_data *td;
2095 unsigned long spent;
213b446c 2096 int i, todo, nr_running, m_rate, t_rate;
fc24389f 2097
fc24389f
JA
2098 printf("Starting %d threads\n", thread_number);
2099 fflush(stdout);
2100
7292613b
JA
2101 signal(SIGINT, sig_handler);
2102
fc24389f 2103 todo = thread_number;
02bdd9ba 2104 nr_running = 0;
213b446c 2105 m_rate = t_rate = 0;
fc24389f 2106
8bdcfab5
JA
2107 for (i = 0; i < thread_number; i++) {
2108 td = &threads[i];
2109
fc097bfe
JA
2110 if (!td->create_serialize)
2111 continue;
2112
8bdcfab5
JA
2113 /*
2114 * do file setup here so it happens sequentially,
2115 * we don't want X number of threads getting their
2116 * client data interspersed on disk
2117 */
2118 if (setup_file(td)) {
40ef7f64 2119 td_set_runstate(td, TD_REAPED);
8bdcfab5
JA
2120 todo--;
2121 }
2122 }
2123
2124 gettimeofday(&genesis, NULL);
2125
213b446c 2126 while (todo) {
3f39453a
JA
2127 /*
2128 * create threads (TD_NOT_CREATED -> TD_CREATED)
2129 */
fc24389f
JA
2130 for (i = 0; i < thread_number; i++) {
2131 td = &threads[i];
2132
02bdd9ba 2133 if (td->runstate != TD_NOT_CREATED)
fc24389f
JA
2134 continue;
2135
213b446c
JA
2136 /*
2137 * never got a chance to start, killed by other
2138 * thread for some reason
2139 */
2140 if (td->terminate) {
2141 todo--;
2142 continue;
2143 }
2144
fc24389f 2145 if (td->start_delay) {
be33abe4 2146 spent = mtime_since_now(&genesis);
fc24389f
JA
2147
2148 if (td->start_delay * 1000 > spent)
2149 continue;
2150 }
2151
40ef7f64
JA
2152 td_set_runstate(td, TD_CREATED);
2153 check_str_update(td, nr_running, t_rate, m_rate);
fc24389f
JA
2154 sem_init(&startup_sem, 1, 1);
2155 todo--;
2156
2157 if (fork())
2158 sem_wait(&startup_sem);
2159 else {
2160 thread_main(shm_id, i, argv);
2161 exit(0);
2162 }
2163 }
2164
3f39453a 2165 /*
e8457004 2166 * start created threads (TD_CREATED -> TD_RUNNING)
3f39453a 2167 */
fc24389f
JA
2168 for (i = 0; i < thread_number; i++) {
2169 struct thread_data *td = &threads[i];
2170
3f39453a
JA
2171 if (td->runstate != TD_CREATED)
2172 continue;
2173
40ef7f64 2174 td_set_runstate(td, TD_RUNNING);
3f39453a
JA
2175 nr_running++;
2176 m_rate += td->ratemin;
2177 t_rate += td->rate;
40ef7f64 2178 check_str_update(td, nr_running, t_rate, m_rate);
3f39453a 2179 sem_post(&td->mutex);
fc24389f
JA
2180 }
2181
e8457004
JA
2182 for (i = 0; i < thread_number; i++) {
2183 struct thread_data *td = &threads[i];
2184
2185 if (td->runstate == TD_RUNNING)
2186 run_str[td->thread_number - 1] = '+';
2187 else if (td->runstate == TD_VERIFYING)
2188 run_str[td->thread_number - 1] = 'V';
2189 else
2190 continue;
2191
40ef7f64 2192 check_str_update(td, nr_running, t_rate, m_rate);
e8457004
JA
2193 }
2194
213b446c 2195 reap_threads(&nr_running, &t_rate, &m_rate);
02bdd9ba 2196
fc24389f
JA
2197 if (todo)
2198 usleep(100000);
2199 }
02bdd9ba
JA
2200
2201 while (nr_running) {
213b446c 2202 reap_threads(&nr_running, &t_rate, &m_rate);
02bdd9ba
JA
2203 usleep(10000);
2204 }
fc24389f
JA
2205}
2206
8867c0a8 2207int setup_thread_area(void)
892199bd 2208{
8867c0a8
JA
2209 /*
2210 * 1024 is too much on some machines, scale max_jobs if
2211 * we get a failure that looks like too large a shm segment
2212 */
2213 do {
2214 int s = max_jobs * sizeof(struct thread_data);
18e0b78c 2215
8867c0a8
JA
2216 shm_id = shmget(0, s, IPC_CREAT | 0600);
2217 if (shm_id != -1)
2218 break;
2219 if (errno != EINVAL) {
2220 perror("shmget");
2221 break;
2222 }
2223
2224 max_jobs >>= 1;
d4fac444 2225 } while (max_jobs);
8867c0a8
JA
2226
2227 if (shm_id == -1)
892199bd 2228 return 1;
892199bd
JA
2229
2230 threads = shmat(shm_id, NULL, 0);
8867c0a8 2231 if (threads == (void *) -1) {
86184d14
JA
2232 perror("shmat");
2233 return 1;
2234 }
892199bd
JA
2235
2236 atexit(free_shm);
8867c0a8
JA
2237 return 0;
2238}
2239
2240int main(int argc, char *argv[])
2241{
8c033f93 2242 static unsigned long max_run[2], min_run[2];
57d753e3 2243 static unsigned long max_bw[2], min_bw[2];
22334044 2244 static unsigned long io_mb[2], agg[2];
8867c0a8
JA
2245 int i;
2246
2247 if (setup_thread_area())
2248 return 1;
892199bd 2249
47d45203 2250 if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &def_thread.cpumask) == -1) {
4240cfa1
JA
2251 perror("sched_getaffinity");
2252 return 1;
2253 }
2254
47d45203
JA
2255 /*
2256 * fill globals
2257 */
2258 def_thread.ddir = DDIR_READ;
2259 def_thread.bs = DEF_BS;
7889f07b
JA
2260 def_thread.min_bs = -1;
2261 def_thread.max_bs = -1;
02983297 2262 def_thread.odirect = DEF_ODIRECT;
47d45203 2263 def_thread.ratecycle = DEF_RATE_CYCLE;
02983297 2264 def_thread.sequential = DEF_SEQUENTIAL;
47d45203 2265 def_thread.timeout = DEF_TIMEOUT;
02983297
JA
2266 def_thread.create_file = DEF_CREATE;
2267 def_thread.overwrite = DEF_OVERWRITE;
b95799ca 2268 def_thread.invalidate_cache = DEF_INVALIDATE;
99c6704f
JA
2269 def_thread.sync_io = DEF_SYNCIO;
2270 def_thread.mem_type = MEM_MALLOC;
1d035750 2271 def_thread.bw_avg_time = DEF_BWAVGTIME;
fc097bfe
JA
2272 def_thread.create_serialize = DEF_CREATE_SER;
2273 def_thread.create_fsync = DEF_CREATE_FSYNC;
b6794fbf 2274 def_thread.loops = DEF_LOOPS;
cfc702bd 2275 def_thread.verify = DEF_VERIFY;
47d45203 2276
892199bd 2277 i = parse_options(argc, argv);
7dd1389e 2278
5961d92c
JA
2279 if (!ini_file) {
2280 printf("Need job file\n");
2281 return 1;
2282 }
2283
2284 if (parse_jobs_ini(ini_file))
2285 return 1;
7dd1389e 2286
4240cfa1
JA
2287 if (!thread_number) {
2288 printf("Nothing to do\n");
2289 return 1;
2290 }
7dd1389e 2291
fc24389f 2292 run_threads(argv);
892199bd 2293
892199bd
JA
2294 min_bw[0] = min_run[0] = ~0UL;
2295 min_bw[1] = min_run[1] = ~0UL;
22334044
JA
2296 io_mb[0] = io_mb[1] = 0;
2297 agg[0] = agg[1] = 0;
892199bd
JA
2298 for (i = 0; i < thread_number; i++) {
2299 struct thread_data *td = &threads[i];
2300 unsigned long bw = 0;
2301
22334044
JA
2302 if (!td->error) {
2303 if (td->runtime < min_run[td->ddir])
2304 min_run[td->ddir] = td->runtime;
2305 if (td->runtime > max_run[td->ddir])
2306 max_run[td->ddir] = td->runtime;
892199bd 2307
892199bd 2308 if (td->runtime)
22334044
JA
2309 bw = td->io_kb * 1024 / td->runtime;
2310 if (bw < min_bw[td->ddir])
2311 min_bw[td->ddir] = bw;
2312 if (bw > max_bw[td->ddir])
2313 max_bw[td->ddir] = bw;
2314
2315 io_mb[td->ddir] += td->io_kb >> 10;
892199bd
JA
2316 }
2317
2318 show_thread_status(td);
2319 }
22334044
JA
2320
2321 if (max_run[0])
2322 agg[0] = io_mb[0] * 1024 * 1000 / max_run[0];
2323 if (max_run[1])
2324 agg[1] = io_mb[1] * 1024 * 1000 / max_run[1];
2325
57d753e3 2326 printf("\nRun status:\n");
892199bd 2327 if (max_run[DDIR_READ])
22334044 2328 printf(" READ: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[0], agg[0], min_bw[0], max_bw[0], min_run[0], max_run[0]);
892199bd 2329 if (max_run[DDIR_WRITE])
22334044 2330 printf(" WRITE: io=%luMiB, aggrb=%lu, minb=%lu, maxb=%lu, mint=%lumsec, maxt=%lumsec\n", io_mb[1], agg[1], min_bw[1], max_bw[1], min_run[1], max_run[1]);
fc24389f 2331
892199bd
JA
2332 return 0;
2333}