[PATCH] disk zone example needs write_bw_log
[fio.git] / fio.c
CommitLineData
ebac4655
JA
1/*
2 * fio - the flexible io tester
3 *
4 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
aae22ca7 5 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
ebac4655
JA
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 *
21 */
ebac4655
JA
22#include <unistd.h>
23#include <fcntl.h>
24#include <string.h>
ebac4655
JA
25#include <signal.h>
26#include <time.h>
ebac4655 27#include <assert.h>
ebac4655
JA
28#include <sys/stat.h>
29#include <sys/wait.h>
30#include <sys/ipc.h>
31#include <sys/shm.h>
32#include <sys/ioctl.h>
33#include <sys/mman.h>
34
35#include "fio.h"
36#include "os.h"
37
38#define MASK (4095)
39
40#define ALIGN(buf) (char *) (((unsigned long) (buf) + MASK) & ~(MASK))
41
42int groupid = 0;
43int thread_number = 0;
ebac4655 44int shm_id = 0;
53cdc686 45int temp_stall_ts;
c1d5725e 46char *fio_inst_prefix = _INST_PREFIX;
ebac4655 47
c04f7ec3
JA
48extern unsigned long long mlock_size;
49
3d60d1ed 50#define should_fsync(td) ((td_write(td) || td_rw(td)) && (!(td)->odirect || (td)->override_sync))
ebac4655 51
bbfd6b00 52static volatile int startup_sem;
ebac4655
JA
53
54#define TERMINATE_ALL (-1)
75154845 55#define JOB_START_TIMEOUT (5 * 1000)
ebac4655
JA
56
57static void terminate_threads(int group_id)
58{
34572e28 59 struct thread_data *td;
ebac4655
JA
60 int i;
61
34572e28 62 for_each_td(td, i) {
ebac4655
JA
63 if (group_id == TERMINATE_ALL || groupid == td->groupid) {
64 td->terminate = 1;
65 td->start_delay = 0;
66 }
67 }
68}
69
70static void sig_handler(int sig)
71{
72 switch (sig) {
73 case SIGALRM:
74 update_io_ticks();
75 disk_util_timer_arm();
76 print_thread_status();
77 break;
78 default:
79 printf("\nfio: terminating on signal\n");
80 fflush(stdout);
81 terminate_threads(TERMINATE_ALL);
82 break;
83 }
84}
85
906c8d75
JA
86/*
87 * The ->file_map[] contains a map of blocks we have or have not done io
88 * to yet. Used to make sure we cover the entire range in a fair fashion.
89 */
53cdc686
JA
90static int random_map_free(struct thread_data *td, struct fio_file *f,
91 unsigned long long block)
ebac4655 92{
53cdc686
JA
93 unsigned int idx = RAND_MAP_IDX(td, f, block);
94 unsigned int bit = RAND_MAP_BIT(td, f, block);
ebac4655 95
53cdc686 96 return (f->file_map[idx] & (1UL << bit)) == 0;
ebac4655
JA
97}
98
906c8d75
JA
99/*
100 * Return the next free block in the map.
101 */
53cdc686
JA
102static int get_next_free_block(struct thread_data *td, struct fio_file *f,
103 unsigned long long *b)
ebac4655
JA
104{
105 int i;
106
107 *b = 0;
108 i = 0;
53cdc686
JA
109 while ((*b) * td->min_bs < f->file_size) {
110 if (f->file_map[i] != -1UL) {
111 *b += ffz(f->file_map[i]);
ebac4655
JA
112 return 0;
113 }
114
115 *b += BLOCKS_PER_MAP;
116 i++;
117 }
118
119 return 1;
120}
121
906c8d75
JA
122/*
123 * Mark a given offset as used in the map.
124 */
53cdc686
JA
125static void mark_random_map(struct thread_data *td, struct fio_file *f,
126 struct io_u *io_u)
ebac4655 127{
200bc855 128 unsigned long long block = io_u->offset / (unsigned long long) td->min_bs;
ebac4655
JA
129 unsigned int blocks = 0;
130
131 while (blocks < (io_u->buflen / td->min_bs)) {
132 unsigned int idx, bit;
133
53cdc686 134 if (!random_map_free(td, f, block))
ebac4655
JA
135 break;
136
53cdc686
JA
137 idx = RAND_MAP_IDX(td, f, block);
138 bit = RAND_MAP_BIT(td, f, block);
ebac4655 139
53cdc686 140 assert(idx < f->num_maps);
ebac4655 141
53cdc686 142 f->file_map[idx] |= (1UL << bit);
ebac4655
JA
143 block++;
144 blocks++;
145 }
146
147 if ((blocks * td->min_bs) < io_u->buflen)
148 io_u->buflen = blocks * td->min_bs;
149}
150
906c8d75
JA
151/*
152 * For random io, generate a random new block and see if it's used. Repeat
153 * until we find a free one. For sequential io, just return the end of
154 * the last io issued.
155 */
53cdc686
JA
156static int get_next_offset(struct thread_data *td, struct fio_file *f,
157 unsigned long long *offset)
20dc95c4
JA
158{
159 unsigned long long b, rb;
160 long r;
161
162 if (!td->sequential) {
085227ad 163 unsigned long long max_blocks = td->io_size / td->min_bs;
20dc95c4
JA
164 int loops = 50;
165
166 do {
6dfd46b9 167 r = os_random_long(&td->random_state);
085227ad 168 b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0));
53cdc686 169 rb = b + (f->file_offset / td->min_bs);
20dc95c4 170 loops--;
53cdc686 171 } while (!random_map_free(td, f, rb) && loops);
20dc95c4
JA
172
173 if (!loops) {
53cdc686 174 if (get_next_free_block(td, f, &b))
20dc95c4
JA
175 return 1;
176 }
177 } else
53cdc686 178 b = f->last_pos / td->min_bs;
20dc95c4 179
53cdc686
JA
180 *offset = (b * td->min_bs) + f->file_offset;
181 if (*offset > f->file_size)
20dc95c4
JA
182 return 1;
183
184 return 0;
185}
186
187static unsigned int get_next_buflen(struct thread_data *td)
188{
189 unsigned int buflen;
190 long r;
191
192 if (td->min_bs == td->max_bs)
193 buflen = td->min_bs;
194 else {
6dfd46b9 195 r = os_random_long(&td->bsrange_state);
20dc95c4
JA
196 buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0));
197 buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1);
198 }
199
b2a15192
JA
200 if (buflen > td->io_size - td->this_io_bytes[td->ddir]) {
201 /*
202 * if using direct/raw io, we may not be able to
203 * shrink the size. so just fail it.
204 */
205 if (td->io_ops->flags & FIO_RAWIO)
206 return 0;
207
20dc95c4 208 buflen = td->io_size - td->this_io_bytes[td->ddir];
b2a15192 209 }
20dc95c4
JA
210
211 return buflen;
212}
213
906c8d75
JA
214/*
215 * Check if we are above the minimum rate given.
216 */
ebac4655
JA
217static int check_min_rate(struct thread_data *td, struct timeval *now)
218{
219 unsigned long spent;
220 unsigned long rate;
221 int ddir = td->ddir;
222
223 /*
224 * allow a 2 second settle period in the beginning
225 */
226 if (mtime_since(&td->start, now) < 2000)
227 return 0;
228
229 /*
230 * if rate blocks is set, sample is running
231 */
232 if (td->rate_bytes) {
233 spent = mtime_since(&td->lastrate, now);
234 if (spent < td->ratecycle)
235 return 0;
236
237 rate = (td->this_io_bytes[ddir] - td->rate_bytes) / spent;
238 if (rate < td->ratemin) {
eb8bbf48 239 fprintf(f_out, "%s: min rate %d not met, got %ldKiB/sec\n", td->name, td->ratemin, rate);
ebac4655
JA
240 if (rate_quit)
241 terminate_threads(td->groupid);
242 return 1;
243 }
244 }
245
246 td->rate_bytes = td->this_io_bytes[ddir];
247 memcpy(&td->lastrate, now, sizeof(*now));
248 return 0;
249}
250
251static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
252{
253 if (!td->timeout)
254 return 0;
255 if (mtime_since(&td->epoch, t) >= td->timeout * 1000)
256 return 1;
257
258 return 0;
259}
260
906c8d75
JA
261/*
262 * Return the data direction for the next io_u. If the job is a
263 * mixed read/write workload, check the rwmix cycle and switch if
264 * necessary.
265 */
a6ccc7be 266static int get_rw_ddir(struct thread_data *td)
3d60d1ed 267{
a6ccc7be
JA
268 if (td_rw(td)) {
269 struct timeval now;
270 unsigned long elapsed;
3d60d1ed 271
a6ccc7be
JA
272 gettimeofday(&now, NULL);
273 elapsed = mtime_since_now(&td->rwmix_switch);
3d60d1ed 274
a6ccc7be
JA
275 /*
276 * Check if it's time to seed a new data direction.
277 */
278 if (elapsed >= td->rwmixcycle) {
0ab8db89 279 unsigned int v;
a6ccc7be 280 long r;
3d60d1ed 281
c1ee2ca4
JA
282 r = os_random_long(&td->rwmix_state);
283 v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0)));
a6ccc7be
JA
284 if (v < td->rwmixread)
285 td->rwmix_ddir = DDIR_READ;
286 else
287 td->rwmix_ddir = DDIR_WRITE;
288 memcpy(&td->rwmix_switch, &now, sizeof(now));
289 }
290 return td->rwmix_ddir;
291 } else if (td_read(td))
3d60d1ed
JA
292 return DDIR_READ;
293 else
294 return DDIR_WRITE;
295}
296
aea47d44 297static int td_io_prep(struct thread_data *td, struct io_u *io_u)
20dc95c4 298{
2866c82d 299 if (td->io_ops->prep && td->io_ops->prep(td, io_u))
20dc95c4
JA
300 return 1;
301
302 return 0;
303}
304
b1ff3403 305void put_io_u(struct thread_data *td, struct io_u *io_u)
ebac4655 306{
53cdc686 307 io_u->file = NULL;
ebac4655
JA
308 list_del(&io_u->list);
309 list_add(&io_u->list, &td->io_u_freelist);
310 td->cur_depth--;
311}
312
53cdc686
JA
313static int fill_io_u(struct thread_data *td, struct fio_file *f,
314 struct io_u *io_u)
843a7413
JA
315{
316 /*
317 * If using an iolog, grab next piece if any available.
318 */
319 if (td->read_iolog)
320 return read_iolog_get(td, io_u);
321
aea47d44
JA
322 /*
323 * No log, let the seq/rand engine retrieve the next position.
324 */
53cdc686 325 if (!get_next_offset(td, f, &io_u->offset)) {
aea47d44
JA
326 io_u->buflen = get_next_buflen(td);
327
328 if (io_u->buflen) {
329 io_u->ddir = get_rw_ddir(td);
843a7413
JA
330
331 /*
332 * If using a write iolog, store this entry.
333 */
334 if (td->write_iolog)
335 write_iolog_put(td, io_u);
336
53cdc686 337 io_u->file = f;
aea47d44
JA
338 return 0;
339 }
340 }
341
342 return 1;
343}
344
22f78b32 345#define queue_full(td) list_empty(&(td)->io_u_freelist)
ebac4655 346
b1ff3403 347struct io_u *__get_io_u(struct thread_data *td)
ebac4655 348{
22f78b32 349 struct io_u *io_u = NULL;
ebac4655 350
22f78b32
JA
351 if (!queue_full(td)) {
352 io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
353
354 io_u->error = 0;
355 io_u->resid = 0;
356 list_del(&io_u->list);
357 list_add(&io_u->list, &td->io_u_busylist);
358 td->cur_depth++;
359 }
ebac4655 360
ebac4655
JA
361 return io_u;
362}
363
906c8d75
JA
364/*
365 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
366 * etc. The returned io_u is fully ready to be prepped and submitted.
367 */
53cdc686 368static struct io_u *get_io_u(struct thread_data *td, struct fio_file *f)
ebac4655
JA
369{
370 struct io_u *io_u;
371
372 io_u = __get_io_u(td);
373 if (!io_u)
374 return NULL;
375
20dc95c4
JA
376 if (td->zone_bytes >= td->zone_size) {
377 td->zone_bytes = 0;
53cdc686 378 f->last_pos += td->zone_skip;
20dc95c4
JA
379 }
380
53cdc686 381 if (fill_io_u(td, f, io_u)) {
ebac4655
JA
382 put_io_u(td, io_u);
383 return NULL;
384 }
385
b2a15192
JA
386 if (io_u->buflen + io_u->offset > f->file_size) {
387 if (td->io_ops->flags & FIO_RAWIO) {
388 put_io_u(td, io_u);
389 return NULL;
390 }
391
53cdc686 392 io_u->buflen = f->file_size - io_u->offset;
b2a15192 393 }
ebac4655
JA
394
395 if (!io_u->buflen) {
396 put_io_u(td, io_u);
397 return NULL;
398 }
399
843a7413 400 if (!td->read_iolog && !td->sequential)
53cdc686 401 mark_random_map(td, f, io_u);
ebac4655 402
53cdc686 403 f->last_pos += io_u->buflen;
ebac4655
JA
404
405 if (td->verify != VERIFY_NONE)
e29d1b70 406 populate_verify_io_u(td, io_u);
ebac4655 407
aea47d44 408 if (td_io_prep(td, io_u)) {
ebac4655
JA
409 put_io_u(td, io_u);
410 return NULL;
411 }
412
413 gettimeofday(&io_u->start_time, NULL);
414 return io_u;
415}
416
417static inline void td_set_runstate(struct thread_data *td, int runstate)
418{
ebac4655
JA
419 td->runstate = runstate;
420}
421
53cdc686
JA
422static struct fio_file *get_next_file(struct thread_data *td)
423{
0ab8db89 424 unsigned int old_next_file = td->next_file;
b2a15192
JA
425 struct fio_file *f;
426
427 do {
428 f = &td->files[td->next_file];
429
430 td->next_file++;
431 if (td->next_file >= td->nr_files)
432 td->next_file = 0;
433
434 if (f->fd != -1)
435 break;
53cdc686 436
b2a15192
JA
437 f = NULL;
438 } while (td->next_file != old_next_file);
53cdc686
JA
439
440 return f;
441}
442
443static int td_io_sync(struct thread_data *td, struct fio_file *f)
ebac4655 444{
2866c82d 445 if (td->io_ops->sync)
53cdc686 446 return td->io_ops->sync(td, f);
ebac4655
JA
447
448 return 0;
449}
450
45bee283 451static int td_io_getevents(struct thread_data *td, int min, int max,
ebac4655
JA
452 struct timespec *t)
453{
2866c82d 454 return td->io_ops->getevents(td, min, max, t);
ebac4655
JA
455}
456
45bee283 457static int td_io_queue(struct thread_data *td, struct io_u *io_u)
ebac4655
JA
458{
459 gettimeofday(&io_u->issue_time, NULL);
460
2866c82d 461 return td->io_ops->queue(td, io_u);
ebac4655
JA
462}
463
464#define iocb_time(iocb) ((unsigned long) (iocb)->data)
465
466static void io_completed(struct thread_data *td, struct io_u *io_u,
467 struct io_completion_data *icd)
468{
469 struct timeval e;
470 unsigned long msec;
471
472 gettimeofday(&e, NULL);
473
474 if (!io_u->error) {
20dc95c4 475 unsigned int bytes = io_u->buflen - io_u->resid;
3d60d1ed 476 const int idx = io_u->ddir;
ebac4655
JA
477
478 td->io_blocks[idx]++;
20dc95c4
JA
479 td->io_bytes[idx] += bytes;
480 td->zone_bytes += bytes;
481 td->this_io_bytes[idx] += bytes;
ebac4655
JA
482
483 msec = mtime_since(&io_u->issue_time, &e);
484
3d60d1ed
JA
485 add_clat_sample(td, idx, msec);
486 add_bw_sample(td, idx);
ebac4655 487
3d60d1ed 488 if ((td_rw(td) || td_write(td)) && idx == DDIR_WRITE)
ebac4655
JA
489 log_io_piece(td, io_u);
490
20dc95c4 491 icd->bytes_done[idx] += bytes;
ebac4655
JA
492 } else
493 icd->error = io_u->error;
494}
495
496static void ios_completed(struct thread_data *td,struct io_completion_data *icd)
497{
498 struct io_u *io_u;
499 int i;
500
501 icd->error = 0;
502 icd->bytes_done[0] = icd->bytes_done[1] = 0;
503
504 for (i = 0; i < icd->nr; i++) {
2866c82d 505 io_u = td->io_ops->event(td, i);
ebac4655
JA
506
507 io_completed(td, io_u, icd);
508 put_io_u(td, io_u);
509 }
510}
511
906c8d75
JA
512/*
513 * When job exits, we can cancel the in-flight IO if we are using async
514 * io. Attempt to do so.
515 */
ebac4655
JA
516static void cleanup_pending_aio(struct thread_data *td)
517{
518 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
519 struct list_head *entry, *n;
520 struct io_completion_data icd;
521 struct io_u *io_u;
522 int r;
523
524 /*
525 * get immediately available events, if any
526 */
45bee283 527 r = td_io_getevents(td, 0, td->cur_depth, &ts);
ebac4655
JA
528 if (r > 0) {
529 icd.nr = r;
530 ios_completed(td, &icd);
531 }
532
533 /*
534 * now cancel remaining active events
535 */
2866c82d 536 if (td->io_ops->cancel) {
ebac4655
JA
537 list_for_each_safe(entry, n, &td->io_u_busylist) {
538 io_u = list_entry(entry, struct io_u, list);
539
2866c82d 540 r = td->io_ops->cancel(td, io_u);
ebac4655
JA
541 if (!r)
542 put_io_u(td, io_u);
543 }
544 }
545
546 if (td->cur_depth) {
45bee283 547 r = td_io_getevents(td, td->cur_depth, td->cur_depth, NULL);
ebac4655
JA
548 if (r > 0) {
549 icd.nr = r;
550 ios_completed(td, &icd);
551 }
552 }
553}
554
906c8d75
JA
555/*
556 * The main verify engine. Runs over the writes we previusly submitted,
557 * reads the blocks back in, and checks the crc/md5 of the data.
558 */
a9619d44 559void do_verify(struct thread_data *td)
ebac4655
JA
560{
561 struct timeval t;
562 struct io_u *io_u, *v_io_u = NULL;
563 struct io_completion_data icd;
53cdc686 564 struct fio_file *f;
e5b401d4
JA
565 int ret, i;
566
567 /*
568 * sync io first and invalidate cache, to make sure we really
569 * read from disk.
570 */
571 for_each_file(td, f, i) {
572 td_io_sync(td, f);
573 file_invalidate_cache(td, f);
574 }
ebac4655
JA
575
576 td_set_runstate(td, TD_VERIFYING);
577
578 do {
579 if (td->terminate)
580 break;
581
582 gettimeofday(&t, NULL);
583 if (runtime_exceeded(td, &t))
584 break;
585
586 io_u = __get_io_u(td);
587 if (!io_u)
588 break;
589
aea47d44 590 if (get_next_verify(td, io_u)) {
ebac4655
JA
591 put_io_u(td, io_u);
592 break;
593 }
594
53cdc686
JA
595 f = get_next_file(td);
596 if (!f)
597 break;
598
599 io_u->file = f;
600
aea47d44 601 if (td_io_prep(td, io_u)) {
ebac4655
JA
602 put_io_u(td, io_u);
603 break;
604 }
605
45bee283 606 ret = td_io_queue(td, io_u);
ebac4655
JA
607 if (ret) {
608 put_io_u(td, io_u);
609 td_verror(td, ret);
610 break;
611 }
612
613 /*
614 * we have one pending to verify, do that while
615 * we are doing io on the next one
616 */
617 if (do_io_u_verify(td, &v_io_u))
618 break;
619
45bee283 620 ret = td_io_getevents(td, 1, 1, NULL);
ebac4655
JA
621 if (ret != 1) {
622 if (ret < 0)
623 td_verror(td, ret);
624 break;
625 }
626
2866c82d 627 v_io_u = td->io_ops->event(td, 0);
ebac4655
JA
628 icd.nr = 1;
629 icd.error = 0;
630 io_completed(td, v_io_u, &icd);
631
632 if (icd.error) {
633 td_verror(td, icd.error);
634 put_io_u(td, v_io_u);
635 v_io_u = NULL;
636 break;
637 }
638
ebac4655
JA
639 /*
640 * if we can't submit more io, we need to verify now
641 */
642 if (queue_full(td) && do_io_u_verify(td, &v_io_u))
643 break;
644
645 } while (1);
646
647 do_io_u_verify(td, &v_io_u);
648
649 if (td->cur_depth)
650 cleanup_pending_aio(td);
651
652 td_set_runstate(td, TD_RUNNING);
653}
654
b990b5c0
JA
655/*
656 * Not really an io thread, all it does is burn CPU cycles in the specified
657 * manner.
658 */
659static void do_cpuio(struct thread_data *td)
660{
661 struct timeval e;
662 int split = 100 / td->cpuload;
663 int i = 0;
664
665 while (!td->terminate) {
666 gettimeofday(&e, NULL);
667
668 if (runtime_exceeded(td, &e))
669 break;
670
671 if (!(i % split))
672 __usec_sleep(10000);
673 else
674 usec_sleep(td, 10000);
675
676 i++;
677 }
678}
679
32cd46a0 680/*
906c8d75 681 * Main IO worker function. It retrieves io_u's to process and queues
32cd46a0
JA
682 * and reaps them, checking for rate and errors along the way.
683 */
ebac4655
JA
684static void do_io(struct thread_data *td)
685{
686 struct io_completion_data icd;
687 struct timeval s, e;
688 unsigned long usec;
53cdc686 689 struct fio_file *f;
84585003 690 int i, ret = 0;
ebac4655 691
5853e5a8
JA
692 td_set_runstate(td, TD_RUNNING);
693
ebac4655
JA
694 while (td->this_io_bytes[td->ddir] < td->io_size) {
695 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
696 struct timespec *timeout;
84585003 697 int min_evts = 0;
ebac4655
JA
698 struct io_u *io_u;
699
700 if (td->terminate)
701 break;
702
53cdc686
JA
703 f = get_next_file(td);
704 if (!f)
705 break;
706
707 io_u = get_io_u(td, f);
ebac4655
JA
708 if (!io_u)
709 break;
710
711 memcpy(&s, &io_u->start_time, sizeof(s));
712
45bee283 713 ret = td_io_queue(td, io_u);
ebac4655
JA
714 if (ret) {
715 put_io_u(td, io_u);
716 td_verror(td, ret);
717 break;
718 }
719
720 add_slat_sample(td, io_u->ddir, mtime_since(&io_u->start_time, &io_u->issue_time));
721
722 if (td->cur_depth < td->iodepth) {
723 timeout = &ts;
724 min_evts = 0;
725 } else {
726 timeout = NULL;
727 min_evts = 1;
728 }
729
84585003 730
45bee283 731 ret = td_io_getevents(td, min_evts, td->cur_depth, timeout);
ebac4655 732 if (ret < 0) {
84585003 733 td_verror(td, -ret);
ebac4655
JA
734 break;
735 } else if (!ret)
736 continue;
737
738 icd.nr = ret;
739 ios_completed(td, &icd);
740 if (icd.error) {
741 td_verror(td, icd.error);
742 break;
743 }
744
745 /*
746 * the rate is batched for now, it should work for batches
747 * of completions except the very first one which may look
748 * a little bursty
749 */
750 gettimeofday(&e, NULL);
751 usec = utime_since(&s, &e);
752
753 rate_throttle(td, usec, icd.bytes_done[td->ddir]);
754
755 if (check_min_rate(td, &e)) {
756 td_verror(td, ENOMEM);
757 break;
758 }
759
760 if (runtime_exceeded(td, &e))
761 break;
762
763 if (td->thinktime)
764 usec_sleep(td, td->thinktime);
765
766 if (should_fsync(td) && td->fsync_blocks &&
767 (td->io_blocks[DDIR_WRITE] % td->fsync_blocks) == 0)
53cdc686 768 td_io_sync(td, f);
ebac4655
JA
769 }
770
84585003
JA
771 if (!ret) {
772 if (td->cur_depth)
773 cleanup_pending_aio(td);
ebac4655 774
84585003
JA
775 if (should_fsync(td) && td->end_fsync) {
776 td_set_runstate(td, TD_FSYNCING);
777 for_each_file(td, f, i)
778 td_io_sync(td, f);
779 }
5853e5a8 780 }
ebac4655
JA
781}
782
45bee283 783static int td_io_init(struct thread_data *td)
ebac4655 784{
2866c82d
JA
785 if (td->io_ops->init)
786 return td->io_ops->init(td);
787
788 return 0;
ebac4655
JA
789}
790
791static void cleanup_io_u(struct thread_data *td)
792{
793 struct list_head *entry, *n;
794 struct io_u *io_u;
795
796 list_for_each_safe(entry, n, &td->io_u_freelist) {
797 io_u = list_entry(entry, struct io_u, list);
798
799 list_del(&io_u->list);
800 free(io_u);
801 }
802
803 if (td->mem_type == MEM_MALLOC)
804 free(td->orig_buffer);
805 else if (td->mem_type == MEM_SHM) {
806 struct shmid_ds sbuf;
807
808 shmdt(td->orig_buffer);
809 shmctl(td->shm_id, IPC_RMID, &sbuf);
810 } else if (td->mem_type == MEM_MMAP)
811 munmap(td->orig_buffer, td->orig_buffer_size);
812 else
3b70d7e5 813 log_err("Bad memory type %d\n", td->mem_type);
ebac4655
JA
814
815 td->orig_buffer = NULL;
816}
817
818static int init_io_u(struct thread_data *td)
819{
820 struct io_u *io_u;
821 int i, max_units;
822 char *p;
823
2866c82d 824 if (td->io_ops->flags & FIO_CPUIO)
b990b5c0
JA
825 return 0;
826
2866c82d 827 if (td->io_ops->flags & FIO_SYNCIO)
ebac4655
JA
828 max_units = 1;
829 else
830 max_units = td->iodepth;
831
832 td->orig_buffer_size = td->max_bs * max_units + MASK;
833
834 if (td->mem_type == MEM_MALLOC)
835 td->orig_buffer = malloc(td->orig_buffer_size);
836 else if (td->mem_type == MEM_SHM) {
837 td->shm_id = shmget(IPC_PRIVATE, td->orig_buffer_size, IPC_CREAT | 0600);
838 if (td->shm_id < 0) {
839 td_verror(td, errno);
840 perror("shmget");
841 return 1;
842 }
843
844 td->orig_buffer = shmat(td->shm_id, NULL, 0);
845 if (td->orig_buffer == (void *) -1) {
846 td_verror(td, errno);
847 perror("shmat");
848 td->orig_buffer = NULL;
849 return 1;
850 }
851 } else if (td->mem_type == MEM_MMAP) {
852 td->orig_buffer = mmap(NULL, td->orig_buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0);
853 if (td->orig_buffer == MAP_FAILED) {
854 td_verror(td, errno);
855 perror("mmap");
856 td->orig_buffer = NULL;
857 return 1;
858 }
859 }
860
ebac4655
JA
861 p = ALIGN(td->orig_buffer);
862 for (i = 0; i < max_units; i++) {
863 io_u = malloc(sizeof(*io_u));
864 memset(io_u, 0, sizeof(*io_u));
865 INIT_LIST_HEAD(&io_u->list);
866
867 io_u->buf = p + td->max_bs * i;
b1ff3403 868 io_u->index = i;
ebac4655
JA
869 list_add(&io_u->list, &td->io_u_freelist);
870 }
871
872 return 0;
873}
874
da86774e
JA
875static int switch_ioscheduler(struct thread_data *td)
876{
877 char tmp[256], tmp2[128];
878 FILE *f;
879 int ret;
880
881 sprintf(tmp, "%s/queue/scheduler", td->sysfs_root);
882
883 f = fopen(tmp, "r+");
884 if (!f) {
885 td_verror(td, errno);
886 return 1;
887 }
888
889 /*
890 * Set io scheduler.
891 */
892 ret = fwrite(td->ioscheduler, strlen(td->ioscheduler), 1, f);
893 if (ferror(f) || ret != 1) {
894 td_verror(td, errno);
895 fclose(f);
896 return 1;
897 }
898
899 rewind(f);
900
901 /*
902 * Read back and check that the selected scheduler is now the default.
903 */
904 ret = fread(tmp, 1, sizeof(tmp), f);
905 if (ferror(f) || ret < 0) {
906 td_verror(td, errno);
907 fclose(f);
908 return 1;
909 }
910
911 sprintf(tmp2, "[%s]", td->ioscheduler);
912 if (!strstr(tmp, tmp2)) {
3b70d7e5 913 log_err("fio: io scheduler %s not found\n", td->ioscheduler);
da86774e
JA
914 td_verror(td, EINVAL);
915 fclose(f);
916 return 1;
917 }
918
919 fclose(f);
920 return 0;
921}
922
ebac4655
JA
923static void clear_io_state(struct thread_data *td)
924{
53cdc686
JA
925 struct fio_file *f;
926 int i;
ebac4655 927
ebac4655
JA
928 td->stat_io_bytes[0] = td->stat_io_bytes[1] = 0;
929 td->this_io_bytes[0] = td->this_io_bytes[1] = 0;
20dc95c4 930 td->zone_bytes = 0;
ebac4655 931
53cdc686
JA
932 for_each_file(td, f, i) {
933 f->last_pos = 0;
934 if (td->io_ops->flags & FIO_SYNCIO)
935 lseek(f->fd, SEEK_SET, 0);
936
937 if (f->file_map)
938 memset(f->file_map, 0, f->num_maps * sizeof(long));
939 }
ebac4655
JA
940}
941
906c8d75
JA
942/*
943 * Entry point for the thread based jobs. The process based jobs end up
944 * here as well, after a little setup.
945 */
ebac4655
JA
946static void *thread_main(void *data)
947{
948 struct thread_data *td = data;
ebac4655
JA
949
950 if (!td->use_thread)
951 setsid();
952
953 td->pid = getpid();
954
aea47d44
JA
955 INIT_LIST_HEAD(&td->io_u_freelist);
956 INIT_LIST_HEAD(&td->io_u_busylist);
957 INIT_LIST_HEAD(&td->io_hist_list);
958 INIT_LIST_HEAD(&td->io_log_list);
959
ebac4655
JA
960 if (init_io_u(td))
961 goto err;
962
963 if (fio_setaffinity(td) == -1) {
964 td_verror(td, errno);
965 goto err;
966 }
967
45bee283 968 if (td_io_init(td))
ebac4655
JA
969 goto err;
970
aea47d44
JA
971 if (init_iolog(td))
972 goto err;
973
ebac4655
JA
974 if (td->ioprio) {
975 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
976 td_verror(td, errno);
977 goto err;
978 }
979 }
980
1056eaad 981 if (nice(td->nice) == -1) {
b6f4d880
JA
982 td_verror(td, errno);
983 goto err;
984 }
985
75154845
JA
986 if (init_random_state(td))
987 goto err;
988
da86774e
JA
989 if (td->ioscheduler && switch_ioscheduler(td))
990 goto err;
991
75154845 992 td_set_runstate(td, TD_INITIALIZED);
bbfd6b00
JA
993 fio_sem_up(&startup_sem);
994 fio_sem_down(&td->mutex);
ebac4655 995
53cdc686 996 if (!td->create_serialize && setup_files(td))
ebac4655
JA
997 goto err;
998
ebac4655
JA
999 gettimeofday(&td->epoch, NULL);
1000
4e0ba8af
JA
1001 if (td->exec_prerun)
1002 system(td->exec_prerun);
1003
ebac4655
JA
1004 while (td->loops--) {
1005 getrusage(RUSAGE_SELF, &td->ru_start);
1006 gettimeofday(&td->start, NULL);
1007 memcpy(&td->stat_sample_time, &td->start, sizeof(td->start));
1008
1009 if (td->ratemin)
1010 memcpy(&td->lastrate, &td->stat_sample_time, sizeof(td->lastrate));
1011
1012 clear_io_state(td);
1013 prune_io_piece_log(td);
1014
2866c82d 1015 if (td->io_ops->flags & FIO_CPUIO)
b990b5c0
JA
1016 do_cpuio(td);
1017 else
1018 do_io(td);
ebac4655
JA
1019
1020 td->runtime[td->ddir] += mtime_since_now(&td->start);
aea47d44 1021 if (td_rw(td) && td->io_bytes[td->ddir ^ 1])
3d60d1ed
JA
1022 td->runtime[td->ddir ^ 1] = td->runtime[td->ddir];
1023
ebac4655
JA
1024 update_rusage_stat(td);
1025
1026 if (td->error || td->terminate)
1027 break;
1028
1029 if (td->verify == VERIFY_NONE)
1030 continue;
1031
1032 clear_io_state(td);
1033 gettimeofday(&td->start, NULL);
1034
1035 do_verify(td);
1036
1037 td->runtime[DDIR_READ] += mtime_since_now(&td->start);
1038
1039 if (td->error || td->terminate)
1040 break;
1041 }
1042
ebac4655
JA
1043 if (td->bw_log)
1044 finish_log(td, td->bw_log, "bw");
1045 if (td->slat_log)
1046 finish_log(td, td->slat_log, "slat");
1047 if (td->clat_log)
1048 finish_log(td, td->clat_log, "clat");
843a7413
JA
1049 if (td->write_iolog)
1050 write_iolog_close(td);
4e0ba8af
JA
1051 if (td->exec_postrun)
1052 system(td->exec_postrun);
ebac4655
JA
1053
1054 if (exitall_on_terminate)
1055 terminate_threads(td->groupid);
1056
1057err:
53cdc686 1058 close_files(td);
2866c82d 1059 close_ioengine(td);
ebac4655 1060 cleanup_io_u(td);
ebac4655
JA
1061 td_set_runstate(td, TD_EXITED);
1062 return NULL;
1063
1064}
1065
906c8d75
JA
1066/*
1067 * We cannot pass the td data into a forked process, so attach the td and
1068 * pass it to the thread worker.
1069 */
ebac4655
JA
1070static void *fork_main(int shmid, int offset)
1071{
1072 struct thread_data *td;
1073 void *data;
1074
1075 data = shmat(shmid, NULL, 0);
1076 if (data == (void *) -1) {
1077 perror("shmat");
1078 return NULL;
1079 }
1080
1081 td = data + offset * sizeof(struct thread_data);
1082 thread_main(td);
1083 shmdt(data);
1084 return NULL;
1085}
1086
906c8d75
JA
1087/*
1088 * Run over the job map and reap the threads that have exited, if any.
1089 */
ebac4655
JA
1090static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
1091{
34572e28 1092 struct thread_data *td;
b990b5c0 1093 int i, cputhreads;
ebac4655
JA
1094
1095 /*
1096 * reap exited threads (TD_EXITED -> TD_REAPED)
1097 */
34572e28
JA
1098 cputhreads = 0;
1099 for_each_td(td, i) {
84585003
JA
1100 /*
1101 * ->io_ops is NULL for a thread that has closed its
1102 * io engine
1103 */
1104 if (td->io_ops && td->io_ops->flags & FIO_CPUIO)
b990b5c0
JA
1105 cputhreads++;
1106
ebac4655
JA
1107 if (td->runstate != TD_EXITED)
1108 continue;
1109
1110 td_set_runstate(td, TD_REAPED);
1111
1112 if (td->use_thread) {
1113 long ret;
1114
1115 if (pthread_join(td->thread, (void *) &ret))
1116 perror("thread_join");
1117 } else
1118 waitpid(td->pid, NULL, 0);
1119
1120 (*nr_running)--;
1121 (*m_rate) -= td->ratemin;
1122 (*t_rate) -= td->rate;
1123 }
b990b5c0
JA
1124
1125 if (*nr_running == cputhreads)
1126 terminate_threads(TERMINATE_ALL);
ebac4655
JA
1127}
1128
fcb6ade2
JA
1129static void fio_unpin_memory(void *pinned)
1130{
1131 if (pinned) {
1132 if (munlock(pinned, mlock_size) < 0)
1133 perror("munlock");
1134 munmap(pinned, mlock_size);
1135 }
1136}
1137
1138static void *fio_pin_memory(void)
1139{
32cd46a0 1140 unsigned long long phys_mem;
fcb6ade2
JA
1141 void *ptr;
1142
1143 if (!mlock_size)
1144 return NULL;
1145
1146 /*
1147 * Don't allow mlock of more than real_mem-128MB
1148 */
32cd46a0
JA
1149 phys_mem = os_phys_mem();
1150 if (phys_mem) {
1151 if ((mlock_size + 128 * 1024 * 1024) > phys_mem) {
1152 mlock_size = phys_mem - 128 * 1024 * 1024;
eb8bbf48 1153 fprintf(f_out, "fio: limiting mlocked memory to %lluMiB\n", mlock_size >> 20);
fcb6ade2
JA
1154 }
1155 }
1156
1157 ptr = mmap(NULL, mlock_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0);
1158 if (!ptr) {
1159 perror("malloc locked mem");
1160 return NULL;
1161 }
1162 if (mlock(ptr, mlock_size) < 0) {
1163 munmap(ptr, mlock_size);
1164 perror("mlock");
1165 return NULL;
1166 }
1167
1168 return ptr;
1169}
1170
906c8d75
JA
1171/*
1172 * Main function for kicking off and reaping jobs, as needed.
1173 */
ebac4655
JA
1174static void run_threads(void)
1175{
ebac4655
JA
1176 struct thread_data *td;
1177 unsigned long spent;
1178 int i, todo, nr_running, m_rate, t_rate, nr_started;
fcb6ade2
JA
1179 void *mlocked_mem;
1180
1181 mlocked_mem = fio_pin_memory();
ebac4655 1182
c6ae0a5b
JA
1183 if (!terse_output) {
1184 printf("Starting %d thread%s\n", thread_number, thread_number > 1 ? "s" : "");
1185 fflush(stdout);
1186 }
c04f7ec3 1187
4efa970e
JA
1188 signal(SIGINT, sig_handler);
1189 signal(SIGALRM, sig_handler);
1190
ebac4655
JA
1191 todo = thread_number;
1192 nr_running = 0;
1193 nr_started = 0;
1194 m_rate = t_rate = 0;
1195
34572e28 1196 for_each_td(td, i) {
263e529f 1197 print_status_init(td->thread_number - 1);
ebac4655
JA
1198
1199 init_disk_util(td);
1200
1201 if (!td->create_serialize)
1202 continue;
1203
1204 /*
1205 * do file setup here so it happens sequentially,
1206 * we don't want X number of threads getting their
1207 * client data interspersed on disk
1208 */
53cdc686 1209 if (setup_files(td)) {
ebac4655
JA
1210 td_set_runstate(td, TD_REAPED);
1211 todo--;
1212 }
1213 }
1214
263e529f 1215 time_init();
ebac4655
JA
1216
1217 while (todo) {
75154845
JA
1218 struct thread_data *map[MAX_JOBS];
1219 struct timeval this_start;
1220 int this_jobs = 0, left;
1221
ebac4655
JA
1222 /*
1223 * create threads (TD_NOT_CREATED -> TD_CREATED)
1224 */
34572e28 1225 for_each_td(td, i) {
ebac4655
JA
1226 if (td->runstate != TD_NOT_CREATED)
1227 continue;
1228
1229 /*
1230 * never got a chance to start, killed by other
1231 * thread for some reason
1232 */
1233 if (td->terminate) {
1234 todo--;
1235 continue;
1236 }
1237
1238 if (td->start_delay) {
263e529f 1239 spent = mtime_since_genesis();
ebac4655
JA
1240
1241 if (td->start_delay * 1000 > spent)
1242 continue;
1243 }
1244
1245 if (td->stonewall && (nr_started || nr_running))
1246 break;
1247
75154845
JA
1248 /*
1249 * Set state to created. Thread will transition
1250 * to TD_INITIALIZED when it's done setting up.
1251 */
ebac4655 1252 td_set_runstate(td, TD_CREATED);
75154845 1253 map[this_jobs++] = td;
bbfd6b00 1254 fio_sem_init(&startup_sem, 1);
ebac4655
JA
1255 nr_started++;
1256
1257 if (td->use_thread) {
1258 if (pthread_create(&td->thread, NULL, thread_main, td)) {
1259 perror("thread_create");
1260 nr_started--;
1261 }
1262 } else {
1263 if (fork())
bbfd6b00 1264 fio_sem_down(&startup_sem);
ebac4655
JA
1265 else {
1266 fork_main(shm_id, i);
1267 exit(0);
1268 }
1269 }
1270 }
1271
1272 /*
75154845
JA
1273 * Wait for the started threads to transition to
1274 * TD_INITIALIZED.
ebac4655 1275 */
75154845
JA
1276 gettimeofday(&this_start, NULL);
1277 left = this_jobs;
1278 while (left) {
1279 if (mtime_since_now(&this_start) > JOB_START_TIMEOUT)
1280 break;
1281
1282 usleep(100000);
1283
1284 for (i = 0; i < this_jobs; i++) {
1285 td = map[i];
1286 if (!td)
1287 continue;
b6f4d880 1288 if (td->runstate == TD_INITIALIZED) {
75154845
JA
1289 map[i] = NULL;
1290 left--;
b6f4d880
JA
1291 } else if (td->runstate >= TD_EXITED) {
1292 map[i] = NULL;
1293 left--;
1294 todo--;
1295 nr_running++; /* work-around... */
75154845
JA
1296 }
1297 }
1298 }
1299
1300 if (left) {
3b70d7e5 1301 log_err("fio: %d jobs failed to start\n", left);
75154845
JA
1302 for (i = 0; i < this_jobs; i++) {
1303 td = map[i];
1304 if (!td)
1305 continue;
1306 kill(td->pid, SIGTERM);
1307 }
1308 break;
1309 }
1310
1311 /*
b6f4d880 1312 * start created threads (TD_INITIALIZED -> TD_RUNNING).
75154845 1313 */
34572e28 1314 for_each_td(td, i) {
75154845 1315 if (td->runstate != TD_INITIALIZED)
ebac4655
JA
1316 continue;
1317
1318 td_set_runstate(td, TD_RUNNING);
1319 nr_running++;
1320 nr_started--;
1321 m_rate += td->ratemin;
1322 t_rate += td->rate;
75154845 1323 todo--;
bbfd6b00 1324 fio_sem_up(&td->mutex);
ebac4655
JA
1325 }
1326
1327 reap_threads(&nr_running, &t_rate, &m_rate);
1328
1329 if (todo)
1330 usleep(100000);
1331 }
1332
1333 while (nr_running) {
1334 reap_threads(&nr_running, &t_rate, &m_rate);
1335 usleep(10000);
1336 }
1337
1338 update_io_ticks();
fcb6ade2 1339 fio_unpin_memory(mlocked_mem);
ebac4655
JA
1340}
1341
ebac4655
JA
1342int main(int argc, char *argv[])
1343{
1344 if (parse_options(argc, argv))
1345 return 1;
1346
1347 if (!thread_number) {
3b70d7e5 1348 log_err("Nothing to do\n");
ebac4655
JA
1349 return 1;
1350 }
1351
1352 disk_util_timer_arm();
1353
1354 run_threads();
1355 show_run_stats();
1356
1357 return 0;
1358}