Use log_info() throughout
[fio.git] / fio.c
CommitLineData
ebac4655
JA
1/*
2 * fio - the flexible io tester
3 *
4 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
aae22ca7 5 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
ebac4655 6 *
8e9fe637
JA
7 * The license below covers all files distributed with fio unless otherwise
8 * noted in the file itself.
9 *
ebac4655 10 * This program is free software; you can redistribute it and/or modify
8e9fe637
JA
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
ebac4655
JA
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 *
23 */
ebac4655
JA
24#include <unistd.h>
25#include <fcntl.h>
26#include <string.h>
ebac4655
JA
27#include <signal.h>
28#include <time.h>
dbe1125e 29#include <locale.h>
36167d82 30#include <assert.h>
ebac4655
JA
31#include <sys/stat.h>
32#include <sys/wait.h>
33#include <sys/ipc.h>
34#include <sys/shm.h>
ebac4655
JA
35#include <sys/mman.h>
36
37#include "fio.h"
38#include "os.h"
39
cfc99db7
JA
40unsigned long page_mask;
41unsigned long page_size;
29d610e1
JA
42#define ALIGN(buf) \
43 (char *) (((unsigned long) (buf) + page_mask) & ~page_mask)
ebac4655
JA
44
45int groupid = 0;
46int thread_number = 0;
9cedf167
JA
47int nr_process = 0;
48int nr_thread = 0;
ebac4655 49int shm_id = 0;
53cdc686 50int temp_stall_ts;
ebac4655 51
07739b57 52static struct fio_sem *startup_sem;
6ce15a32 53static volatile int fio_abort;
437c9b71 54static int exit_value;
ebac4655 55
bb3884d8
JA
56struct io_log *agg_io_log[2];
57
ebac4655 58#define TERMINATE_ALL (-1)
75154845 59#define JOB_START_TIMEOUT (5 * 1000)
ebac4655 60
6ce15a32
JA
61static inline void td_set_runstate(struct thread_data *td, int runstate)
62{
63 td->runstate = runstate;
64}
65
390c40e2 66static void terminate_threads(int group_id)
ebac4655 67{
34572e28 68 struct thread_data *td;
ebac4655
JA
69 int i;
70
34572e28 71 for_each_td(td, i) {
ebac4655 72 if (group_id == TERMINATE_ALL || groupid == td->groupid) {
c1302d44
JA
73 /*
74 * if the thread is running, just let it exit
75 */
76 if (td->runstate < TD_RUNNING)
77 kill(td->pid, SIGQUIT);
ebac4655
JA
78 td->terminate = 1;
79 td->start_delay = 0;
80 }
81 }
82}
83
84static void sig_handler(int sig)
85{
86 switch (sig) {
87 case SIGALRM:
88 update_io_ticks();
89 disk_util_timer_arm();
90 print_thread_status();
91 break;
92 default:
6ce15a32 93 printf("\nfio: terminating on signal %d\n", sig);
ebac4655 94 fflush(stdout);
390c40e2 95 terminate_threads(TERMINATE_ALL);
ebac4655
JA
96 break;
97 }
98}
99
906c8d75
JA
100/*
101 * Check if we are above the minimum rate given.
102 */
ebac4655
JA
103static int check_min_rate(struct thread_data *td, struct timeval *now)
104{
0904200b 105 unsigned long long bytes = 0;
ebac4655
JA
106 unsigned long spent;
107 unsigned long rate;
ebac4655 108
780bf1a1
JA
109 /*
110 * No minimum rate set, always ok
111 */
112 if (!td->ratemin)
113 return 0;
114
ebac4655
JA
115 /*
116 * allow a 2 second settle period in the beginning
117 */
118 if (mtime_since(&td->start, now) < 2000)
119 return 0;
120
0904200b
JA
121 if (td_read(td))
122 bytes += td->this_io_bytes[DDIR_READ];
123 if (td_write(td))
124 bytes += td->this_io_bytes[DDIR_WRITE];
125
ebac4655
JA
126 /*
127 * if rate blocks is set, sample is running
128 */
129 if (td->rate_bytes) {
130 spent = mtime_since(&td->lastrate, now);
131 if (spent < td->ratecycle)
132 return 0;
133
413dd459 134 if (bytes < td->rate_bytes) {
6d86144d 135 log_err("%s: min rate %u not met\n", td->name, td->ratemin);
ebac4655 136 return 1;
413dd459
JA
137 } else {
138 rate = (bytes - td->rate_bytes) / spent;
139 if (rate < td->ratemin || bytes < td->rate_bytes) {
6d86144d 140 log_err("%s: min rate %u not met, got %luKiB/sec\n", td->name, td->ratemin, rate);
413dd459
JA
141 return 1;
142 }
ebac4655
JA
143 }
144 }
145
0904200b 146 td->rate_bytes = bytes;
ebac4655
JA
147 memcpy(&td->lastrate, now, sizeof(*now));
148 return 0;
149}
150
151static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
152{
153 if (!td->timeout)
154 return 0;
155 if (mtime_since(&td->epoch, t) >= td->timeout * 1000)
156 return 1;
157
158 return 0;
159}
160
906c8d75
JA
161/*
162 * When job exits, we can cancel the in-flight IO if we are using async
163 * io. Attempt to do so.
164 */
ebac4655
JA
165static void cleanup_pending_aio(struct thread_data *td)
166{
ebac4655 167 struct list_head *entry, *n;
ebac4655
JA
168 struct io_u *io_u;
169 int r;
170
171 /*
172 * get immediately available events, if any
173 */
d7762cf8 174 r = io_u_queued_complete(td, 0);
b2fdda43
JA
175 if (r < 0)
176 return;
ebac4655
JA
177
178 /*
179 * now cancel remaining active events
180 */
2866c82d 181 if (td->io_ops->cancel) {
ebac4655
JA
182 list_for_each_safe(entry, n, &td->io_u_busylist) {
183 io_u = list_entry(entry, struct io_u, list);
184
0c6e7517
JA
185 /*
186 * if the io_u isn't in flight, then that generally
187 * means someone leaked an io_u. complain but fix
188 * it up, so we don't stall here.
189 */
190 if ((io_u->flags & IO_U_F_FLIGHT) == 0) {
191 log_err("fio: non-busy IO on busy list\n");
ebac4655 192 put_io_u(td, io_u);
0c6e7517
JA
193 } else {
194 r = td->io_ops->cancel(td, io_u);
195 if (!r)
196 put_io_u(td, io_u);
197 }
ebac4655
JA
198 }
199 }
200
97601024 201 if (td->cur_depth)
d7762cf8 202 r = io_u_queued_complete(td, td->cur_depth);
ebac4655
JA
203}
204
858a3d47
JA
205/*
206 * Helper to handle the final sync of a file. Works just like the normal
207 * io path, just does everything sync.
208 */
209static int fio_io_sync(struct thread_data *td, struct fio_file *f)
210{
211 struct io_u *io_u = __get_io_u(td);
858a3d47
JA
212 int ret;
213
214 if (!io_u)
215 return 1;
216
217 io_u->ddir = DDIR_SYNC;
218 io_u->file = f;
219
220 if (td_io_prep(td, io_u)) {
221 put_io_u(td, io_u);
222 return 1;
223 }
224
755200a3 225requeue:
858a3d47 226 ret = td_io_queue(td, io_u);
36167d82 227 if (ret < 0) {
e1161c32 228 td_verror(td, io_u->error, "td_io_queue");
858a3d47 229 put_io_u(td, io_u);
858a3d47 230 return 1;
36167d82 231 } else if (ret == FIO_Q_QUEUED) {
d7762cf8 232 if (io_u_queued_complete(td, 1) < 0)
36167d82 233 return 1;
36167d82
JA
234 } else if (ret == FIO_Q_COMPLETED) {
235 if (io_u->error) {
e1161c32 236 td_verror(td, io_u->error, "td_io_queue");
36167d82
JA
237 return 1;
238 }
858a3d47 239
d7762cf8 240 if (io_u_sync_complete(td, io_u) < 0)
b2fdda43 241 return 1;
755200a3
JA
242 } else if (ret == FIO_Q_BUSY) {
243 if (td_io_commit(td))
244 return 1;
245 goto requeue;
858a3d47
JA
246 }
247
248 return 0;
249}
250
906c8d75 251/*
34403fb1 252 * The main verify engine. Runs over the writes we previously submitted,
906c8d75
JA
253 * reads the blocks back in, and checks the crc/md5 of the data.
254 */
1e97cce9 255static void do_verify(struct thread_data *td)
ebac4655 256{
53cdc686 257 struct fio_file *f;
36167d82 258 struct io_u *io_u;
af52b345
JA
259 int ret, min_events;
260 unsigned int i;
e5b401d4
JA
261
262 /*
263 * sync io first and invalidate cache, to make sure we really
264 * read from disk.
265 */
266 for_each_file(td, f, i) {
3d7b485f
JA
267 if (!(f->flags & FIO_FILE_OPEN))
268 continue;
b2fdda43
JA
269 if (fio_io_sync(td, f))
270 break;
271 if (file_invalidate_cache(td, f))
272 break;
e5b401d4 273 }
ebac4655 274
b2fdda43
JA
275 if (td->error)
276 return;
277
ebac4655
JA
278 td_set_runstate(td, TD_VERIFYING);
279
36167d82
JA
280 io_u = NULL;
281 while (!td->terminate) {
5451792e
JA
282 int ret2;
283
ebac4655
JA
284 io_u = __get_io_u(td);
285 if (!io_u)
286 break;
287
069c2918
JA
288 if (runtime_exceeded(td, &io_u->start_time)) {
289 put_io_u(td, io_u);
02bcaa8c 290 break;
069c2918 291 }
02bcaa8c 292
069c2918
JA
293 if (get_next_verify(td, io_u)) {
294 put_io_u(td, io_u);
ebac4655 295 break;
069c2918 296 }
ebac4655 297
069c2918
JA
298 if (td_io_prep(td, io_u)) {
299 put_io_u(td, io_u);
53cdc686 300 break;
069c2918 301 }
d7762cf8
JA
302
303 io_u->end_io = verify_io_u;
53cdc686 304
11786802 305 ret = td_io_queue(td, io_u);
36167d82
JA
306 switch (ret) {
307 case FIO_Q_COMPLETED:
308 if (io_u->error)
22819ec2 309 ret = -io_u->error;
9e9d164e 310 else if (io_u->resid) {
36167d82 311 int bytes = io_u->xfer_buflen - io_u->resid;
ebac4655 312
9e9d164e
JA
313 /*
314 * zero read, fail
315 */
316 if (!bytes) {
317 td_verror(td, ENODATA, "full resid");
318 put_io_u(td, io_u);
319 break;
320 }
36167d82
JA
321 io_u->xfer_buflen = io_u->resid;
322 io_u->xfer_buf += bytes;
11786802
JA
323 requeue_io_u(td, &io_u);
324 } else {
325 ret = io_u_sync_complete(td, io_u);
326 if (ret < 0)
327 break;
36167d82 328 }
36167d82
JA
329 continue;
330 case FIO_Q_QUEUED:
331 break;
755200a3
JA
332 case FIO_Q_BUSY:
333 requeue_io_u(td, &io_u);
5451792e
JA
334 ret2 = td_io_commit(td);
335 if (ret2 < 0)
336 ret = ret2;
755200a3 337 break;
36167d82
JA
338 default:
339 assert(ret < 0);
e1161c32 340 td_verror(td, -ret, "td_io_queue");
ebac4655
JA
341 break;
342 }
343
99784632 344 if (ret < 0 || td->error)
3af6ef39
JA
345 break;
346
ebac4655 347 /*
3af6ef39
JA
348 * if we can queue more, do so. but check if there are
349 * completed io_u's first.
ebac4655 350 */
97601024 351 min_events = 0;
e916b390 352 if (queue_full(td) || ret == FIO_Q_BUSY) {
3af6ef39 353 min_events = 1;
3af6ef39 354
e916b390
JA
355 if (td->cur_depth > td->iodepth_low)
356 min_events = td->cur_depth - td->iodepth_low;
357 }
358
3af6ef39
JA
359 /*
360 * Reap required number of io units, if any, and do the
361 * verification on them through the callback handler
362 */
d7762cf8 363 if (io_u_queued_complete(td, min_events) < 0)
ebac4655 364 break;
36167d82 365 }
ebac4655 366
c01c0395
JA
367 if (!td->error) {
368 min_events = td->cur_depth;
369
370 if (min_events)
371 ret = io_u_queued_complete(td, min_events);
372 } else
ebac4655
JA
373 cleanup_pending_aio(td);
374
375 td_set_runstate(td, TD_RUNNING);
376}
377
32cd46a0 378/*
906c8d75 379 * Main IO worker function. It retrieves io_u's to process and queues
32cd46a0
JA
380 * and reaps them, checking for rate and errors along the way.
381 */
ebac4655
JA
382static void do_io(struct thread_data *td)
383{
02bcaa8c 384 struct timeval s;
ebac4655 385 unsigned long usec;
af52b345
JA
386 unsigned int i;
387 int ret = 0;
ebac4655 388
5853e5a8
JA
389 td_set_runstate(td, TD_RUNNING);
390
3951414d 391 while ((td->this_io_bytes[0] + td->this_io_bytes[1]) < td->io_size) {
97601024
JA
392 struct timeval comp_time;
393 long bytes_done = 0;
84585003 394 int min_evts = 0;
ebac4655 395 struct io_u *io_u;
5451792e 396 int ret2;
ebac4655
JA
397
398 if (td->terminate)
399 break;
400
3d7c391d 401 io_u = get_io_u(td);
ebac4655
JA
402 if (!io_u)
403 break;
404
405 memcpy(&s, &io_u->start_time, sizeof(s));
97601024
JA
406
407 if (runtime_exceeded(td, &s)) {
408 put_io_u(td, io_u);
409 break;
410 }
36167d82 411
11786802 412 ret = td_io_queue(td, io_u);
36167d82
JA
413 switch (ret) {
414 case FIO_Q_COMPLETED:
5451792e
JA
415 if (io_u->error)
416 ret = -io_u->error;
9e9d164e 417 else if (io_u->resid) {
36167d82
JA
418 int bytes = io_u->xfer_buflen - io_u->resid;
419
9e9d164e
JA
420 /*
421 * zero read, fail
422 */
423 if (!bytes) {
424 td_verror(td, ENODATA, "full resid");
425 put_io_u(td, io_u);
426 break;
427 }
428
cec6b55d 429 io_u->xfer_buflen = io_u->resid;
36167d82 430 io_u->xfer_buf += bytes;
11786802
JA
431 requeue_io_u(td, &io_u);
432 } else {
433 fio_gettime(&comp_time, NULL);
434 bytes_done = io_u_sync_complete(td, io_u);
435 if (bytes_done < 0)
436 ret = bytes_done;
cec6b55d 437 }
36167d82
JA
438 break;
439 case FIO_Q_QUEUED:
7e77dd02
JA
440 /*
441 * if the engine doesn't have a commit hook,
442 * the io_u is really queued. if it does have such
443 * a hook, it has to call io_u_queued() itself.
444 */
445 if (td->io_ops->commit == NULL)
446 io_u_queued(td, io_u);
36167d82 447 break;
755200a3
JA
448 case FIO_Q_BUSY:
449 requeue_io_u(td, &io_u);
5451792e
JA
450 ret2 = td_io_commit(td);
451 if (ret2 < 0)
452 ret = ret2;
755200a3 453 break;
36167d82
JA
454 default:
455 assert(ret < 0);
456 put_io_u(td, io_u);
457 break;
ebac4655
JA
458 }
459
99784632 460 if (ret < 0 || td->error)
36167d82
JA
461 break;
462
97601024
JA
463 /*
464 * See if we need to complete some commands
465 */
755200a3 466 if (ret == FIO_Q_QUEUED || ret == FIO_Q_BUSY) {
97601024 467 min_evts = 0;
e916b390 468 if (queue_full(td) || ret == FIO_Q_BUSY) {
36167d82 469 min_evts = 1;
ebac4655 470
e916b390
JA
471 if (td->cur_depth > td->iodepth_low)
472 min_evts = td->cur_depth - td->iodepth_low;
473 }
474
97601024 475 fio_gettime(&comp_time, NULL);
d7762cf8 476 bytes_done = io_u_queued_complete(td, min_evts);
97601024 477 if (bytes_done < 0)
36167d82 478 break;
ebac4655
JA
479 }
480
97601024
JA
481 if (!bytes_done)
482 continue;
483
ebac4655
JA
484 /*
485 * the rate is batched for now, it should work for batches
486 * of completions except the very first one which may look
487 * a little bursty
488 */
97601024 489 usec = utime_since(&s, &comp_time);
ebac4655 490
413dd459 491 rate_throttle(td, usec, bytes_done);
ebac4655 492
97601024 493 if (check_min_rate(td, &comp_time)) {
98aa62d8 494 if (exitall_on_terminate)
390c40e2 495 terminate_threads(td->groupid);
e1161c32 496 td_verror(td, ENODATA, "check_min_rate");
ebac4655
JA
497 break;
498 }
499
9c1f7434
JA
500 if (td->thinktime) {
501 unsigned long long b;
502
503 b = td->io_blocks[0] + td->io_blocks[1];
48097d5c
JA
504 if (!(b % td->thinktime_blocks)) {
505 int left;
506
507 if (td->thinktime_spin)
508 __usec_sleep(td->thinktime_spin);
509
510 left = td->thinktime - td->thinktime_spin;
511 if (left)
512 usec_sleep(td, left);
513 }
9c1f7434 514 }
ebac4655
JA
515 }
516
4d2413c6 517 if (!td->error) {
3d7c391d
JA
518 struct fio_file *f;
519
c01c0395
JA
520 i = td->cur_depth;
521 if (i)
522 ret = io_u_queued_complete(td, i);
ebac4655 523
84585003
JA
524 if (should_fsync(td) && td->end_fsync) {
525 td_set_runstate(td, TD_FSYNCING);
3d7b485f
JA
526
527 for_each_file(td, f, i) {
528 if (!(f->flags & FIO_FILE_OPEN))
529 continue;
858a3d47 530 fio_io_sync(td, f);
3d7b485f 531 }
84585003 532 }
c01c0395
JA
533 } else
534 cleanup_pending_aio(td);
ebac4655
JA
535}
536
ebac4655
JA
537static void cleanup_io_u(struct thread_data *td)
538{
539 struct list_head *entry, *n;
540 struct io_u *io_u;
541
542 list_for_each_safe(entry, n, &td->io_u_freelist) {
543 io_u = list_entry(entry, struct io_u, list);
544
545 list_del(&io_u->list);
546 free(io_u);
547 }
548
2f9ade3c 549 free_io_mem(td);
ebac4655
JA
550}
551
6b9cea23
JA
552/*
553 * "randomly" fill the buffer contents
554 */
66eeb296 555static void fill_rand_buf(struct io_u *io_u, int max_bs)
6b9cea23 556{
66eeb296 557 int *ptr = io_u->buf;
6b9cea23
JA
558
559 while ((void *) ptr - io_u->buf < max_bs) {
560 *ptr = rand() * 0x9e370001;
561 ptr++;
562 }
563}
564
ebac4655
JA
565static int init_io_u(struct thread_data *td)
566{
567 struct io_u *io_u;
a00735e6 568 unsigned int max_bs;
ebac4655
JA
569 int i, max_units;
570 char *p;
571
2866c82d 572 if (td->io_ops->flags & FIO_SYNCIO)
ebac4655
JA
573 max_units = 1;
574 else
575 max_units = td->iodepth;
576
a00735e6 577 max_bs = max(td->max_bs[DDIR_READ], td->max_bs[DDIR_WRITE]);
74b025b0
JA
578 td->orig_buffer_size = max_bs * max_units;
579
d0bdaf49 580 if (td->mem_type == MEM_SHMHUGE || td->mem_type == MEM_MMAPHUGE)
56bb17f2 581 td->orig_buffer_size = (td->orig_buffer_size + td->hugepage_size - 1) & ~(td->hugepage_size - 1);
74b025b0 582 else
29d610e1 583 td->orig_buffer_size += page_mask;
ebac4655 584
2f9ade3c
JA
585 if (allocate_io_mem(td))
586 return 1;
ebac4655 587
ebac4655
JA
588 p = ALIGN(td->orig_buffer);
589 for (i = 0; i < max_units; i++) {
590 io_u = malloc(sizeof(*io_u));
591 memset(io_u, 0, sizeof(*io_u));
592 INIT_LIST_HEAD(&io_u->list);
593
a00735e6 594 io_u->buf = p + max_bs * i;
6b9cea23 595 if (td_write(td) || td_rw(td))
a00735e6 596 fill_rand_buf(io_u, max_bs);
6b9cea23 597
b1ff3403 598 io_u->index = i;
0c6e7517 599 io_u->flags = IO_U_F_FREE;
ebac4655
JA
600 list_add(&io_u->list, &td->io_u_freelist);
601 }
602
433afcb4
JA
603 io_u_init_timeout();
604
ebac4655
JA
605 return 0;
606}
607
da86774e
JA
608static int switch_ioscheduler(struct thread_data *td)
609{
610 char tmp[256], tmp2[128];
611 FILE *f;
612 int ret;
613
ba0fbe10 614 if (td->io_ops->flags & FIO_DISKLESSIO)
f48b467c
JA
615 return 0;
616
da86774e
JA
617 sprintf(tmp, "%s/queue/scheduler", td->sysfs_root);
618
619 f = fopen(tmp, "r+");
620 if (!f) {
e1161c32 621 td_verror(td, errno, "fopen");
da86774e
JA
622 return 1;
623 }
624
625 /*
626 * Set io scheduler.
627 */
628 ret = fwrite(td->ioscheduler, strlen(td->ioscheduler), 1, f);
629 if (ferror(f) || ret != 1) {
e1161c32 630 td_verror(td, errno, "fwrite");
da86774e
JA
631 fclose(f);
632 return 1;
633 }
634
635 rewind(f);
636
637 /*
638 * Read back and check that the selected scheduler is now the default.
639 */
640 ret = fread(tmp, 1, sizeof(tmp), f);
641 if (ferror(f) || ret < 0) {
e1161c32 642 td_verror(td, errno, "fread");
da86774e
JA
643 fclose(f);
644 return 1;
645 }
646
647 sprintf(tmp2, "[%s]", td->ioscheduler);
648 if (!strstr(tmp, tmp2)) {
3b70d7e5 649 log_err("fio: io scheduler %s not found\n", td->ioscheduler);
e1161c32 650 td_verror(td, EINVAL, "iosched_switch");
da86774e
JA
651 fclose(f);
652 return 1;
653 }
654
655 fclose(f);
656 return 0;
657}
658
a978ba68 659static int clear_io_state(struct thread_data *td)
ebac4655 660{
53cdc686 661 struct fio_file *f;
af52b345
JA
662 unsigned int i;
663 int ret;
ebac4655 664
756867bd 665 td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0;
ebac4655 666 td->this_io_bytes[0] = td->this_io_bytes[1] = 0;
20dc95c4 667 td->zone_bytes = 0;
d7d3b49b 668 td->rate_bytes = 0;
ebac4655 669
c1324df1
JA
670 td->last_was_sync = 0;
671
a978ba68
JA
672 for_each_file(td, f, i)
673 td_io_close_file(td, f);
53cdc686 674
a978ba68
JA
675 ret = 0;
676 for_each_file(td, f, i) {
677 ret = td_io_open_file(td, f);
678 if (ret)
679 break;
53cdc686 680 }
a978ba68
JA
681
682 return ret;
ebac4655
JA
683}
684
906c8d75
JA
685/*
686 * Entry point for the thread based jobs. The process based jobs end up
687 * here as well, after a little setup.
688 */
ebac4655
JA
689static void *thread_main(void *data)
690{
69008999 691 unsigned long long runtime[2];
ebac4655 692 struct thread_data *td = data;
a978ba68 693 int clear_state;
ebac4655
JA
694
695 if (!td->use_thread)
696 setsid();
697
698 td->pid = getpid();
699
aea47d44
JA
700 INIT_LIST_HEAD(&td->io_u_freelist);
701 INIT_LIST_HEAD(&td->io_u_busylist);
755200a3 702 INIT_LIST_HEAD(&td->io_u_requeues);
aea47d44
JA
703 INIT_LIST_HEAD(&td->io_hist_list);
704 INIT_LIST_HEAD(&td->io_log_list);
705
ebac4655 706 if (init_io_u(td))
bda4fd9e 707 goto err_sem;
ebac4655
JA
708
709 if (fio_setaffinity(td) == -1) {
e1161c32 710 td_verror(td, errno, "cpu_set_affinity");
bda4fd9e 711 goto err_sem;
ebac4655
JA
712 }
713
aea47d44 714 if (init_iolog(td))
bda4fd9e 715 goto err_sem;
aea47d44 716
ebac4655
JA
717 if (td->ioprio) {
718 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
e1161c32 719 td_verror(td, errno, "ioprio_set");
bda4fd9e 720 goto err_sem;
ebac4655
JA
721 }
722 }
723
1056eaad 724 if (nice(td->nice) == -1) {
e1161c32 725 td_verror(td, errno, "nice");
bda4fd9e 726 goto err_sem;
b6f4d880
JA
727 }
728
56f9498d 729 if (td->ioscheduler && switch_ioscheduler(td))
bda4fd9e 730 goto err_sem;
da86774e 731
75154845 732 td_set_runstate(td, TD_INITIALIZED);
07739b57
JA
733 fio_sem_up(startup_sem);
734 fio_sem_down(td->mutex);
ebac4655 735
37f56873
JA
736 /*
737 * the ->mutex semaphore is now no longer used, close it to avoid
738 * eating a file descriptor
739 */
740 fio_sem_remove(td->mutex);
741
53cdc686 742 if (!td->create_serialize && setup_files(td))
ebac4655
JA
743 goto err;
744
7d6c5283
JA
745 if (td_io_init(td))
746 goto err;
747
b5af8293
JA
748 if (open_files(td))
749 goto err;
750
69cfd7e0
JA
751 if (td->exec_prerun) {
752 if (system(td->exec_prerun) < 0)
753 goto err;
754 }
4e0ba8af 755
69008999 756 fio_gettime(&td->epoch, NULL);
433afcb4 757 memcpy(&td->timeout_end, &td->epoch, sizeof(td->epoch));
756867bd 758 getrusage(RUSAGE_SELF, &td->ts.ru_start);
69008999
JA
759
760 runtime[0] = runtime[1] = 0;
a978ba68 761 clear_state = 0;
ebac4655 762 while (td->loops--) {
02bcaa8c 763 fio_gettime(&td->start, NULL);
756867bd 764 memcpy(&td->ts.stat_sample_time, &td->start, sizeof(td->start));
ebac4655
JA
765
766 if (td->ratemin)
756867bd 767 memcpy(&td->lastrate, &td->ts.stat_sample_time, sizeof(td->lastrate));
ebac4655 768
a978ba68
JA
769 if (clear_state && clear_io_state(td))
770 break;
771
ebac4655
JA
772 prune_io_piece_log(td);
773
ba0fbe10 774 do_io(td);
ebac4655 775
a978ba68
JA
776 clear_state = 1;
777
413dd459
JA
778 if (td_read(td) && td->io_bytes[DDIR_READ])
779 runtime[DDIR_READ] += utime_since_now(&td->start);
780 if (td_write(td) && td->io_bytes[DDIR_WRITE])
781 runtime[DDIR_WRITE] += utime_since_now(&td->start);
782
ebac4655
JA
783 if (td->error || td->terminate)
784 break;
785
786 if (td->verify == VERIFY_NONE)
787 continue;
788
a978ba68
JA
789 if (clear_io_state(td))
790 break;
791
02bcaa8c 792 fio_gettime(&td->start, NULL);
ebac4655
JA
793
794 do_verify(td);
795
69008999 796 runtime[DDIR_READ] += utime_since_now(&td->start);
ebac4655
JA
797
798 if (td->error || td->terminate)
799 break;
800 }
801
36dff966 802 update_rusage_stat(td);
756867bd
JA
803 td->ts.runtime[0] = runtime[0] / 1000;
804 td->ts.runtime[1] = runtime[1] / 1000;
805 td->ts.total_run_time = mtime_since_now(&td->epoch);
806 td->ts.io_bytes[0] = td->io_bytes[0];
807 td->ts.io_bytes[1] = td->io_bytes[1];
808
809 if (td->ts.bw_log)
810 finish_log(td, td->ts.bw_log, "bw");
811 if (td->ts.slat_log)
812 finish_log(td, td->ts.slat_log, "slat");
813 if (td->ts.clat_log)
814 finish_log(td, td->ts.clat_log, "clat");
076efc7c 815 if (td->write_iolog_file)
843a7413 816 write_iolog_close(td);
69cfd7e0
JA
817 if (td->exec_postrun) {
818 if (system(td->exec_postrun) < 0)
819 log_err("fio: postrun %s failed\n", td->exec_postrun);
820 }
ebac4655
JA
821
822 if (exitall_on_terminate)
390c40e2 823 terminate_threads(td->groupid);
ebac4655
JA
824
825err:
5bf13a5a
JA
826 if (td->error)
827 printf("fio: pid=%d, err=%d/%s\n", td->pid, td->error, td->verror);
53cdc686 828 close_files(td);
2866c82d 829 close_ioengine(td);
ebac4655 830 cleanup_io_u(td);
ebac4655 831 td_set_runstate(td, TD_EXITED);
43d76807 832 return (void *) (unsigned long) td->error;
bda4fd9e
JA
833err_sem:
834 fio_sem_up(startup_sem);
835 goto err;
ebac4655
JA
836}
837
906c8d75
JA
838/*
839 * We cannot pass the td data into a forked process, so attach the td and
840 * pass it to the thread worker.
841 */
a6418147 842static int fork_main(int shmid, int offset)
ebac4655
JA
843{
844 struct thread_data *td;
a6418147 845 void *data, *ret;
ebac4655
JA
846
847 data = shmat(shmid, NULL, 0);
848 if (data == (void *) -1) {
a6418147
JA
849 int __err = errno;
850
ebac4655 851 perror("shmat");
a6418147 852 return __err;
ebac4655
JA
853 }
854
855 td = data + offset * sizeof(struct thread_data);
a6418147 856 ret = thread_main(td);
ebac4655 857 shmdt(data);
43d76807 858 return (int) (unsigned long) ret;
ebac4655
JA
859}
860
906c8d75
JA
861/*
862 * Run over the job map and reap the threads that have exited, if any.
863 */
ebac4655
JA
864static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
865{
34572e28 866 struct thread_data *td;
fab6aa71 867 int i, cputhreads, pending, status, ret;
ebac4655
JA
868
869 /*
870 * reap exited threads (TD_EXITED -> TD_REAPED)
871 */
4d2413c6 872 pending = cputhreads = 0;
34572e28 873 for_each_td(td, i) {
3707f45b 874 int flags = 0;
a2f77c9f 875
84585003
JA
876 /*
877 * ->io_ops is NULL for a thread that has closed its
878 * io engine
879 */
ba0fbe10 880 if (td->io_ops && !strcmp(td->io_ops->name, "cpuio"))
b990b5c0
JA
881 cputhreads++;
882
a2f77c9f
JA
883 if (!td->pid || td->runstate == TD_REAPED)
884 continue;
3707f45b
JA
885 if (td->use_thread) {
886 if (td->runstate == TD_EXITED) {
887 td_set_runstate(td, TD_REAPED);
888 goto reaped;
889 }
890 continue;
891 }
a2f77c9f
JA
892
893 flags = WNOHANG;
894 if (td->runstate == TD_EXITED)
895 flags = 0;
896
897 /*
898 * check if someone quit or got killed in an unusual way
899 */
900 ret = waitpid(td->pid, &status, flags);
3707f45b 901 if (ret < 0) {
a2f77c9f
JA
902 if (errno == ECHILD) {
903 log_err("fio: pid=%d disappeared %d\n", td->pid, td->runstate);
904 td_set_runstate(td, TD_REAPED);
905 goto reaped;
906 }
907 perror("waitpid");
908 } else if (ret == td->pid) {
909 if (WIFSIGNALED(status)) {
fab6aa71
JA
910 int sig = WTERMSIG(status);
911
d9244941
JA
912 if (sig != SIGQUIT)
913 log_err("fio: pid=%d, got signal=%d\n", td->pid, sig);
fab6aa71
JA
914 td_set_runstate(td, TD_REAPED);
915 goto reaped;
916 }
a2f77c9f
JA
917 if (WIFEXITED(status)) {
918 if (WEXITSTATUS(status) && !td->error)
919 td->error = WEXITSTATUS(status);
a2f77c9f 920
a2f77c9f
JA
921 td_set_runstate(td, TD_REAPED);
922 goto reaped;
a6418147
JA
923 }
924 }
ebac4655 925
a2f77c9f
JA
926 /*
927 * thread is not dead, continue
928 */
929 continue;
fab6aa71 930reaped:
3707f45b
JA
931 if (td->use_thread) {
932 long ret;
933
934 if (pthread_join(td->thread, (void *) &ret))
935 perror("pthread_join");
936 }
937
ebac4655
JA
938 (*nr_running)--;
939 (*m_rate) -= td->ratemin;
940 (*t_rate) -= td->rate;
a2f77c9f
JA
941
942 if (td->error)
943 exit_value++;
ebac4655 944 }
b990b5c0 945
4d2413c6 946 if (*nr_running == cputhreads && !pending)
390c40e2 947 terminate_threads(TERMINATE_ALL);
ebac4655
JA
948}
949
906c8d75
JA
950/*
951 * Main function for kicking off and reaping jobs, as needed.
952 */
ebac4655
JA
953static void run_threads(void)
954{
ebac4655
JA
955 struct thread_data *td;
956 unsigned long spent;
957 int i, todo, nr_running, m_rate, t_rate, nr_started;
fcb6ade2 958
2f9ade3c
JA
959 if (fio_pin_memory())
960 return;
ebac4655 961
c6ae0a5b 962 if (!terse_output) {
9cedf167
JA
963 printf("Starting ");
964 if (nr_thread)
965 printf("%d thread%s", nr_thread, nr_thread > 1 ? "s" : "");
966 if (nr_process) {
967 if (nr_thread)
968 printf(" and ");
969 printf("%d process%s", nr_process, nr_process > 1 ? "es" : "");
970 }
971 printf("\n");
c6ae0a5b
JA
972 fflush(stdout);
973 }
c04f7ec3 974
4efa970e
JA
975 signal(SIGINT, sig_handler);
976 signal(SIGALRM, sig_handler);
977
ebac4655
JA
978 todo = thread_number;
979 nr_running = 0;
980 nr_started = 0;
981 m_rate = t_rate = 0;
982
34572e28 983 for_each_td(td, i) {
263e529f 984 print_status_init(td->thread_number - 1);
ebac4655 985
380cf265
JA
986 if (!td->create_serialize) {
987 init_disk_util(td);
ebac4655 988 continue;
380cf265 989 }
ebac4655
JA
990
991 /*
992 * do file setup here so it happens sequentially,
993 * we don't want X number of threads getting their
994 * client data interspersed on disk
995 */
53cdc686 996 if (setup_files(td)) {
5bf13a5a
JA
997 exit_value++;
998 if (td->error)
999 log_err("fio: pid=%d, err=%d/%s\n", td->pid, td->error, td->verror);
ebac4655
JA
1000 td_set_runstate(td, TD_REAPED);
1001 todo--;
1002 }
380cf265
JA
1003
1004 init_disk_util(td);
ebac4655
JA
1005 }
1006
a2f77c9f
JA
1007 set_genesis_time();
1008
ebac4655 1009 while (todo) {
75154845
JA
1010 struct thread_data *map[MAX_JOBS];
1011 struct timeval this_start;
1012 int this_jobs = 0, left;
1013
ebac4655
JA
1014 /*
1015 * create threads (TD_NOT_CREATED -> TD_CREATED)
1016 */
34572e28 1017 for_each_td(td, i) {
ebac4655
JA
1018 if (td->runstate != TD_NOT_CREATED)
1019 continue;
1020
1021 /*
1022 * never got a chance to start, killed by other
1023 * thread for some reason
1024 */
1025 if (td->terminate) {
1026 todo--;
1027 continue;
1028 }
1029
1030 if (td->start_delay) {
263e529f 1031 spent = mtime_since_genesis();
ebac4655
JA
1032
1033 if (td->start_delay * 1000 > spent)
1034 continue;
1035 }
1036
1037 if (td->stonewall && (nr_started || nr_running))
1038 break;
1039
75154845
JA
1040 /*
1041 * Set state to created. Thread will transition
1042 * to TD_INITIALIZED when it's done setting up.
1043 */
ebac4655 1044 td_set_runstate(td, TD_CREATED);
75154845 1045 map[this_jobs++] = td;
ebac4655
JA
1046 nr_started++;
1047
1048 if (td->use_thread) {
1049 if (pthread_create(&td->thread, NULL, thread_main, td)) {
1050 perror("thread_create");
1051 nr_started--;
c8bb6faf 1052 break;
ebac4655
JA
1053 }
1054 } else {
07739b57 1055 if (!fork()) {
a6418147
JA
1056 int ret = fork_main(shm_id, i);
1057
1058 exit(ret);
ebac4655
JA
1059 }
1060 }
07739b57 1061 fio_sem_down(startup_sem);
ebac4655
JA
1062 }
1063
1064 /*
75154845
JA
1065 * Wait for the started threads to transition to
1066 * TD_INITIALIZED.
ebac4655 1067 */
02bcaa8c 1068 fio_gettime(&this_start, NULL);
75154845 1069 left = this_jobs;
6ce15a32 1070 while (left && !fio_abort) {
75154845
JA
1071 if (mtime_since_now(&this_start) > JOB_START_TIMEOUT)
1072 break;
1073
1074 usleep(100000);
1075
1076 for (i = 0; i < this_jobs; i++) {
1077 td = map[i];
1078 if (!td)
1079 continue;
b6f4d880 1080 if (td->runstate == TD_INITIALIZED) {
75154845
JA
1081 map[i] = NULL;
1082 left--;
b6f4d880
JA
1083 } else if (td->runstate >= TD_EXITED) {
1084 map[i] = NULL;
1085 left--;
1086 todo--;
1087 nr_running++; /* work-around... */
75154845
JA
1088 }
1089 }
1090 }
1091
1092 if (left) {
3b70d7e5 1093 log_err("fio: %d jobs failed to start\n", left);
75154845
JA
1094 for (i = 0; i < this_jobs; i++) {
1095 td = map[i];
1096 if (!td)
1097 continue;
1098 kill(td->pid, SIGTERM);
1099 }
1100 break;
1101 }
1102
1103 /*
b6f4d880 1104 * start created threads (TD_INITIALIZED -> TD_RUNNING).
75154845 1105 */
34572e28 1106 for_each_td(td, i) {
75154845 1107 if (td->runstate != TD_INITIALIZED)
ebac4655
JA
1108 continue;
1109
1110 td_set_runstate(td, TD_RUNNING);
1111 nr_running++;
1112 nr_started--;
1113 m_rate += td->ratemin;
1114 t_rate += td->rate;
75154845 1115 todo--;
07739b57 1116 fio_sem_up(td->mutex);
ebac4655
JA
1117 }
1118
1119 reap_threads(&nr_running, &t_rate, &m_rate);
1120
1121 if (todo)
1122 usleep(100000);
1123 }
1124
1125 while (nr_running) {
1126 reap_threads(&nr_running, &t_rate, &m_rate);
1127 usleep(10000);
1128 }
1129
1130 update_io_ticks();
2f9ade3c 1131 fio_unpin_memory();
ebac4655
JA
1132}
1133
ebac4655
JA
1134int main(int argc, char *argv[])
1135{
29d610e1
JA
1136 long ps;
1137
dbe1125e
JA
1138 /*
1139 * We need locale for number printing, if it isn't set then just
1140 * go with the US format.
1141 */
1142 if (!getenv("LC_NUMERIC"))
1143 setlocale(LC_NUMERIC, "en_US");
1144
ebac4655
JA
1145 if (parse_options(argc, argv))
1146 return 1;
1147
1148 if (!thread_number) {
3b70d7e5 1149 log_err("Nothing to do\n");
ebac4655
JA
1150 return 1;
1151 }
1152
29d610e1
JA
1153 ps = sysconf(_SC_PAGESIZE);
1154 if (ps < 0) {
1155 log_err("Failed to get page size\n");
1156 return 1;
1157 }
1158
cfc99db7 1159 page_size = ps;
29d610e1
JA
1160 page_mask = ps - 1;
1161
bb3884d8
JA
1162 if (write_bw_log) {
1163 setup_log(&agg_io_log[DDIR_READ]);
1164 setup_log(&agg_io_log[DDIR_WRITE]);
1165 }
1166
07739b57
JA
1167 startup_sem = fio_sem_init(0);
1168
a2f77c9f
JA
1169 set_genesis_time();
1170
ebac4655
JA
1171 disk_util_timer_arm();
1172
1173 run_threads();
6ce15a32 1174
bb3884d8 1175 if (!fio_abort) {
6ce15a32 1176 show_run_stats();
bb3884d8
JA
1177 if (write_bw_log) {
1178 __finish_log(agg_io_log[DDIR_READ],"agg-read_bw.log");
1179 __finish_log(agg_io_log[DDIR_WRITE],"agg-write_bw.log");
1180 }
1181 }
ebac4655 1182
07739b57 1183 fio_sem_remove(startup_sem);
437c9b71 1184 return exit_value;
ebac4655 1185}