Fix leaks with using get_opt_postfix()
[fio.git] / fio.c
CommitLineData
ebac4655
JA
1/*
2 * fio - the flexible io tester
3 *
4 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
aae22ca7 5 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
ebac4655 6 *
8e9fe637
JA
7 * The license below covers all files distributed with fio unless otherwise
8 * noted in the file itself.
9 *
ebac4655 10 * This program is free software; you can redistribute it and/or modify
8e9fe637
JA
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
ebac4655
JA
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 *
23 */
ebac4655
JA
24#include <unistd.h>
25#include <fcntl.h>
26#include <string.h>
ebac4655
JA
27#include <signal.h>
28#include <time.h>
dbe1125e 29#include <locale.h>
36167d82 30#include <assert.h>
ebac4655
JA
31#include <sys/stat.h>
32#include <sys/wait.h>
33#include <sys/ipc.h>
34#include <sys/shm.h>
ebac4655
JA
35#include <sys/mman.h>
36
37#include "fio.h"
210f43b3 38#include "hash.h"
2e5cdb11 39#include "smalloc.h"
ebac4655 40
cfc99db7
JA
41unsigned long page_mask;
42unsigned long page_size;
29d610e1
JA
43#define ALIGN(buf) \
44 (char *) (((unsigned long) (buf) + page_mask) & ~page_mask)
ebac4655
JA
45
46int groupid = 0;
47int thread_number = 0;
9cedf167
JA
48int nr_process = 0;
49int nr_thread = 0;
ebac4655 50int shm_id = 0;
53cdc686 51int temp_stall_ts;
d3eeeabc 52unsigned long done_secs = 0;
ebac4655 53
cdd18ad8 54static struct fio_mutex *startup_mutex;
6ce15a32 55static volatile int fio_abort;
437c9b71 56static int exit_value;
ccb0fa24 57static struct itimerval itimer;
ebac4655 58
bb3884d8
JA
59struct io_log *agg_io_log[2];
60
ebac4655 61#define TERMINATE_ALL (-1)
75154845 62#define JOB_START_TIMEOUT (5 * 1000)
ebac4655 63
b29ee5b3 64void td_set_runstate(struct thread_data *td, int runstate)
6ce15a32 65{
d8fab1b6
JA
66 if (td->runstate == runstate)
67 return;
68
5921e80c
JA
69 dprint(FD_PROCESS, "pid=%d: runstate %d -> %d\n", (int) td->pid,
70 td->runstate, runstate);
6ce15a32
JA
71 td->runstate = runstate;
72}
73
390c40e2 74static void terminate_threads(int group_id)
ebac4655 75{
34572e28 76 struct thread_data *td;
ebac4655
JA
77 int i;
78
6f88bcb0
JA
79 dprint(FD_PROCESS, "terminate group_id=%d\n", group_id);
80
34572e28 81 for_each_td(td, i) {
ebac4655 82 if (group_id == TERMINATE_ALL || groupid == td->groupid) {
6f88bcb0 83 dprint(FD_PROCESS, "setting terminate on %s/%d\n",
5921e80c 84 td->o.name, (int) td->pid);
ad830ed7
JA
85 td->terminate = 1;
86 td->o.start_delay = 0;
87
7a93ab19
JA
88 /*
89 * if the thread is running, just let it exit
90 */
91 if (td->runstate < TD_RUNNING)
c1302d44 92 kill(td->pid, SIGQUIT);
f63f7f3b
JA
93 else {
94 struct ioengine_ops *ops = td->io_ops;
95
96 if (ops && (ops->flags & FIO_SIGQUIT))
97 kill(td->pid, SIGQUIT);
98 }
ebac4655
JA
99 }
100 }
101}
102
ccb0fa24
JA
103static void status_timer_arm(void)
104{
105 itimer.it_value.tv_sec = 0;
106 itimer.it_value.tv_usec = DISK_UTIL_MSEC * 1000;
107 setitimer(ITIMER_REAL, &itimer, NULL);
108}
109
02444ad1 110static void sig_alrm(int fio_unused sig)
ebac4655 111{
697a606c 112 if (threads) {
5ec10eaa 113 update_io_ticks();
5ec10eaa 114 print_thread_status();
ccb0fa24 115 status_timer_arm();
697a606c
JA
116 }
117}
118
119static void sig_int(int sig)
120{
121 if (threads) {
5ec10eaa
JA
122 printf("\nfio: terminating on signal %d\n", sig);
123 fflush(stdout);
124 terminate_threads(TERMINATE_ALL);
ebac4655
JA
125 }
126}
127
02444ad1 128static void sig_ill(int fio_unused sig)
de79c915
JA
129{
130 if (!threads)
131 return;
132
133 log_err("fio: illegal instruction. your cpu does not support "
134 "the sse4.2 instruction for crc32c\n");
135 terminate_threads(TERMINATE_ALL);
136 exit(4);
137}
138
697a606c
JA
139static void set_sig_handlers(void)
140{
141 struct sigaction act;
142
143 memset(&act, 0, sizeof(act));
144 act.sa_handler = sig_alrm;
145 act.sa_flags = SA_RESTART;
146 sigaction(SIGALRM, &act, NULL);
147
148 memset(&act, 0, sizeof(act));
149 act.sa_handler = sig_int;
150 act.sa_flags = SA_RESTART;
151 sigaction(SIGINT, &act, NULL);
de79c915
JA
152
153 memset(&act, 0, sizeof(act));
154 act.sa_handler = sig_ill;
155 act.sa_flags = SA_RESTART;
156 sigaction(SIGILL, &act, NULL);
697a606c
JA
157}
158
993bf48b
JA
159static inline int should_check_rate(struct thread_data *td)
160{
161 /*
162 * No minimum rate set, always ok
163 */
164 if (!td->o.ratemin && !td->o.rate_iops_min)
165 return 0;
166
167 return 1;
168}
169
906c8d75
JA
170/*
171 * Check if we are above the minimum rate given.
172 */
ebac4655
JA
173static int check_min_rate(struct thread_data *td, struct timeval *now)
174{
0904200b 175 unsigned long long bytes = 0;
4e991c23 176 unsigned long iops = 0;
ebac4655
JA
177 unsigned long spent;
178 unsigned long rate;
ebac4655
JA
179
180 /*
181 * allow a 2 second settle period in the beginning
182 */
183 if (mtime_since(&td->start, now) < 2000)
184 return 0;
185
4e991c23
JA
186 if (td_read(td)) {
187 iops += td->io_blocks[DDIR_READ];
0904200b 188 bytes += td->this_io_bytes[DDIR_READ];
4e991c23
JA
189 }
190 if (td_write(td)) {
191 iops += td->io_blocks[DDIR_WRITE];
0904200b 192 bytes += td->this_io_bytes[DDIR_WRITE];
4e991c23 193 }
0904200b 194
ebac4655
JA
195 /*
196 * if rate blocks is set, sample is running
197 */
4e991c23 198 if (td->rate_bytes || td->rate_blocks) {
ebac4655 199 spent = mtime_since(&td->lastrate, now);
2dc1bbeb 200 if (spent < td->o.ratecycle)
ebac4655
JA
201 return 0;
202
2dc1bbeb 203 if (td->o.rate) {
4e991c23
JA
204 /*
205 * check bandwidth specified rate
206 */
207 if (bytes < td->rate_bytes) {
5ec10eaa
JA
208 log_err("%s: min rate %u not met\n", td->o.name,
209 td->o.ratemin);
4e991c23
JA
210 return 1;
211 } else {
212 rate = (bytes - td->rate_bytes) / spent;
5ec10eaa
JA
213 if (rate < td->o.ratemin ||
214 bytes < td->rate_bytes) {
215 log_err("%s: min rate %u not met, got"
216 " %luKiB/sec\n", td->o.name,
217 td->o.ratemin, rate);
4e991c23
JA
218 return 1;
219 }
220 }
413dd459 221 } else {
4e991c23
JA
222 /*
223 * checks iops specified rate
224 */
2dc1bbeb 225 if (iops < td->o.rate_iops) {
5ec10eaa
JA
226 log_err("%s: min iops rate %u not met\n",
227 td->o.name, td->o.rate_iops);
413dd459 228 return 1;
4e991c23
JA
229 } else {
230 rate = (iops - td->rate_blocks) / spent;
5ec10eaa
JA
231 if (rate < td->o.rate_iops_min ||
232 iops < td->rate_blocks) {
233 log_err("%s: min iops rate %u not met,"
234 " got %lu\n", td->o.name,
235 td->o.rate_iops_min,
236 rate);
4e991c23 237 }
413dd459 238 }
ebac4655
JA
239 }
240 }
241
0904200b 242 td->rate_bytes = bytes;
4e991c23 243 td->rate_blocks = iops;
ebac4655
JA
244 memcpy(&td->lastrate, now, sizeof(*now));
245 return 0;
246}
247
248static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
249{
2dc1bbeb 250 if (!td->o.timeout)
ebac4655 251 return 0;
2dc1bbeb 252 if (mtime_since(&td->epoch, t) >= td->o.timeout * 1000)
ebac4655
JA
253 return 1;
254
255 return 0;
256}
257
906c8d75
JA
258/*
259 * When job exits, we can cancel the in-flight IO if we are using async
260 * io. Attempt to do so.
261 */
ebac4655
JA
262static void cleanup_pending_aio(struct thread_data *td)
263{
01743ee1 264 struct flist_head *entry, *n;
ebac4655
JA
265 struct io_u *io_u;
266 int r;
267
268 /*
269 * get immediately available events, if any
270 */
d7762cf8 271 r = io_u_queued_complete(td, 0);
b2fdda43
JA
272 if (r < 0)
273 return;
ebac4655
JA
274
275 /*
276 * now cancel remaining active events
277 */
2866c82d 278 if (td->io_ops->cancel) {
01743ee1
JA
279 flist_for_each_safe(entry, n, &td->io_u_busylist) {
280 io_u = flist_entry(entry, struct io_u, list);
ebac4655 281
0c6e7517
JA
282 /*
283 * if the io_u isn't in flight, then that generally
284 * means someone leaked an io_u. complain but fix
285 * it up, so we don't stall here.
286 */
287 if ((io_u->flags & IO_U_F_FLIGHT) == 0) {
288 log_err("fio: non-busy IO on busy list\n");
ebac4655 289 put_io_u(td, io_u);
0c6e7517
JA
290 } else {
291 r = td->io_ops->cancel(td, io_u);
292 if (!r)
293 put_io_u(td, io_u);
294 }
ebac4655
JA
295 }
296 }
297
97601024 298 if (td->cur_depth)
d7762cf8 299 r = io_u_queued_complete(td, td->cur_depth);
ebac4655
JA
300}
301
858a3d47
JA
302/*
303 * Helper to handle the final sync of a file. Works just like the normal
304 * io path, just does everything sync.
305 */
306static int fio_io_sync(struct thread_data *td, struct fio_file *f)
307{
308 struct io_u *io_u = __get_io_u(td);
858a3d47
JA
309 int ret;
310
311 if (!io_u)
312 return 1;
313
314 io_u->ddir = DDIR_SYNC;
315 io_u->file = f;
316
317 if (td_io_prep(td, io_u)) {
318 put_io_u(td, io_u);
319 return 1;
320 }
321
755200a3 322requeue:
858a3d47 323 ret = td_io_queue(td, io_u);
36167d82 324 if (ret < 0) {
e1161c32 325 td_verror(td, io_u->error, "td_io_queue");
858a3d47 326 put_io_u(td, io_u);
858a3d47 327 return 1;
36167d82 328 } else if (ret == FIO_Q_QUEUED) {
d7762cf8 329 if (io_u_queued_complete(td, 1) < 0)
36167d82 330 return 1;
36167d82
JA
331 } else if (ret == FIO_Q_COMPLETED) {
332 if (io_u->error) {
e1161c32 333 td_verror(td, io_u->error, "td_io_queue");
36167d82
JA
334 return 1;
335 }
858a3d47 336
d7762cf8 337 if (io_u_sync_complete(td, io_u) < 0)
b2fdda43 338 return 1;
755200a3
JA
339 } else if (ret == FIO_Q_BUSY) {
340 if (td_io_commit(td))
341 return 1;
342 goto requeue;
858a3d47
JA
343 }
344
345 return 0;
346}
347
993bf48b
JA
348static inline void update_tv_cache(struct thread_data *td)
349{
350 if ((++td->tv_cache_nr & td->tv_cache_mask) == td->tv_cache_mask)
351 fio_gettime(&td->tv_cache, NULL);
352}
353
906c8d75 354/*
34403fb1 355 * The main verify engine. Runs over the writes we previously submitted,
906c8d75
JA
356 * reads the blocks back in, and checks the crc/md5 of the data.
357 */
1e97cce9 358static void do_verify(struct thread_data *td)
ebac4655 359{
53cdc686 360 struct fio_file *f;
36167d82 361 struct io_u *io_u;
af52b345
JA
362 int ret, min_events;
363 unsigned int i;
e5b401d4
JA
364
365 /*
366 * sync io first and invalidate cache, to make sure we really
367 * read from disk.
368 */
369 for_each_file(td, f, i) {
3d7b485f
JA
370 if (!(f->flags & FIO_FILE_OPEN))
371 continue;
b2fdda43
JA
372 if (fio_io_sync(td, f))
373 break;
374 if (file_invalidate_cache(td, f))
375 break;
e5b401d4 376 }
ebac4655 377
b2fdda43
JA
378 if (td->error)
379 return;
380
ebac4655
JA
381 td_set_runstate(td, TD_VERIFYING);
382
36167d82
JA
383 io_u = NULL;
384 while (!td->terminate) {
4950421a 385 int ret2, full;
5451792e 386
ebac4655
JA
387 io_u = __get_io_u(td);
388 if (!io_u)
389 break;
390
993bf48b
JA
391 update_tv_cache(td);
392
393 if (runtime_exceeded(td, &td->tv_cache)) {
069c2918 394 put_io_u(td, io_u);
90a87d4b 395 td->terminate = 1;
02bcaa8c 396 break;
069c2918 397 }
02bcaa8c 398
069c2918
JA
399 if (get_next_verify(td, io_u)) {
400 put_io_u(td, io_u);
ebac4655 401 break;
069c2918 402 }
ebac4655 403
069c2918
JA
404 if (td_io_prep(td, io_u)) {
405 put_io_u(td, io_u);
53cdc686 406 break;
069c2918 407 }
d7762cf8
JA
408
409 io_u->end_io = verify_io_u;
53cdc686 410
11786802 411 ret = td_io_queue(td, io_u);
36167d82
JA
412 switch (ret) {
413 case FIO_Q_COMPLETED:
414 if (io_u->error)
22819ec2 415 ret = -io_u->error;
9e9d164e 416 else if (io_u->resid) {
36167d82 417 int bytes = io_u->xfer_buflen - io_u->resid;
d460eb31 418 struct fio_file *f = io_u->file;
ebac4655 419
9e9d164e
JA
420 /*
421 * zero read, fail
422 */
423 if (!bytes) {
41d9f0fd 424 td_verror(td, EIO, "full resid");
9e9d164e
JA
425 put_io_u(td, io_u);
426 break;
427 }
8400d9b2 428
36167d82
JA
429 io_u->xfer_buflen = io_u->resid;
430 io_u->xfer_buf += bytes;
8400d9b2
JA
431 io_u->offset += bytes;
432
30061b97
JA
433 td->ts.short_io_u[io_u->ddir]++;
434
d460eb31 435 if (io_u->offset == f->real_file_size)
8400d9b2
JA
436 goto sync_done;
437
11786802
JA
438 requeue_io_u(td, &io_u);
439 } else {
8400d9b2 440sync_done:
11786802
JA
441 ret = io_u_sync_complete(td, io_u);
442 if (ret < 0)
443 break;
36167d82 444 }
36167d82
JA
445 continue;
446 case FIO_Q_QUEUED:
447 break;
755200a3
JA
448 case FIO_Q_BUSY:
449 requeue_io_u(td, &io_u);
5451792e
JA
450 ret2 = td_io_commit(td);
451 if (ret2 < 0)
452 ret = ret2;
755200a3 453 break;
36167d82
JA
454 default:
455 assert(ret < 0);
e1161c32 456 td_verror(td, -ret, "td_io_queue");
ebac4655
JA
457 break;
458 }
459
99784632 460 if (ret < 0 || td->error)
3af6ef39
JA
461 break;
462
ebac4655 463 /*
3af6ef39
JA
464 * if we can queue more, do so. but check if there are
465 * completed io_u's first.
ebac4655 466 */
4950421a
JA
467 full = queue_full(td) || ret == FIO_Q_BUSY;
468 if (full || !td->o.iodepth_batch_complete) {
469 min_events = td->o.iodepth_batch_complete;
470 if (full && !min_events)
838bc709 471 min_events = 1;
e916b390 472
4950421a
JA
473 do {
474 /*
475 * Reap required number of io units, if any,
476 * and do the verification on them through
477 * the callback handler
478 */
479 if (io_u_queued_complete(td, min_events) < 0) {
480 ret = -1;
481 break;
482 }
483 } while (full && (td->cur_depth > td->o.iodepth_low));
484 }
485 if (ret < 0)
ebac4655 486 break;
36167d82 487 }
ebac4655 488
c01c0395
JA
489 if (!td->error) {
490 min_events = td->cur_depth;
491
492 if (min_events)
493 ret = io_u_queued_complete(td, min_events);
494 } else
ebac4655
JA
495 cleanup_pending_aio(td);
496
497 td_set_runstate(td, TD_RUNNING);
498}
499
32cd46a0 500/*
906c8d75 501 * Main IO worker function. It retrieves io_u's to process and queues
32cd46a0
JA
502 * and reaps them, checking for rate and errors along the way.
503 */
ebac4655
JA
504static void do_io(struct thread_data *td)
505{
ebac4655 506 unsigned long usec;
af52b345
JA
507 unsigned int i;
508 int ret = 0;
ebac4655 509
b29ee5b3
JA
510 if (in_ramp_time(td))
511 td_set_runstate(td, TD_RAMP);
512 else
513 td_set_runstate(td, TD_RUNNING);
5853e5a8 514
7bb48f84 515 while ((td->this_io_bytes[0] + td->this_io_bytes[1]) < td->o.size) {
97601024
JA
516 struct timeval comp_time;
517 long bytes_done = 0;
84585003 518 int min_evts = 0;
ebac4655 519 struct io_u *io_u;
4950421a 520 int ret2, full;
ebac4655
JA
521
522 if (td->terminate)
523 break;
524
3d7c391d 525 io_u = get_io_u(td);
ebac4655
JA
526 if (!io_u)
527 break;
528
993bf48b 529 update_tv_cache(td);
97601024 530
993bf48b 531 if (runtime_exceeded(td, &td->tv_cache)) {
97601024 532 put_io_u(td, io_u);
90a87d4b 533 td->terminate = 1;
97601024
JA
534 break;
535 }
36167d82 536
b67740d3
JA
537 /*
538 * Add verification end_io handler, if asked to verify
539 * a previously written file.
540 */
cc3fc855 541 if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_READ) {
b67740d3 542 io_u->end_io = verify_io_u;
d8fab1b6 543 td_set_runstate(td, TD_VERIFYING);
b29ee5b3
JA
544 } else if (in_ramp_time(td))
545 td_set_runstate(td, TD_RAMP);
546 else
d8fab1b6 547 td_set_runstate(td, TD_RUNNING);
b67740d3 548
11786802 549 ret = td_io_queue(td, io_u);
36167d82
JA
550 switch (ret) {
551 case FIO_Q_COMPLETED:
5451792e
JA
552 if (io_u->error)
553 ret = -io_u->error;
9e9d164e 554 else if (io_u->resid) {
36167d82 555 int bytes = io_u->xfer_buflen - io_u->resid;
d460eb31 556 struct fio_file *f = io_u->file;
36167d82 557
9e9d164e
JA
558 /*
559 * zero read, fail
560 */
561 if (!bytes) {
41d9f0fd 562 td_verror(td, EIO, "full resid");
9e9d164e
JA
563 put_io_u(td, io_u);
564 break;
565 }
566
cec6b55d 567 io_u->xfer_buflen = io_u->resid;
36167d82 568 io_u->xfer_buf += bytes;
5a7c5680
JA
569 io_u->offset += bytes;
570
30061b97
JA
571 td->ts.short_io_u[io_u->ddir]++;
572
d460eb31 573 if (io_u->offset == f->real_file_size)
5a7c5680
JA
574 goto sync_done;
575
11786802
JA
576 requeue_io_u(td, &io_u);
577 } else {
5a7c5680 578sync_done:
993bf48b
JA
579 if (should_check_rate(td))
580 fio_gettime(&comp_time, NULL);
581
11786802
JA
582 bytes_done = io_u_sync_complete(td, io_u);
583 if (bytes_done < 0)
584 ret = bytes_done;
cec6b55d 585 }
36167d82
JA
586 break;
587 case FIO_Q_QUEUED:
7e77dd02
JA
588 /*
589 * if the engine doesn't have a commit hook,
590 * the io_u is really queued. if it does have such
591 * a hook, it has to call io_u_queued() itself.
592 */
593 if (td->io_ops->commit == NULL)
594 io_u_queued(td, io_u);
36167d82 595 break;
755200a3
JA
596 case FIO_Q_BUSY:
597 requeue_io_u(td, &io_u);
5451792e
JA
598 ret2 = td_io_commit(td);
599 if (ret2 < 0)
600 ret = ret2;
755200a3 601 break;
36167d82
JA
602 default:
603 assert(ret < 0);
604 put_io_u(td, io_u);
605 break;
ebac4655
JA
606 }
607
99784632 608 if (ret < 0 || td->error)
36167d82
JA
609 break;
610
97601024
JA
611 /*
612 * See if we need to complete some commands
613 */
4950421a
JA
614 full = queue_full(td) || ret == FIO_Q_BUSY;
615 if (full || !td->o.iodepth_batch_complete) {
616 min_evts = td->o.iodepth_batch_complete;
617 if (full && !min_evts)
36167d82 618 min_evts = 1;
4950421a 619
993bf48b
JA
620 if (should_check_rate(td))
621 fio_gettime(&comp_time, NULL);
4950421a
JA
622
623 do {
624 ret = io_u_queued_complete(td, min_evts);
625 if (ret <= 0)
626 break;
627
628 bytes_done += ret;
629 } while (full && (td->cur_depth > td->o.iodepth_low));
ebac4655
JA
630 }
631
4950421a
JA
632 if (ret < 0)
633 break;
97601024
JA
634 if (!bytes_done)
635 continue;
636
ebac4655
JA
637 /*
638 * the rate is batched for now, it should work for batches
639 * of completions except the very first one which may look
640 * a little bursty
641 */
993bf48b
JA
642 if (!in_ramp_time(td) && should_check_rate(td)) {
643 usec = utime_since(&td->tv_cache, &comp_time);
ebac4655 644
721938ae 645 rate_throttle(td, usec, bytes_done);
ebac4655 646
721938ae
JA
647 if (check_min_rate(td, &comp_time)) {
648 if (exitall_on_terminate)
649 terminate_threads(td->groupid);
650 td_verror(td, EIO, "check_min_rate");
651 break;
652 }
ebac4655
JA
653 }
654
2dc1bbeb 655 if (td->o.thinktime) {
9c1f7434
JA
656 unsigned long long b;
657
658 b = td->io_blocks[0] + td->io_blocks[1];
2dc1bbeb 659 if (!(b % td->o.thinktime_blocks)) {
48097d5c
JA
660 int left;
661
2dc1bbeb
JA
662 if (td->o.thinktime_spin)
663 __usec_sleep(td->o.thinktime_spin);
48097d5c 664
2dc1bbeb 665 left = td->o.thinktime - td->o.thinktime_spin;
48097d5c
JA
666 if (left)
667 usec_sleep(td, left);
668 }
9c1f7434 669 }
ebac4655
JA
670 }
671
aa31f1f1
SL
672 if (td->o.fill_device && td->error == ENOSPC) {
673 td->error = 0;
674 td->terminate = 1;
675 }
4d2413c6 676 if (!td->error) {
3d7c391d
JA
677 struct fio_file *f;
678
c01c0395
JA
679 i = td->cur_depth;
680 if (i)
681 ret = io_u_queued_complete(td, i);
ebac4655 682
2dc1bbeb 683 if (should_fsync(td) && td->o.end_fsync) {
84585003 684 td_set_runstate(td, TD_FSYNCING);
3d7b485f
JA
685
686 for_each_file(td, f, i) {
687 if (!(f->flags & FIO_FILE_OPEN))
688 continue;
858a3d47 689 fio_io_sync(td, f);
3d7b485f 690 }
84585003 691 }
c01c0395
JA
692 } else
693 cleanup_pending_aio(td);
2ba1c290
JA
694
695 /*
696 * stop job if we failed doing any IO
697 */
698 if ((td->this_io_bytes[0] + td->this_io_bytes[1]) == 0)
699 td->done = 1;
ebac4655
JA
700}
701
ebac4655
JA
702static void cleanup_io_u(struct thread_data *td)
703{
01743ee1 704 struct flist_head *entry, *n;
ebac4655
JA
705 struct io_u *io_u;
706
01743ee1
JA
707 flist_for_each_safe(entry, n, &td->io_u_freelist) {
708 io_u = flist_entry(entry, struct io_u, list);
ebac4655 709
01743ee1 710 flist_del(&io_u->list);
ebac4655
JA
711 free(io_u);
712 }
713
2f9ade3c 714 free_io_mem(td);
ebac4655
JA
715}
716
717static int init_io_u(struct thread_data *td)
718{
719 struct io_u *io_u;
a00735e6 720 unsigned int max_bs;
ebac4655
JA
721 int i, max_units;
722 char *p;
723
1e3c09af 724 max_units = td->o.iodepth;
2dc1bbeb 725 max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
5ec10eaa
JA
726 td->orig_buffer_size = (unsigned long long) max_bs
727 * (unsigned long long) max_units;
728
729 if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) {
730 unsigned long bs;
74b025b0 731
5ec10eaa
JA
732 bs = td->orig_buffer_size + td->o.hugepage_size - 1;
733 td->orig_buffer_size = bs & ~(td->o.hugepage_size - 1);
734 }
ebac4655 735
c3990645
JA
736 if (td->orig_buffer_size != (size_t) td->orig_buffer_size) {
737 log_err("fio: IO memory too large. Reduce max_bs or iodepth\n");
738 return 1;
739 }
740
2f9ade3c
JA
741 if (allocate_io_mem(td))
742 return 1;
ebac4655 743
b3f378e9
JA
744 if (td->o.odirect)
745 p = ALIGN(td->orig_buffer);
746 else
747 p = td->orig_buffer;
748
ebac4655 749 for (i = 0; i < max_units; i++) {
db1cd90b
JA
750 if (td->terminate)
751 return 1;
ebac4655
JA
752 io_u = malloc(sizeof(*io_u));
753 memset(io_u, 0, sizeof(*io_u));
01743ee1 754 INIT_FLIST_HEAD(&io_u->list);
ebac4655 755
b4c5e1ac
JA
756 if (!(td->io_ops->flags & FIO_NOIO)) {
757 io_u->buf = p + max_bs * i;
e9459e5a 758
5973cafb
JA
759 if (td_write(td) && !td->o.refill_buffers)
760 io_u_fill_buffer(td, io_u, max_bs);
b4c5e1ac 761 }
6b9cea23 762
b1ff3403 763 io_u->index = i;
0c6e7517 764 io_u->flags = IO_U_F_FREE;
01743ee1 765 flist_add(&io_u->list, &td->io_u_freelist);
ebac4655
JA
766 }
767
768 return 0;
769}
770
da86774e
JA
771static int switch_ioscheduler(struct thread_data *td)
772{
773 char tmp[256], tmp2[128];
774 FILE *f;
775 int ret;
776
ba0fbe10 777 if (td->io_ops->flags & FIO_DISKLESSIO)
f48b467c
JA
778 return 0;
779
da86774e
JA
780 sprintf(tmp, "%s/queue/scheduler", td->sysfs_root);
781
782 f = fopen(tmp, "r+");
783 if (!f) {
cbf5c1d7 784 if (errno == ENOENT) {
5ec10eaa
JA
785 log_err("fio: os or kernel doesn't support IO scheduler"
786 " switching\n");
cbf5c1d7
JA
787 return 0;
788 }
789 td_verror(td, errno, "fopen iosched");
da86774e
JA
790 return 1;
791 }
792
793 /*
794 * Set io scheduler.
795 */
2dc1bbeb 796 ret = fwrite(td->o.ioscheduler, strlen(td->o.ioscheduler), 1, f);
da86774e 797 if (ferror(f) || ret != 1) {
e1161c32 798 td_verror(td, errno, "fwrite");
da86774e
JA
799 fclose(f);
800 return 1;
801 }
802
803 rewind(f);
804
805 /*
806 * Read back and check that the selected scheduler is now the default.
807 */
808 ret = fread(tmp, 1, sizeof(tmp), f);
809 if (ferror(f) || ret < 0) {
e1161c32 810 td_verror(td, errno, "fread");
da86774e
JA
811 fclose(f);
812 return 1;
813 }
814
2dc1bbeb 815 sprintf(tmp2, "[%s]", td->o.ioscheduler);
da86774e 816 if (!strstr(tmp, tmp2)) {
2dc1bbeb 817 log_err("fio: io scheduler %s not found\n", td->o.ioscheduler);
e1161c32 818 td_verror(td, EINVAL, "iosched_switch");
da86774e
JA
819 fclose(f);
820 return 1;
821 }
822
823 fclose(f);
824 return 0;
825}
826
492158cf
JA
827static int keep_running(struct thread_data *td)
828{
829 unsigned long long io_done;
830
20e354ef
JA
831 if (td->done)
832 return 0;
492158cf
JA
833 if (td->o.time_based)
834 return 1;
835 if (td->o.loops) {
836 td->o.loops--;
837 return 1;
838 }
839
5ec10eaa
JA
840 io_done = td->io_bytes[DDIR_READ] + td->io_bytes[DDIR_WRITE]
841 + td->io_skip_bytes;
492158cf
JA
842 if (io_done < td->o.size)
843 return 1;
844
845 return 0;
846}
847
b29ee5b3 848static void reset_io_counters(struct thread_data *td)
ebac4655 849{
756867bd 850 td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0;
ebac4655 851 td->this_io_bytes[0] = td->this_io_bytes[1] = 0;
20dc95c4 852 td->zone_bytes = 0;
d7d3b49b 853 td->rate_bytes = 0;
4e991c23 854 td->rate_blocks = 0;
38d77cae 855 td->rw_end_set[0] = td->rw_end_set[1] = 0;
ebac4655 856
c1324df1
JA
857 td->last_was_sync = 0;
858
2ba1c290
JA
859 /*
860 * reset file done count if we are to start over
861 */
862 if (td->o.time_based || td->o.loops)
48f5abd3 863 td->nr_done_files = 0;
b29ee5b3
JA
864}
865
866void reset_all_stats(struct thread_data *td)
867{
868 struct timeval tv;
869 int i;
870
871 reset_io_counters(td);
872
873 for (i = 0; i < 2; i++) {
874 td->io_bytes[i] = 0;
875 td->io_blocks[i] = 0;
876 td->io_issues[i] = 0;
877 td->ts.total_io_u[i] = 0;
878 }
879
880 fio_gettime(&tv, NULL);
881 memcpy(&td->epoch, &tv, sizeof(tv));
882 memcpy(&td->start, &tv, sizeof(tv));
883}
884
885static int clear_io_state(struct thread_data *td)
886{
887 struct fio_file *f;
888 unsigned int i;
889 int ret;
890
891 reset_io_counters(td);
281f9b63 892
24ffd2c2 893 close_files(td);
53cdc686 894
a978ba68
JA
895 ret = 0;
896 for_each_file(td, f, i) {
281f9b63 897 f->flags &= ~FIO_FILE_DONE;
a978ba68
JA
898 ret = td_io_open_file(td, f);
899 if (ret)
900 break;
53cdc686 901 }
a978ba68
JA
902
903 return ret;
ebac4655
JA
904}
905
906c8d75
JA
906/*
907 * Entry point for the thread based jobs. The process based jobs end up
908 * here as well, after a little setup.
909 */
ebac4655
JA
910static void *thread_main(void *data)
911{
10927316 912 unsigned long long runtime[2], elapsed;
ebac4655 913 struct thread_data *td = data;
a978ba68 914 int clear_state;
ebac4655 915
2dc1bbeb 916 if (!td->o.use_thread)
ebac4655
JA
917 setsid();
918
919 td->pid = getpid();
920
5921e80c 921 dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid);
ee56ad50 922
01743ee1
JA
923 INIT_FLIST_HEAD(&td->io_u_freelist);
924 INIT_FLIST_HEAD(&td->io_u_busylist);
925 INIT_FLIST_HEAD(&td->io_u_requeues);
926 INIT_FLIST_HEAD(&td->io_log_list);
927 INIT_FLIST_HEAD(&td->io_hist_list);
bb5d7d0b 928 td->io_hist_tree = RB_ROOT;
aea47d44 929
db1cd90b 930 td_set_runstate(td, TD_INITIALIZED);
cdd18ad8
JA
931 fio_mutex_up(startup_mutex);
932 fio_mutex_down(td->mutex);
db1cd90b
JA
933
934 /*
cdd18ad8 935 * the ->mutex mutex is now no longer used, close it to avoid
db1cd90b
JA
936 * eating a file descriptor
937 */
cdd18ad8 938 fio_mutex_remove(td->mutex);
db1cd90b 939
d84f8d49
JA
940 /*
941 * May alter parameters that init_io_u() will use, so we need to
942 * do this first.
943 */
944 if (init_iolog(td))
945 goto err;
946
ebac4655 947 if (init_io_u(td))
db1cd90b 948 goto err;
ebac4655 949
375b2695 950 if (td->o.cpumask_set && fio_setaffinity(td) == -1) {
e1161c32 951 td_verror(td, errno, "cpu_set_affinity");
db1cd90b 952 goto err;
ebac4655
JA
953 }
954
ac684785 955 if (td->ioprio_set) {
ebac4655 956 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
e1161c32 957 td_verror(td, errno, "ioprio_set");
db1cd90b 958 goto err;
ebac4655
JA
959 }
960 }
961
2dc1bbeb 962 if (nice(td->o.nice) == -1) {
e1161c32 963 td_verror(td, errno, "nice");
db1cd90b 964 goto err;
b6f4d880
JA
965 }
966
2dc1bbeb 967 if (td->o.ioscheduler && switch_ioscheduler(td))
db1cd90b 968 goto err;
37f56873 969
2dc1bbeb 970 if (!td->o.create_serialize && setup_files(td))
ebac4655
JA
971 goto err;
972
7d6c5283
JA
973 if (td_io_init(td))
974 goto err;
975
b5af8293
JA
976 if (open_files(td))
977 goto err;
978
68727076
JA
979 if (init_random_map(td))
980 goto err;
981
2dc1bbeb
JA
982 if (td->o.exec_prerun) {
983 if (system(td->o.exec_prerun) < 0)
69cfd7e0
JA
984 goto err;
985 }
4e0ba8af 986
69008999 987 fio_gettime(&td->epoch, NULL);
756867bd 988 getrusage(RUSAGE_SELF, &td->ts.ru_start);
69008999
JA
989
990 runtime[0] = runtime[1] = 0;
a978ba68 991 clear_state = 0;
492158cf 992 while (keep_running(td)) {
02bcaa8c 993 fio_gettime(&td->start, NULL);
756867bd 994 memcpy(&td->ts.stat_sample_time, &td->start, sizeof(td->start));
993bf48b 995 memcpy(&td->tv_cache, &td->start, sizeof(td->start));
ebac4655 996
2dc1bbeb 997 if (td->o.ratemin)
5ec10eaa
JA
998 memcpy(&td->lastrate, &td->ts.stat_sample_time,
999 sizeof(td->lastrate));
ebac4655 1000
a978ba68
JA
1001 if (clear_state && clear_io_state(td))
1002 break;
1003
ebac4655
JA
1004 prune_io_piece_log(td);
1005
ba0fbe10 1006 do_io(td);
ebac4655 1007
a978ba68
JA
1008 clear_state = 1;
1009
38d77cae
JA
1010 if (td_read(td) && td->io_bytes[DDIR_READ]) {
1011 if (td->rw_end_set[DDIR_READ])
5ec10eaa
JA
1012 elapsed = utime_since(&td->start,
1013 &td->rw_end[DDIR_READ]);
38d77cae
JA
1014 else
1015 elapsed = utime_since_now(&td->start);
1016
1017 runtime[DDIR_READ] += elapsed;
1018 }
1019 if (td_write(td) && td->io_bytes[DDIR_WRITE]) {
1020 if (td->rw_end_set[DDIR_WRITE])
5ec10eaa
JA
1021 elapsed = utime_since(&td->start,
1022 &td->rw_end[DDIR_WRITE]);
38d77cae
JA
1023 else
1024 elapsed = utime_since_now(&td->start);
1025
1026 runtime[DDIR_WRITE] += elapsed;
1027 }
5ec10eaa 1028
ebac4655
JA
1029 if (td->error || td->terminate)
1030 break;
1031
e84c73a8
SL
1032 if (!td->o.do_verify ||
1033 td->o.verify == VERIFY_NONE ||
b67740d3 1034 (td->io_ops->flags & FIO_UNIDIR))
ebac4655
JA
1035 continue;
1036
a978ba68
JA
1037 if (clear_io_state(td))
1038 break;
1039
02bcaa8c 1040 fio_gettime(&td->start, NULL);
ebac4655
JA
1041
1042 do_verify(td);
1043
69008999 1044 runtime[DDIR_READ] += utime_since_now(&td->start);
ebac4655
JA
1045
1046 if (td->error || td->terminate)
1047 break;
1048 }
1049
36dff966 1050 update_rusage_stat(td);
47f0cc48
YHJT
1051 td->ts.runtime[0] = (runtime[0] + 999) / 1000;
1052 td->ts.runtime[1] = (runtime[1] + 999) / 1000;
756867bd
JA
1053 td->ts.total_run_time = mtime_since_now(&td->epoch);
1054 td->ts.io_bytes[0] = td->io_bytes[0];
1055 td->ts.io_bytes[1] = td->io_bytes[1];
1056
1057 if (td->ts.bw_log)
1058 finish_log(td, td->ts.bw_log, "bw");
1059 if (td->ts.slat_log)
1060 finish_log(td, td->ts.slat_log, "slat");
1061 if (td->ts.clat_log)
1062 finish_log(td, td->ts.clat_log, "clat");
2dc1bbeb
JA
1063 if (td->o.exec_postrun) {
1064 if (system(td->o.exec_postrun) < 0)
1065 log_err("fio: postrun %s failed\n", td->o.exec_postrun);
69cfd7e0 1066 }
ebac4655
JA
1067
1068 if (exitall_on_terminate)
390c40e2 1069 terminate_threads(td->groupid);
ebac4655
JA
1070
1071err:
5bf13a5a 1072 if (td->error)
5921e80c 1073 printf("fio: pid=%d, err=%d/%s\n", (int) td->pid, td->error,
5ec10eaa 1074 td->verror);
24ffd2c2 1075 close_and_free_files(td);
2866c82d 1076 close_ioengine(td);
ebac4655 1077 cleanup_io_u(td);
f29b25a3
JA
1078
1079 /*
1080 * do this very late, it will log file closing as well
1081 */
1082 if (td->o.write_iolog_file)
1083 write_iolog_close(td);
1084
d23bb327 1085 options_mem_free(td);
ebac4655 1086 td_set_runstate(td, TD_EXITED);
43d76807 1087 return (void *) (unsigned long) td->error;
ebac4655
JA
1088}
1089
906c8d75
JA
1090/*
1091 * We cannot pass the td data into a forked process, so attach the td and
1092 * pass it to the thread worker.
1093 */
a6418147 1094static int fork_main(int shmid, int offset)
ebac4655
JA
1095{
1096 struct thread_data *td;
a6418147 1097 void *data, *ret;
ebac4655
JA
1098
1099 data = shmat(shmid, NULL, 0);
1100 if (data == (void *) -1) {
a6418147
JA
1101 int __err = errno;
1102
ebac4655 1103 perror("shmat");
a6418147 1104 return __err;
ebac4655
JA
1105 }
1106
1107 td = data + offset * sizeof(struct thread_data);
a6418147 1108 ret = thread_main(td);
ebac4655 1109 shmdt(data);
43d76807 1110 return (int) (unsigned long) ret;
ebac4655
JA
1111}
1112
906c8d75
JA
1113/*
1114 * Run over the job map and reap the threads that have exited, if any.
1115 */
ebac4655
JA
1116static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
1117{
34572e28 1118 struct thread_data *td;
1f809d15 1119 int i, cputhreads, realthreads, pending, status, ret;
ebac4655
JA
1120
1121 /*
1122 * reap exited threads (TD_EXITED -> TD_REAPED)
1123 */
1f809d15 1124 realthreads = pending = cputhreads = 0;
34572e28 1125 for_each_td(td, i) {
3707f45b 1126 int flags = 0;
a2f77c9f 1127
84585003
JA
1128 /*
1129 * ->io_ops is NULL for a thread that has closed its
1130 * io engine
1131 */
ba0fbe10 1132 if (td->io_ops && !strcmp(td->io_ops->name, "cpuio"))
b990b5c0 1133 cputhreads++;
1f809d15
JA
1134 else
1135 realthreads++;
b990b5c0 1136
a1dedc1f
AC
1137 if (!td->pid) {
1138 pending++;
1139 continue;
1140 }
1141 if (td->runstate == TD_REAPED)
a2f77c9f 1142 continue;
2dc1bbeb 1143 if (td->o.use_thread) {
3707f45b
JA
1144 if (td->runstate == TD_EXITED) {
1145 td_set_runstate(td, TD_REAPED);
1146 goto reaped;
1147 }
1148 continue;
1149 }
a2f77c9f
JA
1150
1151 flags = WNOHANG;
1152 if (td->runstate == TD_EXITED)
1153 flags = 0;
1154
1155 /*
1156 * check if someone quit or got killed in an unusual way
1157 */
1158 ret = waitpid(td->pid, &status, flags);
3707f45b 1159 if (ret < 0) {
a2f77c9f 1160 if (errno == ECHILD) {
5921e80c
JA
1161 log_err("fio: pid=%d disappeared %d\n",
1162 (int) td->pid, td->runstate);
a2f77c9f
JA
1163 td_set_runstate(td, TD_REAPED);
1164 goto reaped;
1165 }
1166 perror("waitpid");
1167 } else if (ret == td->pid) {
1168 if (WIFSIGNALED(status)) {
fab6aa71
JA
1169 int sig = WTERMSIG(status);
1170
d9244941 1171 if (sig != SIGQUIT)
5ec10eaa 1172 log_err("fio: pid=%d, got signal=%d\n",
5921e80c 1173 (int) td->pid, sig);
fab6aa71
JA
1174 td_set_runstate(td, TD_REAPED);
1175 goto reaped;
1176 }
a2f77c9f
JA
1177 if (WIFEXITED(status)) {
1178 if (WEXITSTATUS(status) && !td->error)
1179 td->error = WEXITSTATUS(status);
a2f77c9f 1180
a2f77c9f
JA
1181 td_set_runstate(td, TD_REAPED);
1182 goto reaped;
a6418147
JA
1183 }
1184 }
ebac4655 1185
a2f77c9f
JA
1186 /*
1187 * thread is not dead, continue
1188 */
e48676ba 1189 pending++;
a2f77c9f 1190 continue;
fab6aa71 1191reaped:
ebac4655 1192 (*nr_running)--;
2dc1bbeb
JA
1193 (*m_rate) -= td->o.ratemin;
1194 (*t_rate) -= td->o.rate;
6f88bcb0
JA
1195 if (!td->pid)
1196 pending--;
a2f77c9f
JA
1197
1198 if (td->error)
1199 exit_value++;
d3eeeabc
JA
1200
1201 done_secs += mtime_since_now(&td->epoch) / 1000;
ebac4655 1202 }
b990b5c0 1203
1f809d15 1204 if (*nr_running == cputhreads && !pending && realthreads)
390c40e2 1205 terminate_threads(TERMINATE_ALL);
ebac4655
JA
1206}
1207
906c8d75
JA
1208/*
1209 * Main function for kicking off and reaping jobs, as needed.
1210 */
ebac4655
JA
1211static void run_threads(void)
1212{
ebac4655
JA
1213 struct thread_data *td;
1214 unsigned long spent;
1215 int i, todo, nr_running, m_rate, t_rate, nr_started;
fcb6ade2 1216
2f9ade3c
JA
1217 if (fio_pin_memory())
1218 return;
ebac4655 1219
c6ae0a5b 1220 if (!terse_output) {
9cedf167
JA
1221 printf("Starting ");
1222 if (nr_thread)
5ec10eaa
JA
1223 printf("%d thread%s", nr_thread,
1224 nr_thread > 1 ? "s" : "");
9cedf167
JA
1225 if (nr_process) {
1226 if (nr_thread)
1227 printf(" and ");
5ec10eaa
JA
1228 printf("%d process%s", nr_process,
1229 nr_process > 1 ? "es" : "");
9cedf167
JA
1230 }
1231 printf("\n");
c6ae0a5b
JA
1232 fflush(stdout);
1233 }
c04f7ec3 1234
697a606c 1235 set_sig_handlers();
4efa970e 1236
ebac4655
JA
1237 todo = thread_number;
1238 nr_running = 0;
1239 nr_started = 0;
1240 m_rate = t_rate = 0;
1241
34572e28 1242 for_each_td(td, i) {
263e529f 1243 print_status_init(td->thread_number - 1);
ebac4655 1244
2dc1bbeb 1245 if (!td->o.create_serialize) {
380cf265 1246 init_disk_util(td);
ebac4655 1247 continue;
380cf265 1248 }
ebac4655
JA
1249
1250 /*
1251 * do file setup here so it happens sequentially,
1252 * we don't want X number of threads getting their
1253 * client data interspersed on disk
1254 */
53cdc686 1255 if (setup_files(td)) {
5bf13a5a
JA
1256 exit_value++;
1257 if (td->error)
5921e80c
JA
1258 log_err("fio: pid=%d, err=%d/%s\n",
1259 (int) td->pid, td->error, td->verror);
ebac4655
JA
1260 td_set_runstate(td, TD_REAPED);
1261 todo--;
c69e8875
JA
1262 } else {
1263 struct fio_file *f;
1264 unsigned int i;
1265
1266 /*
1267 * for sharing to work, each job must always open
1268 * its own files. so close them, if we opened them
1269 * for creation
1270 */
1271 for_each_file(td, f, i)
1272 td_io_close_file(td, f);
5ec10eaa 1273 }
380cf265
JA
1274
1275 init_disk_util(td);
ebac4655
JA
1276 }
1277
a2f77c9f
JA
1278 set_genesis_time();
1279
ebac4655 1280 while (todo) {
75154845
JA
1281 struct thread_data *map[MAX_JOBS];
1282 struct timeval this_start;
1283 int this_jobs = 0, left;
1284
ebac4655
JA
1285 /*
1286 * create threads (TD_NOT_CREATED -> TD_CREATED)
1287 */
34572e28 1288 for_each_td(td, i) {
ebac4655
JA
1289 if (td->runstate != TD_NOT_CREATED)
1290 continue;
1291
1292 /*
1293 * never got a chance to start, killed by other
1294 * thread for some reason
1295 */
1296 if (td->terminate) {
1297 todo--;
1298 continue;
1299 }
1300
2dc1bbeb 1301 if (td->o.start_delay) {
263e529f 1302 spent = mtime_since_genesis();
ebac4655 1303
2dc1bbeb 1304 if (td->o.start_delay * 1000 > spent)
ebac4655
JA
1305 continue;
1306 }
1307
6f88bcb0
JA
1308 if (td->o.stonewall && (nr_started || nr_running)) {
1309 dprint(FD_PROCESS, "%s: stonewall wait\n",
1310 td->o.name);
ebac4655 1311 break;
6f88bcb0 1312 }
ebac4655 1313
75154845
JA
1314 /*
1315 * Set state to created. Thread will transition
1316 * to TD_INITIALIZED when it's done setting up.
1317 */
ebac4655 1318 td_set_runstate(td, TD_CREATED);
75154845 1319 map[this_jobs++] = td;
ebac4655
JA
1320 nr_started++;
1321
2dc1bbeb 1322 if (td->o.use_thread) {
ee56ad50 1323 dprint(FD_PROCESS, "will pthread_create\n");
5ec10eaa
JA
1324 if (pthread_create(&td->thread, NULL,
1325 thread_main, td)) {
f4705a73 1326 perror("pthread_create");
ebac4655 1327 nr_started--;
c8bb6faf 1328 break;
ebac4655 1329 }
f4705a73
JA
1330 if (pthread_detach(td->thread) < 0)
1331 perror("pthread_detach");
ebac4655 1332 } else {
5e1d306e 1333 pid_t pid;
ee56ad50 1334 dprint(FD_PROCESS, "will fork\n");
5e1d306e
JA
1335 pid = fork();
1336 if (!pid) {
a6418147
JA
1337 int ret = fork_main(shm_id, i);
1338
5e1d306e
JA
1339 _exit(ret);
1340 } else if (i == fio_debug_jobno)
1341 *fio_debug_jobp = pid;
ebac4655 1342 }
cdd18ad8 1343 fio_mutex_down(startup_mutex);
ebac4655
JA
1344 }
1345
1346 /*
75154845
JA
1347 * Wait for the started threads to transition to
1348 * TD_INITIALIZED.
ebac4655 1349 */
02bcaa8c 1350 fio_gettime(&this_start, NULL);
75154845 1351 left = this_jobs;
6ce15a32 1352 while (left && !fio_abort) {
75154845
JA
1353 if (mtime_since_now(&this_start) > JOB_START_TIMEOUT)
1354 break;
1355
1356 usleep(100000);
1357
1358 for (i = 0; i < this_jobs; i++) {
1359 td = map[i];
1360 if (!td)
1361 continue;
b6f4d880 1362 if (td->runstate == TD_INITIALIZED) {
75154845
JA
1363 map[i] = NULL;
1364 left--;
b6f4d880
JA
1365 } else if (td->runstate >= TD_EXITED) {
1366 map[i] = NULL;
1367 left--;
1368 todo--;
1369 nr_running++; /* work-around... */
75154845
JA
1370 }
1371 }
1372 }
1373
1374 if (left) {
3b70d7e5 1375 log_err("fio: %d jobs failed to start\n", left);
75154845
JA
1376 for (i = 0; i < this_jobs; i++) {
1377 td = map[i];
1378 if (!td)
1379 continue;
1380 kill(td->pid, SIGTERM);
1381 }
1382 break;
1383 }
1384
1385 /*
b6f4d880 1386 * start created threads (TD_INITIALIZED -> TD_RUNNING).
75154845 1387 */
34572e28 1388 for_each_td(td, i) {
75154845 1389 if (td->runstate != TD_INITIALIZED)
ebac4655
JA
1390 continue;
1391
b29ee5b3
JA
1392 if (in_ramp_time(td))
1393 td_set_runstate(td, TD_RAMP);
1394 else
1395 td_set_runstate(td, TD_RUNNING);
ebac4655
JA
1396 nr_running++;
1397 nr_started--;
2dc1bbeb
JA
1398 m_rate += td->o.ratemin;
1399 t_rate += td->o.rate;
75154845 1400 todo--;
cdd18ad8 1401 fio_mutex_up(td->mutex);
ebac4655
JA
1402 }
1403
1404 reap_threads(&nr_running, &t_rate, &m_rate);
1405
1406 if (todo)
1407 usleep(100000);
1408 }
1409
1410 while (nr_running) {
1411 reap_threads(&nr_running, &t_rate, &m_rate);
1412 usleep(10000);
1413 }
1414
1415 update_io_ticks();
2f9ade3c 1416 fio_unpin_memory();
ebac4655
JA
1417}
1418
ebac4655
JA
1419int main(int argc, char *argv[])
1420{
29d610e1
JA
1421 long ps;
1422
2e5cdb11
JA
1423 sinit();
1424
dbe1125e
JA
1425 /*
1426 * We need locale for number printing, if it isn't set then just
1427 * go with the US format.
1428 */
1429 if (!getenv("LC_NUMERIC"))
1430 setlocale(LC_NUMERIC, "en_US");
1431
ebac4655
JA
1432 if (parse_options(argc, argv))
1433 return 1;
1434
4b472fa3
JA
1435 if (!thread_number)
1436 return 0;
ebac4655 1437
29d610e1
JA
1438 ps = sysconf(_SC_PAGESIZE);
1439 if (ps < 0) {
1440 log_err("Failed to get page size\n");
1441 return 1;
1442 }
1443
cfc99db7 1444 page_size = ps;
29d610e1
JA
1445 page_mask = ps - 1;
1446
bb3884d8
JA
1447 if (write_bw_log) {
1448 setup_log(&agg_io_log[DDIR_READ]);
1449 setup_log(&agg_io_log[DDIR_WRITE]);
1450 }
1451
cdd18ad8 1452 startup_mutex = fio_mutex_init(0);
07739b57 1453
a2f77c9f
JA
1454 set_genesis_time();
1455
ccb0fa24 1456 status_timer_arm();
ebac4655
JA
1457
1458 run_threads();
6ce15a32 1459
bb3884d8 1460 if (!fio_abort) {
6ce15a32 1461 show_run_stats();
bb3884d8 1462 if (write_bw_log) {
5ec10eaa
JA
1463 __finish_log(agg_io_log[DDIR_READ], "agg-read_bw.log");
1464 __finish_log(agg_io_log[DDIR_WRITE],
1465 "agg-write_bw.log");
bb3884d8
JA
1466 }
1467 }
ebac4655 1468
cdd18ad8 1469 fio_mutex_remove(startup_mutex);
437c9b71 1470 return exit_value;
ebac4655 1471}