server: add ETA as a specific command
[fio.git] / fio.c
CommitLineData
ebac4655
JA
1/*
2 * fio - the flexible io tester
3 *
4 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
aae22ca7 5 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
ebac4655 6 *
8e9fe637
JA
7 * The license below covers all files distributed with fio unless otherwise
8 * noted in the file itself.
9 *
ebac4655 10 * This program is free software; you can redistribute it and/or modify
8e9fe637
JA
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
ebac4655
JA
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 *
23 */
ebac4655
JA
24#include <unistd.h>
25#include <fcntl.h>
26#include <string.h>
9b836561 27#include <limits.h>
ebac4655
JA
28#include <signal.h>
29#include <time.h>
dbe1125e 30#include <locale.h>
36167d82 31#include <assert.h>
a5b01f1b 32#include <time.h>
ebac4655
JA
33#include <sys/stat.h>
34#include <sys/wait.h>
35#include <sys/ipc.h>
36#include <sys/shm.h>
ebac4655
JA
37#include <sys/mman.h>
38
39#include "fio.h"
210f43b3 40#include "hash.h"
2e5cdb11 41#include "smalloc.h"
4f5af7b2 42#include "verify.h"
a917a8b3 43#include "trim.h"
7c9b1bce 44#include "diskutil.h"
a696fa2a 45#include "cgroup.h"
07b3232d 46#include "profile.h"
d54a144d 47#include "lib/rand.h"
91aea6ec 48#include "memalign.h"
009b1be4 49#include "server.h"
ebac4655 50
cfc99db7
JA
51unsigned long page_mask;
52unsigned long page_size;
d529ee19
JA
53
54#define PAGE_ALIGN(buf) \
29d610e1 55 (char *) (((unsigned long) (buf) + page_mask) & ~page_mask)
ebac4655
JA
56
57int groupid = 0;
58int thread_number = 0;
9cedf167
JA
59int nr_process = 0;
60int nr_thread = 0;
ebac4655 61int shm_id = 0;
53cdc686 62int temp_stall_ts;
d3eeeabc 63unsigned long done_secs = 0;
ebac4655 64
cdd18ad8 65static struct fio_mutex *startup_mutex;
d2bbb8d9 66static struct fio_mutex *writeout_mutex;
6ce15a32 67static volatile int fio_abort;
437c9b71 68static int exit_value;
be4ecfdf 69static pthread_t gtod_thread;
7985701b 70static pthread_t disk_util_thread;
dae5341c 71static struct flist_head *cgroup_list;
6adb38a1 72static char *cgroup_mnt;
ebac4655 73
d3cc4ebf
JA
74unsigned long arch_flags = 0;
75
bb3884d8
JA
76struct io_log *agg_io_log[2];
77
ebac4655 78#define TERMINATE_ALL (-1)
75154845 79#define JOB_START_TIMEOUT (5 * 1000)
ebac4655 80
b29ee5b3 81void td_set_runstate(struct thread_data *td, int runstate)
6ce15a32 82{
d8fab1b6
JA
83 if (td->runstate == runstate)
84 return;
85
5921e80c
JA
86 dprint(FD_PROCESS, "pid=%d: runstate %d -> %d\n", (int) td->pid,
87 td->runstate, runstate);
6ce15a32
JA
88 td->runstate = runstate;
89}
90
390c40e2 91static void terminate_threads(int group_id)
ebac4655 92{
34572e28 93 struct thread_data *td;
ebac4655
JA
94 int i;
95
6f88bcb0
JA
96 dprint(FD_PROCESS, "terminate group_id=%d\n", group_id);
97
34572e28 98 for_each_td(td, i) {
ebac4655 99 if (group_id == TERMINATE_ALL || groupid == td->groupid) {
6f88bcb0 100 dprint(FD_PROCESS, "setting terminate on %s/%d\n",
5921e80c 101 td->o.name, (int) td->pid);
ad830ed7
JA
102 td->terminate = 1;
103 td->o.start_delay = 0;
104
7a93ab19
JA
105 /*
106 * if the thread is running, just let it exit
107 */
6aa27816
MC
108 if (!td->pid)
109 continue;
110 else if (td->runstate < TD_RAMP)
03e20d68 111 kill(td->pid, SIGTERM);
f63f7f3b
JA
112 else {
113 struct ioengine_ops *ops = td->io_ops;
114
03e20d68
BC
115 if (ops && (ops->flags & FIO_SIGTERM))
116 kill(td->pid, SIGTERM);
f63f7f3b 117 }
ebac4655
JA
118 }
119 }
120}
121
697a606c
JA
122static void sig_int(int sig)
123{
124 if (threads) {
4ceb30d4 125 log_info("\nfio: terminating on signal %d\n", sig);
009b1be4 126 exit_backend = 1;
5ec10eaa 127 fflush(stdout);
deaa1af2 128 exit_value = 128;
5ec10eaa 129 terminate_threads(TERMINATE_ALL);
ebac4655
JA
130 }
131}
132
7985701b 133static void *disk_thread_main(void *data)
a5b01f1b 134{
7985701b
JA
135 fio_mutex_up(startup_mutex);
136
137 while (threads) {
138 usleep(DISK_UTIL_MSEC * 1000);
9a793c2f
JA
139 if (!threads)
140 break;
7985701b 141 update_io_ticks();
cf451d1e
JA
142
143 if (is_backend)
144 fio_server_send_status();
145 else
146 print_thread_status();
7985701b
JA
147 }
148
149 return NULL;
a5b01f1b
JA
150}
151
7985701b 152static int create_disk_util_thread(void)
a5b01f1b 153{
7985701b 154 int ret;
a5b01f1b 155
69204d6e 156 ret = pthread_create(&disk_util_thread, NULL, disk_thread_main, NULL);
7985701b
JA
157 if (ret) {
158 log_err("Can't create disk util thread: %s\n", strerror(ret));
159 return 1;
160 }
a5b01f1b 161
7985701b
JA
162 ret = pthread_detach(disk_util_thread);
163 if (ret) {
164 log_err("Can't detatch disk util thread: %s\n", strerror(ret));
165 return 1;
166 }
03e20d68 167
7985701b
JA
168 dprint(FD_MUTEX, "wait on startup_mutex\n");
169 fio_mutex_down(startup_mutex);
170 dprint(FD_MUTEX, "done waiting on startup_mutex\n");
171 return 0;
a5b01f1b
JA
172}
173
697a606c
JA
174static void set_sig_handlers(void)
175{
176 struct sigaction act;
177
697a606c
JA
178 memset(&act, 0, sizeof(act));
179 act.sa_handler = sig_int;
180 act.sa_flags = SA_RESTART;
181 sigaction(SIGINT, &act, NULL);
de79c915 182
5f8f9726 183 memset(&act, 0, sizeof(act));
b20b6be4 184 act.sa_handler = sig_int;
5f8f9726 185 act.sa_flags = SA_RESTART;
03e20d68 186 sigaction(SIGTERM, &act, NULL);
697a606c
JA
187}
188
906c8d75
JA
189/*
190 * Check if we are above the minimum rate given.
191 */
581e7141 192static int __check_min_rate(struct thread_data *td, struct timeval *now,
bd93e34b 193 enum fio_ddir ddir)
ebac4655 194{
0904200b 195 unsigned long long bytes = 0;
4e991c23 196 unsigned long iops = 0;
ebac4655
JA
197 unsigned long spent;
198 unsigned long rate;
581e7141
JA
199 unsigned int ratemin = 0;
200 unsigned int rate_iops = 0;
201 unsigned int rate_iops_min = 0;
ebac4655 202
ff58fced
JA
203 assert(ddir_rw(ddir));
204
d982d873
JA
205 if (!td->o.ratemin[ddir] && !td->o.rate_iops_min[ddir])
206 return 0;
207
ebac4655
JA
208 /*
209 * allow a 2 second settle period in the beginning
210 */
211 if (mtime_since(&td->start, now) < 2000)
212 return 0;
213
581e7141
JA
214 iops += td->io_blocks[ddir];
215 bytes += td->this_io_bytes[ddir];
216 ratemin += td->o.ratemin[ddir];
217 rate_iops += td->o.rate_iops[ddir];
218 rate_iops_min += td->o.rate_iops_min[ddir];
0904200b 219
ebac4655
JA
220 /*
221 * if rate blocks is set, sample is running
222 */
581e7141
JA
223 if (td->rate_bytes[ddir] || td->rate_blocks[ddir]) {
224 spent = mtime_since(&td->lastrate[ddir], now);
2dc1bbeb 225 if (spent < td->o.ratecycle)
ebac4655
JA
226 return 0;
227
581e7141 228 if (td->o.rate[ddir]) {
4e991c23
JA
229 /*
230 * check bandwidth specified rate
231 */
581e7141 232 if (bytes < td->rate_bytes[ddir]) {
5ec10eaa 233 log_err("%s: min rate %u not met\n", td->o.name,
581e7141 234 ratemin);
4e991c23
JA
235 return 1;
236 } else {
581e7141
JA
237 rate = ((bytes - td->rate_bytes[ddir]) * 1000) / spent;
238 if (rate < ratemin ||
239 bytes < td->rate_bytes[ddir]) {
5ec10eaa 240 log_err("%s: min rate %u not met, got"
b22989b9 241 " %luKB/sec\n", td->o.name,
581e7141 242 ratemin, rate);
4e991c23
JA
243 return 1;
244 }
245 }
413dd459 246 } else {
4e991c23
JA
247 /*
248 * checks iops specified rate
249 */
581e7141 250 if (iops < rate_iops) {
5ec10eaa 251 log_err("%s: min iops rate %u not met\n",
581e7141 252 td->o.name, rate_iops);
413dd459 253 return 1;
4e991c23 254 } else {
581e7141
JA
255 rate = ((iops - td->rate_blocks[ddir]) * 1000) / spent;
256 if (rate < rate_iops_min ||
257 iops < td->rate_blocks[ddir]) {
5ec10eaa
JA
258 log_err("%s: min iops rate %u not met,"
259 " got %lu\n", td->o.name,
581e7141 260 rate_iops_min, rate);
4e991c23 261 }
413dd459 262 }
ebac4655
JA
263 }
264 }
265
581e7141
JA
266 td->rate_bytes[ddir] = bytes;
267 td->rate_blocks[ddir] = iops;
268 memcpy(&td->lastrate[ddir], now, sizeof(*now));
ebac4655
JA
269 return 0;
270}
271
581e7141
JA
272static int check_min_rate(struct thread_data *td, struct timeval *now,
273 unsigned long *bytes_done)
274{
275 int ret = 0;
276
277 if (bytes_done[0])
278 ret |= __check_min_rate(td, now, 0);
279 if (bytes_done[1])
280 ret |= __check_min_rate(td, now, 1);
281
282 return ret;
283}
284
ebac4655
JA
285static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
286{
2dc1bbeb 287 if (!td->o.timeout)
ebac4655 288 return 0;
2dc1bbeb 289 if (mtime_since(&td->epoch, t) >= td->o.timeout * 1000)
ebac4655
JA
290 return 1;
291
292 return 0;
293}
294
906c8d75
JA
295/*
296 * When job exits, we can cancel the in-flight IO if we are using async
297 * io. Attempt to do so.
298 */
ebac4655
JA
299static void cleanup_pending_aio(struct thread_data *td)
300{
01743ee1 301 struct flist_head *entry, *n;
ebac4655
JA
302 struct io_u *io_u;
303 int r;
304
305 /*
306 * get immediately available events, if any
307 */
581e7141 308 r = io_u_queued_complete(td, 0, NULL);
b2fdda43
JA
309 if (r < 0)
310 return;
ebac4655
JA
311
312 /*
313 * now cancel remaining active events
314 */
2866c82d 315 if (td->io_ops->cancel) {
01743ee1
JA
316 flist_for_each_safe(entry, n, &td->io_u_busylist) {
317 io_u = flist_entry(entry, struct io_u, list);
ebac4655 318
0c6e7517
JA
319 /*
320 * if the io_u isn't in flight, then that generally
321 * means someone leaked an io_u. complain but fix
322 * it up, so we don't stall here.
323 */
324 if ((io_u->flags & IO_U_F_FLIGHT) == 0) {
325 log_err("fio: non-busy IO on busy list\n");
ebac4655 326 put_io_u(td, io_u);
0c6e7517
JA
327 } else {
328 r = td->io_ops->cancel(td, io_u);
329 if (!r)
330 put_io_u(td, io_u);
331 }
ebac4655
JA
332 }
333 }
334
97601024 335 if (td->cur_depth)
581e7141 336 r = io_u_queued_complete(td, td->cur_depth, NULL);
ebac4655
JA
337}
338
858a3d47
JA
339/*
340 * Helper to handle the final sync of a file. Works just like the normal
341 * io path, just does everything sync.
342 */
343static int fio_io_sync(struct thread_data *td, struct fio_file *f)
344{
345 struct io_u *io_u = __get_io_u(td);
858a3d47
JA
346 int ret;
347
348 if (!io_u)
349 return 1;
350
351 io_u->ddir = DDIR_SYNC;
352 io_u->file = f;
353
354 if (td_io_prep(td, io_u)) {
355 put_io_u(td, io_u);
356 return 1;
357 }
358
755200a3 359requeue:
858a3d47 360 ret = td_io_queue(td, io_u);
36167d82 361 if (ret < 0) {
e1161c32 362 td_verror(td, io_u->error, "td_io_queue");
858a3d47 363 put_io_u(td, io_u);
858a3d47 364 return 1;
36167d82 365 } else if (ret == FIO_Q_QUEUED) {
581e7141 366 if (io_u_queued_complete(td, 1, NULL) < 0)
36167d82 367 return 1;
36167d82
JA
368 } else if (ret == FIO_Q_COMPLETED) {
369 if (io_u->error) {
e1161c32 370 td_verror(td, io_u->error, "td_io_queue");
36167d82
JA
371 return 1;
372 }
858a3d47 373
581e7141 374 if (io_u_sync_complete(td, io_u, NULL) < 0)
b2fdda43 375 return 1;
755200a3
JA
376 } else if (ret == FIO_Q_BUSY) {
377 if (td_io_commit(td))
378 return 1;
379 goto requeue;
858a3d47
JA
380 }
381
382 return 0;
383}
384
687c6a23
JA
385static inline void __update_tv_cache(struct thread_data *td)
386{
387 fio_gettime(&td->tv_cache, NULL);
388}
389
993bf48b
JA
390static inline void update_tv_cache(struct thread_data *td)
391{
392 if ((++td->tv_cache_nr & td->tv_cache_mask) == td->tv_cache_mask)
687c6a23 393 __update_tv_cache(td);
993bf48b
JA
394}
395
f2bba182
RR
396static int break_on_this_error(struct thread_data *td, int *retptr)
397{
398 int ret = *retptr;
399
400 if (ret < 0 || td->error) {
401 int err;
402
1ec99eea
JA
403 if (!td->o.continue_on_error)
404 return 1;
f2bba182
RR
405
406 if (ret < 0)
407 err = -ret;
408 else
409 err = td->error;
410
f2bba182
RR
411 if (td_non_fatal_error(err)) {
412 /*
413 * Continue with the I/Os in case of
414 * a non fatal error.
415 */
8a4f8a24 416 update_error_count(td, err);
f2bba182
RR
417 td_clear_error(td);
418 *retptr = 0;
419 return 0;
8a4f8a24
RR
420 } else if (td->o.fill_device && err == ENOSPC) {
421 /*
422 * We expect to hit this error if
423 * fill_device option is set.
424 */
425 td_clear_error(td);
426 td->terminate = 1;
427 return 1;
f2bba182
RR
428 } else {
429 /*
430 * Stop the I/O in case of a fatal
431 * error.
432 */
8a4f8a24 433 update_error_count(td, err);
f2bba182
RR
434 return 1;
435 }
436 }
437
438 return 0;
439}
440
906c8d75 441/*
34403fb1 442 * The main verify engine. Runs over the writes we previously submitted,
906c8d75
JA
443 * reads the blocks back in, and checks the crc/md5 of the data.
444 */
1e97cce9 445static void do_verify(struct thread_data *td)
ebac4655 446{
53cdc686 447 struct fio_file *f;
36167d82 448 struct io_u *io_u;
af52b345
JA
449 int ret, min_events;
450 unsigned int i;
e5b401d4 451
008618a8
JA
452 dprint(FD_VERIFY, "starting loop\n");
453
e5b401d4
JA
454 /*
455 * sync io first and invalidate cache, to make sure we really
456 * read from disk.
457 */
458 for_each_file(td, f, i) {
d6aed795 459 if (!fio_file_open(f))
3d7b485f 460 continue;
b2fdda43
JA
461 if (fio_io_sync(td, f))
462 break;
463 if (file_invalidate_cache(td, f))
464 break;
e5b401d4 465 }
ebac4655 466
b2fdda43
JA
467 if (td->error)
468 return;
469
ebac4655
JA
470 td_set_runstate(td, TD_VERIFYING);
471
36167d82
JA
472 io_u = NULL;
473 while (!td->terminate) {
4950421a 474 int ret2, full;
5451792e 475
993bf48b
JA
476 update_tv_cache(td);
477
478 if (runtime_exceeded(td, &td->tv_cache)) {
687c6a23
JA
479 __update_tv_cache(td);
480 if (runtime_exceeded(td, &td->tv_cache)) {
481 td->terminate = 1;
482 break;
483 }
069c2918 484 }
02bcaa8c 485
dcb4830d
RR
486 io_u = __get_io_u(td);
487 if (!io_u)
488 break;
489
069c2918
JA
490 if (get_next_verify(td, io_u)) {
491 put_io_u(td, io_u);
ebac4655 492 break;
069c2918 493 }
ebac4655 494
069c2918
JA
495 if (td_io_prep(td, io_u)) {
496 put_io_u(td, io_u);
53cdc686 497 break;
069c2918 498 }
d7762cf8 499
e8462bd8
JA
500 if (td->o.verify_async)
501 io_u->end_io = verify_io_u_async;
502 else
503 io_u->end_io = verify_io_u;
53cdc686 504
11786802 505 ret = td_io_queue(td, io_u);
36167d82
JA
506 switch (ret) {
507 case FIO_Q_COMPLETED:
f2bba182 508 if (io_u->error) {
22819ec2 509 ret = -io_u->error;
f2bba182
RR
510 clear_io_u(td, io_u);
511 } else if (io_u->resid) {
36167d82 512 int bytes = io_u->xfer_buflen - io_u->resid;
ebac4655 513
9e9d164e
JA
514 /*
515 * zero read, fail
516 */
517 if (!bytes) {
41d9f0fd 518 td_verror(td, EIO, "full resid");
9e9d164e
JA
519 put_io_u(td, io_u);
520 break;
521 }
8400d9b2 522
36167d82
JA
523 io_u->xfer_buflen = io_u->resid;
524 io_u->xfer_buf += bytes;
8400d9b2
JA
525 io_u->offset += bytes;
526
ff58fced
JA
527 if (ddir_rw(io_u->ddir))
528 td->ts.short_io_u[io_u->ddir]++;
30061b97 529
2b13e716 530 f = io_u->file;
d460eb31 531 if (io_u->offset == f->real_file_size)
8400d9b2
JA
532 goto sync_done;
533
11786802
JA
534 requeue_io_u(td, &io_u);
535 } else {
8400d9b2 536sync_done:
581e7141 537 ret = io_u_sync_complete(td, io_u, NULL);
11786802
JA
538 if (ret < 0)
539 break;
36167d82 540 }
36167d82
JA
541 continue;
542 case FIO_Q_QUEUED:
543 break;
755200a3
JA
544 case FIO_Q_BUSY:
545 requeue_io_u(td, &io_u);
5451792e
JA
546 ret2 = td_io_commit(td);
547 if (ret2 < 0)
548 ret = ret2;
755200a3 549 break;
36167d82
JA
550 default:
551 assert(ret < 0);
e1161c32 552 td_verror(td, -ret, "td_io_queue");
ebac4655
JA
553 break;
554 }
555
f2bba182 556 if (break_on_this_error(td, &ret))
3af6ef39
JA
557 break;
558
ebac4655 559 /*
3af6ef39 560 * if we can queue more, do so. but check if there are
ccc02050
JA
561 * completed io_u's first. Note that we can get BUSY even
562 * without IO queued, if the system is resource starved.
ebac4655 563 */
ccc02050 564 full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
4950421a 565 if (full || !td->o.iodepth_batch_complete) {
6a96276c
RR
566 min_events = min(td->o.iodepth_batch_complete,
567 td->cur_depth);
1f84f1d9 568 if (full && !min_events && td->o.iodepth_batch_complete != 0)
838bc709 569 min_events = 1;
e916b390 570
4950421a
JA
571 do {
572 /*
573 * Reap required number of io units, if any,
574 * and do the verification on them through
575 * the callback handler
576 */
581e7141 577 if (io_u_queued_complete(td, min_events, NULL) < 0) {
4950421a
JA
578 ret = -1;
579 break;
580 }
581 } while (full && (td->cur_depth > td->o.iodepth_low));
582 }
583 if (ret < 0)
ebac4655 584 break;
36167d82 585 }
ebac4655 586
c01c0395
JA
587 if (!td->error) {
588 min_events = td->cur_depth;
589
590 if (min_events)
581e7141 591 ret = io_u_queued_complete(td, min_events, NULL);
c01c0395 592 } else
ebac4655
JA
593 cleanup_pending_aio(td);
594
595 td_set_runstate(td, TD_RUNNING);
008618a8
JA
596
597 dprint(FD_VERIFY, "exiting loop\n");
ebac4655
JA
598}
599
32cd46a0 600/*
906c8d75 601 * Main IO worker function. It retrieves io_u's to process and queues
32cd46a0
JA
602 * and reaps them, checking for rate and errors along the way.
603 */
ebac4655
JA
604static void do_io(struct thread_data *td)
605{
af52b345
JA
606 unsigned int i;
607 int ret = 0;
ebac4655 608
b29ee5b3
JA
609 if (in_ramp_time(td))
610 td_set_runstate(td, TD_RAMP);
611 else
612 td_set_runstate(td, TD_RUNNING);
5853e5a8 613
457bf399 614 while ( (td->o.read_iolog_file && !flist_empty(&td->io_log_list)) ||
a917a8b3 615 (!flist_empty(&td->trim_list)) ||
457bf399 616 ((td->this_io_bytes[0] + td->this_io_bytes[1]) < td->o.size) ) {
97601024 617 struct timeval comp_time;
581e7141 618 unsigned long bytes_done[2] = { 0, 0 };
84585003 619 int min_evts = 0;
ebac4655 620 struct io_u *io_u;
4950421a 621 int ret2, full;
ebac4655
JA
622
623 if (td->terminate)
624 break;
625
993bf48b 626 update_tv_cache(td);
97601024 627
993bf48b 628 if (runtime_exceeded(td, &td->tv_cache)) {
687c6a23
JA
629 __update_tv_cache(td);
630 if (runtime_exceeded(td, &td->tv_cache)) {
631 td->terminate = 1;
632 break;
633 }
97601024 634 }
36167d82 635
dcb4830d
RR
636 io_u = get_io_u(td);
637 if (!io_u)
638 break;
639
b67740d3
JA
640 /*
641 * Add verification end_io handler, if asked to verify
642 * a previously written file.
643 */
715d2b3e
JA
644 if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_READ &&
645 !td_rw(td)) {
e8462bd8
JA
646 if (td->o.verify_async)
647 io_u->end_io = verify_io_u_async;
648 else
649 io_u->end_io = verify_io_u;
d8fab1b6 650 td_set_runstate(td, TD_VERIFYING);
b29ee5b3
JA
651 } else if (in_ramp_time(td))
652 td_set_runstate(td, TD_RAMP);
653 else
d8fab1b6 654 td_set_runstate(td, TD_RUNNING);
b67740d3 655
11786802 656 ret = td_io_queue(td, io_u);
36167d82
JA
657 switch (ret) {
658 case FIO_Q_COMPLETED:
f2bba182 659 if (io_u->error) {
5451792e 660 ret = -io_u->error;
f2bba182
RR
661 clear_io_u(td, io_u);
662 } else if (io_u->resid) {
36167d82 663 int bytes = io_u->xfer_buflen - io_u->resid;
d460eb31 664 struct fio_file *f = io_u->file;
36167d82 665
9e9d164e
JA
666 /*
667 * zero read, fail
668 */
669 if (!bytes) {
41d9f0fd 670 td_verror(td, EIO, "full resid");
9e9d164e
JA
671 put_io_u(td, io_u);
672 break;
673 }
674
cec6b55d 675 io_u->xfer_buflen = io_u->resid;
36167d82 676 io_u->xfer_buf += bytes;
5a7c5680
JA
677 io_u->offset += bytes;
678
ff58fced
JA
679 if (ddir_rw(io_u->ddir))
680 td->ts.short_io_u[io_u->ddir]++;
30061b97 681
d460eb31 682 if (io_u->offset == f->real_file_size)
5a7c5680
JA
683 goto sync_done;
684
11786802
JA
685 requeue_io_u(td, &io_u);
686 } else {
5a7c5680 687sync_done:
581e7141
JA
688 if (__should_check_rate(td, 0) ||
689 __should_check_rate(td, 1))
993bf48b
JA
690 fio_gettime(&comp_time, NULL);
691
581e7141
JA
692 ret = io_u_sync_complete(td, io_u, bytes_done);
693 if (ret < 0)
694 break;
cec6b55d 695 }
36167d82
JA
696 break;
697 case FIO_Q_QUEUED:
7e77dd02
JA
698 /*
699 * if the engine doesn't have a commit hook,
700 * the io_u is really queued. if it does have such
701 * a hook, it has to call io_u_queued() itself.
702 */
703 if (td->io_ops->commit == NULL)
704 io_u_queued(td, io_u);
36167d82 705 break;
755200a3
JA
706 case FIO_Q_BUSY:
707 requeue_io_u(td, &io_u);
5451792e
JA
708 ret2 = td_io_commit(td);
709 if (ret2 < 0)
710 ret = ret2;
755200a3 711 break;
36167d82
JA
712 default:
713 assert(ret < 0);
714 put_io_u(td, io_u);
715 break;
ebac4655
JA
716 }
717
f2bba182 718 if (break_on_this_error(td, &ret))
36167d82
JA
719 break;
720
97601024 721 /*
ccc02050
JA
722 * See if we need to complete some commands. Note that we
723 * can get BUSY even without IO queued, if the system is
724 * resource starved.
97601024 725 */
ccc02050 726 full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
4950421a 727 if (full || !td->o.iodepth_batch_complete) {
6a96276c
RR
728 min_evts = min(td->o.iodepth_batch_complete,
729 td->cur_depth);
1f84f1d9 730 if (full && !min_evts && td->o.iodepth_batch_complete != 0)
36167d82 731 min_evts = 1;
4950421a 732
581e7141
JA
733 if (__should_check_rate(td, 0) ||
734 __should_check_rate(td, 1))
993bf48b 735 fio_gettime(&comp_time, NULL);
4950421a
JA
736
737 do {
581e7141
JA
738 ret = io_u_queued_complete(td, min_evts, bytes_done);
739 if (ret < 0)
4950421a
JA
740 break;
741
4950421a 742 } while (full && (td->cur_depth > td->o.iodepth_low));
ebac4655
JA
743 }
744
4950421a
JA
745 if (ret < 0)
746 break;
581e7141 747 if (!(bytes_done[0] + bytes_done[1]))
97601024
JA
748 continue;
749
581e7141
JA
750 if (!in_ramp_time(td) && should_check_rate(td, bytes_done)) {
751 if (check_min_rate(td, &comp_time, bytes_done)) {
721938ae
JA
752 if (exitall_on_terminate)
753 terminate_threads(td->groupid);
754 td_verror(td, EIO, "check_min_rate");
755 break;
756 }
ebac4655
JA
757 }
758
2dc1bbeb 759 if (td->o.thinktime) {
9c1f7434
JA
760 unsigned long long b;
761
762 b = td->io_blocks[0] + td->io_blocks[1];
2dc1bbeb 763 if (!(b % td->o.thinktime_blocks)) {
48097d5c
JA
764 int left;
765
2dc1bbeb 766 if (td->o.thinktime_spin)
6dd6f2cd 767 usec_spin(td->o.thinktime_spin);
48097d5c 768
2dc1bbeb 769 left = td->o.thinktime - td->o.thinktime_spin;
48097d5c
JA
770 if (left)
771 usec_sleep(td, left);
772 }
9c1f7434 773 }
ebac4655
JA
774 }
775
d708ac5e
JA
776 if (td->trim_entries)
777 printf("trim entries %ld\n", td->trim_entries);
a917a8b3 778
aa31f1f1
SL
779 if (td->o.fill_device && td->error == ENOSPC) {
780 td->error = 0;
781 td->terminate = 1;
782 }
4d2413c6 783 if (!td->error) {
3d7c391d
JA
784 struct fio_file *f;
785
c01c0395 786 i = td->cur_depth;
d06093e7 787 if (i) {
581e7141 788 ret = io_u_queued_complete(td, i, NULL);
d06093e7
SL
789 if (td->o.fill_device && td->error == ENOSPC)
790 td->error = 0;
791 }
ebac4655 792
2dc1bbeb 793 if (should_fsync(td) && td->o.end_fsync) {
84585003 794 td_set_runstate(td, TD_FSYNCING);
3d7b485f
JA
795
796 for_each_file(td, f, i) {
d6aed795 797 if (!fio_file_open(f))
3d7b485f 798 continue;
858a3d47 799 fio_io_sync(td, f);
3d7b485f 800 }
84585003 801 }
c01c0395
JA
802 } else
803 cleanup_pending_aio(td);
2ba1c290
JA
804
805 /*
806 * stop job if we failed doing any IO
807 */
808 if ((td->this_io_bytes[0] + td->this_io_bytes[1]) == 0)
809 td->done = 1;
ebac4655
JA
810}
811
ebac4655
JA
812static void cleanup_io_u(struct thread_data *td)
813{
01743ee1 814 struct flist_head *entry, *n;
ebac4655
JA
815 struct io_u *io_u;
816
01743ee1
JA
817 flist_for_each_safe(entry, n, &td->io_u_freelist) {
818 io_u = flist_entry(entry, struct io_u, list);
ebac4655 819
01743ee1 820 flist_del(&io_u->list);
91aea6ec 821 fio_memfree(io_u, sizeof(*io_u));
ebac4655
JA
822 }
823
2f9ade3c 824 free_io_mem(td);
ebac4655
JA
825}
826
827static int init_io_u(struct thread_data *td)
828{
829 struct io_u *io_u;
a00735e6 830 unsigned int max_bs;
eb7ccf38 831 int cl_align, i, max_units;
ebac4655
JA
832 char *p;
833
1e3c09af 834 max_units = td->o.iodepth;
2dc1bbeb 835 max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
5ec10eaa
JA
836 td->orig_buffer_size = (unsigned long long) max_bs
837 * (unsigned long long) max_units;
838
839 if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) {
840 unsigned long bs;
74b025b0 841
5ec10eaa
JA
842 bs = td->orig_buffer_size + td->o.hugepage_size - 1;
843 td->orig_buffer_size = bs & ~(td->o.hugepage_size - 1);
844 }
ebac4655 845
c3990645
JA
846 if (td->orig_buffer_size != (size_t) td->orig_buffer_size) {
847 log_err("fio: IO memory too large. Reduce max_bs or iodepth\n");
848 return 1;
849 }
850
2f9ade3c
JA
851 if (allocate_io_mem(td))
852 return 1;
ebac4655 853
1c04bf78
JA
854 if (td->o.odirect || td->o.mem_align ||
855 (td->io_ops->flags & FIO_RAWIO))
d529ee19 856 p = PAGE_ALIGN(td->orig_buffer) + td->o.mem_align;
b3f378e9
JA
857 else
858 p = td->orig_buffer;
859
eb7ccf38
JA
860 cl_align = os_cache_line_size();
861
ebac4655 862 for (i = 0; i < max_units; i++) {
eb7ccf38
JA
863 void *ptr;
864
db1cd90b
JA
865 if (td->terminate)
866 return 1;
eb7ccf38 867
91aea6ec
JA
868 ptr = fio_memalign(cl_align, sizeof(*io_u));
869 if (!ptr) {
870 log_err("fio: unable to allocate aligned memory\n");
eb7ccf38
JA
871 break;
872 }
873
874 io_u = ptr;
ebac4655 875 memset(io_u, 0, sizeof(*io_u));
01743ee1 876 INIT_FLIST_HEAD(&io_u->list);
d529ee19 877 dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i);
ebac4655 878
b4c5e1ac
JA
879 if (!(td->io_ops->flags & FIO_NOIO)) {
880 io_u->buf = p + max_bs * i;
d529ee19 881 dprint(FD_MEM, "io_u %p, mem %p\n", io_u, io_u->buf);
e9459e5a 882
3545a109 883 if (td_write(td))
5973cafb 884 io_u_fill_buffer(td, io_u, max_bs);
3545a109 885 if (td_write(td) && td->o.verify_pattern_bytes) {
cbe8d756
RR
886 /*
887 * Fill the buffer with the pattern if we are
888 * going to be doing writes.
889 */
7d9fb455 890 fill_pattern(td, io_u->buf, max_bs, io_u, 0, 0);
cbe8d756 891 }
b4c5e1ac 892 }
6b9cea23 893
b1ff3403 894 io_u->index = i;
0c6e7517 895 io_u->flags = IO_U_F_FREE;
01743ee1 896 flist_add(&io_u->list, &td->io_u_freelist);
ebac4655
JA
897 }
898
899 return 0;
900}
901
da86774e
JA
902static int switch_ioscheduler(struct thread_data *td)
903{
904 char tmp[256], tmp2[128];
905 FILE *f;
906 int ret;
907
ba0fbe10 908 if (td->io_ops->flags & FIO_DISKLESSIO)
f48b467c
JA
909 return 0;
910
da86774e
JA
911 sprintf(tmp, "%s/queue/scheduler", td->sysfs_root);
912
913 f = fopen(tmp, "r+");
914 if (!f) {
cbf5c1d7 915 if (errno == ENOENT) {
5ec10eaa
JA
916 log_err("fio: os or kernel doesn't support IO scheduler"
917 " switching\n");
cbf5c1d7
JA
918 return 0;
919 }
920 td_verror(td, errno, "fopen iosched");
da86774e
JA
921 return 1;
922 }
923
924 /*
925 * Set io scheduler.
926 */
2dc1bbeb 927 ret = fwrite(td->o.ioscheduler, strlen(td->o.ioscheduler), 1, f);
da86774e 928 if (ferror(f) || ret != 1) {
e1161c32 929 td_verror(td, errno, "fwrite");
da86774e
JA
930 fclose(f);
931 return 1;
932 }
933
934 rewind(f);
935
936 /*
937 * Read back and check that the selected scheduler is now the default.
938 */
939 ret = fread(tmp, 1, sizeof(tmp), f);
940 if (ferror(f) || ret < 0) {
e1161c32 941 td_verror(td, errno, "fread");
da86774e
JA
942 fclose(f);
943 return 1;
944 }
945
2dc1bbeb 946 sprintf(tmp2, "[%s]", td->o.ioscheduler);
da86774e 947 if (!strstr(tmp, tmp2)) {
2dc1bbeb 948 log_err("fio: io scheduler %s not found\n", td->o.ioscheduler);
e1161c32 949 td_verror(td, EINVAL, "iosched_switch");
da86774e
JA
950 fclose(f);
951 return 1;
952 }
953
954 fclose(f);
955 return 0;
956}
957
492158cf
JA
958static int keep_running(struct thread_data *td)
959{
960 unsigned long long io_done;
961
20e354ef
JA
962 if (td->done)
963 return 0;
492158cf
JA
964 if (td->o.time_based)
965 return 1;
966 if (td->o.loops) {
967 td->o.loops--;
968 return 1;
969 }
970
5ec10eaa
JA
971 io_done = td->io_bytes[DDIR_READ] + td->io_bytes[DDIR_WRITE]
972 + td->io_skip_bytes;
492158cf
JA
973 if (io_done < td->o.size)
974 return 1;
975
976 return 0;
977}
978
b29ee5b3 979static void reset_io_counters(struct thread_data *td)
ebac4655 980{
f0505a14 981 td->stat_io_bytes[0] = td->stat_io_bytes[1] = 0;
ebac4655 982 td->this_io_bytes[0] = td->this_io_bytes[1] = 0;
20dc95c4 983 td->zone_bytes = 0;
581e7141
JA
984 td->rate_bytes[0] = td->rate_bytes[1] = 0;
985 td->rate_blocks[0] = td->rate_blocks[1] = 0;
ebac4655 986
c1324df1
JA
987 td->last_was_sync = 0;
988
2ba1c290
JA
989 /*
990 * reset file done count if we are to start over
991 */
992 if (td->o.time_based || td->o.loops)
48f5abd3 993 td->nr_done_files = 0;
b29ee5b3
JA
994}
995
996void reset_all_stats(struct thread_data *td)
997{
998 struct timeval tv;
999 int i;
1000
1001 reset_io_counters(td);
1002
1003 for (i = 0; i < 2; i++) {
1004 td->io_bytes[i] = 0;
1005 td->io_blocks[i] = 0;
1006 td->io_issues[i] = 0;
1007 td->ts.total_io_u[i] = 0;
1008 }
9b836561 1009
b29ee5b3 1010 fio_gettime(&tv, NULL);
cda99fa0
SSP
1011 td->ts.runtime[0] = 0;
1012 td->ts.runtime[1] = 0;
b29ee5b3
JA
1013 memcpy(&td->epoch, &tv, sizeof(tv));
1014 memcpy(&td->start, &tv, sizeof(tv));
1015}
1016
df9c26b1 1017static void clear_io_state(struct thread_data *td)
b29ee5b3
JA
1018{
1019 struct fio_file *f;
1020 unsigned int i;
b29ee5b3
JA
1021
1022 reset_io_counters(td);
281f9b63 1023
24ffd2c2 1024 close_files(td);
df9c26b1 1025 for_each_file(td, f, i)
d6aed795 1026 fio_file_clear_done(f);
f0153948
JA
1027
1028 /*
1029 * Set the same seed to get repeatable runs
1030 */
1031 td_fill_rand_seeds(td);
ebac4655
JA
1032}
1033
398e8c4d
JA
1034static int exec_string(const char *string)
1035{
1036 int ret, newlen = strlen(string) + 1 + 8;
1037 char *str;
1038
1039 str = malloc(newlen);
1040 sprintf(str, "sh -c %s", string);
1041
1042 ret = system(str);
1043 if (ret == -1)
1044 log_err("fio: exec of cmd <%s> failed\n", str);
1045
1046 free(str);
1047 return ret;
1048}
1049
906c8d75
JA
1050/*
1051 * Entry point for the thread based jobs. The process based jobs end up
1052 * here as well, after a little setup.
1053 */
ebac4655
JA
1054static void *thread_main(void *data)
1055{
cda99fa0 1056 unsigned long long elapsed;
ebac4655 1057 struct thread_data *td = data;
e8462bd8 1058 pthread_condattr_t attr;
a978ba68 1059 int clear_state;
ebac4655 1060
47f767c1 1061 if (!td->o.use_thread) {
ebac4655 1062 setsid();
47f767c1
JA
1063 td->pid = getpid();
1064 } else
1065 td->pid = gettid();
ebac4655 1066
5921e80c 1067 dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid);
ee56ad50 1068
01743ee1
JA
1069 INIT_FLIST_HEAD(&td->io_u_freelist);
1070 INIT_FLIST_HEAD(&td->io_u_busylist);
1071 INIT_FLIST_HEAD(&td->io_u_requeues);
1072 INIT_FLIST_HEAD(&td->io_log_list);
1073 INIT_FLIST_HEAD(&td->io_hist_list);
e8462bd8 1074 INIT_FLIST_HEAD(&td->verify_list);
0d29de83 1075 INIT_FLIST_HEAD(&td->trim_list);
e8462bd8 1076 pthread_mutex_init(&td->io_u_lock, NULL);
bb5d7d0b 1077 td->io_hist_tree = RB_ROOT;
aea47d44 1078
e8462bd8
JA
1079 pthread_condattr_init(&attr);
1080 pthread_cond_init(&td->verify_cond, &attr);
1081 pthread_cond_init(&td->free_cond, &attr);
1082
db1cd90b 1083 td_set_runstate(td, TD_INITIALIZED);
29adda3c 1084 dprint(FD_MUTEX, "up startup_mutex\n");
cdd18ad8 1085 fio_mutex_up(startup_mutex);
29adda3c 1086 dprint(FD_MUTEX, "wait on td->mutex\n");
cdd18ad8 1087 fio_mutex_down(td->mutex);
29adda3c 1088 dprint(FD_MUTEX, "done waiting on td->mutex\n");
db1cd90b
JA
1089
1090 /*
cdd18ad8 1091 * the ->mutex mutex is now no longer used, close it to avoid
db1cd90b
JA
1092 * eating a file descriptor
1093 */
cdd18ad8 1094 fio_mutex_remove(td->mutex);
db1cd90b 1095
dfc6e751
SH
1096 /*
1097 * A new gid requires privilege, so we need to do this before setting
1098 * the uid.
1099 */
e0b0d892
JA
1100 if (td->o.gid != -1U && setgid(td->o.gid)) {
1101 td_verror(td, errno, "setgid");
1102 goto err;
1103 }
dfc6e751
SH
1104 if (td->o.uid != -1U && setuid(td->o.uid)) {
1105 td_verror(td, errno, "setuid");
1106 goto err;
1107 }
e0b0d892 1108
be18a94f
JA
1109 /*
1110 * If we have a gettimeofday() thread, make sure we exclude that
1111 * thread from this job
1112 */
1113 if (td->o.gtod_cpu)
1114 fio_cpu_clear(&td->o.cpumask, td->o.gtod_cpu);
1115
1116 /*
1117 * Set affinity first, in case it has an impact on the memory
1118 * allocations.
1119 */
1120 if (td->o.cpumask_set && fio_setaffinity(td->pid, td->o.cpumask) == -1) {
1121 td_verror(td, errno, "cpu_set_affinity");
1122 goto err;
1123 }
1124
d84f8d49
JA
1125 /*
1126 * May alter parameters that init_io_u() will use, so we need to
1127 * do this first.
1128 */
1129 if (init_iolog(td))
1130 goto err;
1131
ebac4655 1132 if (init_io_u(td))
db1cd90b 1133 goto err;
ebac4655 1134
e8462bd8
JA
1135 if (td->o.verify_async && verify_async_init(td))
1136 goto err;
1137
ac684785 1138 if (td->ioprio_set) {
ebac4655 1139 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
e1161c32 1140 td_verror(td, errno, "ioprio_set");
db1cd90b 1141 goto err;
ebac4655
JA
1142 }
1143 }
1144
6adb38a1 1145 if (td->o.cgroup_weight && cgroup_setup(td, cgroup_list, &cgroup_mnt))
a696fa2a
JA
1146 goto err;
1147
2dc1bbeb 1148 if (nice(td->o.nice) == -1) {
e1161c32 1149 td_verror(td, errno, "nice");
db1cd90b 1150 goto err;
b6f4d880
JA
1151 }
1152
2dc1bbeb 1153 if (td->o.ioscheduler && switch_ioscheduler(td))
db1cd90b 1154 goto err;
37f56873 1155
2dc1bbeb 1156 if (!td->o.create_serialize && setup_files(td))
ebac4655
JA
1157 goto err;
1158
7d6c5283
JA
1159 if (td_io_init(td))
1160 goto err;
1161
68727076
JA
1162 if (init_random_map(td))
1163 goto err;
1164
2dc1bbeb 1165 if (td->o.exec_prerun) {
398e8c4d 1166 if (exec_string(td->o.exec_prerun))
69cfd7e0
JA
1167 goto err;
1168 }
4e0ba8af 1169
afad68f7
ZY
1170 if (td->o.pre_read) {
1171 if (pre_read_files(td) < 0)
1172 goto err;
1173 }
1174
69008999 1175 fio_gettime(&td->epoch, NULL);
c8aaba19 1176 getrusage(RUSAGE_SELF, &td->ru_start);
69008999 1177
a978ba68 1178 clear_state = 0;
492158cf 1179 while (keep_running(td)) {
02bcaa8c 1180 fio_gettime(&td->start, NULL);
f0505a14
JA
1181 memcpy(&td->stat_sample_time[0], &td->start, sizeof(td->start));
1182 memcpy(&td->stat_sample_time[1], &td->start, sizeof(td->start));
993bf48b 1183 memcpy(&td->tv_cache, &td->start, sizeof(td->start));
ebac4655 1184
cf2dd8d6 1185 if (td->o.ratemin[0] || td->o.ratemin[1])
f0505a14 1186 memcpy(&td->lastrate, &td->stat_sample_time,
5ec10eaa 1187 sizeof(td->lastrate));
ebac4655 1188
df9c26b1
JA
1189 if (clear_state)
1190 clear_io_state(td);
a978ba68 1191
ebac4655
JA
1192 prune_io_piece_log(td);
1193
ba0fbe10 1194 do_io(td);
ebac4655 1195
a978ba68
JA
1196 clear_state = 1;
1197
38d77cae 1198 if (td_read(td) && td->io_bytes[DDIR_READ]) {
0a4957c6 1199 elapsed = utime_since_now(&td->start);
cda99fa0 1200 td->ts.runtime[DDIR_READ] += elapsed;
38d77cae
JA
1201 }
1202 if (td_write(td) && td->io_bytes[DDIR_WRITE]) {
0a4957c6 1203 elapsed = utime_since_now(&td->start);
cda99fa0 1204 td->ts.runtime[DDIR_WRITE] += elapsed;
38d77cae 1205 }
5ec10eaa 1206
ebac4655
JA
1207 if (td->error || td->terminate)
1208 break;
1209
e84c73a8
SL
1210 if (!td->o.do_verify ||
1211 td->o.verify == VERIFY_NONE ||
b67740d3 1212 (td->io_ops->flags & FIO_UNIDIR))
ebac4655
JA
1213 continue;
1214
df9c26b1 1215 clear_io_state(td);
a978ba68 1216
02bcaa8c 1217 fio_gettime(&td->start, NULL);
ebac4655
JA
1218
1219 do_verify(td);
1220
cda99fa0 1221 td->ts.runtime[DDIR_READ] += utime_since_now(&td->start);
ebac4655
JA
1222
1223 if (td->error || td->terminate)
1224 break;
1225 }
1226
36dff966 1227 update_rusage_stat(td);
cda99fa0
SSP
1228 td->ts.runtime[0] = (td->ts.runtime[0] + 999) / 1000;
1229 td->ts.runtime[1] = (td->ts.runtime[1] + 999) / 1000;
756867bd
JA
1230 td->ts.total_run_time = mtime_since_now(&td->epoch);
1231 td->ts.io_bytes[0] = td->io_bytes[0];
1232 td->ts.io_bytes[1] = td->io_bytes[1];
1233
d2bbb8d9 1234 fio_mutex_down(writeout_mutex);
7b9f733a 1235 if (td->bw_log) {
f484470b 1236 if (td->o.bw_log_file) {
7b9f733a 1237 finish_log_named(td, td->bw_log,
f484470b
JA
1238 td->o.bw_log_file, "bw");
1239 } else
7b9f733a 1240 finish_log(td, td->bw_log, "bw");
e3cedca7 1241 }
7b9f733a 1242 if (td->lat_log) {
02af0988 1243 if (td->o.lat_log_file) {
7b9f733a 1244 finish_log_named(td, td->lat_log,
02af0988
JA
1245 td->o.lat_log_file, "lat");
1246 } else
7b9f733a 1247 finish_log(td, td->lat_log, "lat");
02af0988 1248 }
7b9f733a 1249 if (td->slat_log) {
f484470b 1250 if (td->o.lat_log_file) {
7b9f733a 1251 finish_log_named(td, td->slat_log,
1ca95738 1252 td->o.lat_log_file, "slat");
f484470b 1253 } else
7b9f733a 1254 finish_log(td, td->slat_log, "slat");
e3cedca7 1255 }
7b9f733a 1256 if (td->clat_log) {
f484470b 1257 if (td->o.lat_log_file) {
7b9f733a 1258 finish_log_named(td, td->clat_log,
f484470b
JA
1259 td->o.lat_log_file, "clat");
1260 } else
7b9f733a 1261 finish_log(td, td->clat_log, "clat");
e3cedca7 1262 }
d2bbb8d9 1263 fio_mutex_up(writeout_mutex);
398e8c4d
JA
1264 if (td->o.exec_postrun)
1265 exec_string(td->o.exec_postrun);
ebac4655
JA
1266
1267 if (exitall_on_terminate)
390c40e2 1268 terminate_threads(td->groupid);
ebac4655
JA
1269
1270err:
5bf13a5a 1271 if (td->error)
4ceb30d4 1272 log_info("fio: pid=%d, err=%d/%s\n", (int) td->pid, td->error,
5ec10eaa 1273 td->verror);
b4707ed7
JA
1274
1275 if (td->o.verify_async)
1276 verify_async_exit(td);
1277
24ffd2c2 1278 close_and_free_files(td);
2866c82d 1279 close_ioengine(td);
ebac4655 1280 cleanup_io_u(td);
6adb38a1 1281 cgroup_shutdown(td, &cgroup_mnt);
f29b25a3 1282
d2ce18b5
JA
1283 if (td->o.cpumask_set) {
1284 int ret = fio_cpuset_exit(&td->o.cpumask);
1285
1286 td_verror(td, ret, "fio_cpuset_exit");
1287 }
4e78e405 1288
f29b25a3
JA
1289 /*
1290 * do this very late, it will log file closing as well
1291 */
1292 if (td->o.write_iolog_file)
1293 write_iolog_close(td);
1294
d23bb327 1295 options_mem_free(td);
ebac4655 1296 td_set_runstate(td, TD_EXITED);
43d76807 1297 return (void *) (unsigned long) td->error;
ebac4655
JA
1298}
1299
906c8d75
JA
1300/*
1301 * We cannot pass the td data into a forked process, so attach the td and
1302 * pass it to the thread worker.
1303 */
a6418147 1304static int fork_main(int shmid, int offset)
ebac4655
JA
1305{
1306 struct thread_data *td;
a6418147 1307 void *data, *ret;
ebac4655 1308
65258961 1309#ifndef __hpux
ebac4655
JA
1310 data = shmat(shmid, NULL, 0);
1311 if (data == (void *) -1) {
a6418147
JA
1312 int __err = errno;
1313
ebac4655 1314 perror("shmat");
a6418147 1315 return __err;
ebac4655 1316 }
65258961
JA
1317#else
1318 /*
1319 * HP-UX inherits shm mappings?
1320 */
1321 data = threads;
1322#endif
ebac4655
JA
1323
1324 td = data + offset * sizeof(struct thread_data);
a6418147 1325 ret = thread_main(td);
ebac4655 1326 shmdt(data);
43d76807 1327 return (int) (unsigned long) ret;
ebac4655
JA
1328}
1329
906c8d75
JA
1330/*
1331 * Run over the job map and reap the threads that have exited, if any.
1332 */
ebac4655
JA
1333static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
1334{
34572e28 1335 struct thread_data *td;
1f809d15 1336 int i, cputhreads, realthreads, pending, status, ret;
ebac4655
JA
1337
1338 /*
1339 * reap exited threads (TD_EXITED -> TD_REAPED)
1340 */
1f809d15 1341 realthreads = pending = cputhreads = 0;
34572e28 1342 for_each_td(td, i) {
3707f45b 1343 int flags = 0;
a2f77c9f 1344
84585003
JA
1345 /*
1346 * ->io_ops is NULL for a thread that has closed its
1347 * io engine
1348 */
ba0fbe10 1349 if (td->io_ops && !strcmp(td->io_ops->name, "cpuio"))
b990b5c0 1350 cputhreads++;
1f809d15
JA
1351 else
1352 realthreads++;
b990b5c0 1353
a1dedc1f
AC
1354 if (!td->pid) {
1355 pending++;
1356 continue;
1357 }
1358 if (td->runstate == TD_REAPED)
a2f77c9f 1359 continue;
2dc1bbeb 1360 if (td->o.use_thread) {
3707f45b
JA
1361 if (td->runstate == TD_EXITED) {
1362 td_set_runstate(td, TD_REAPED);
1363 goto reaped;
1364 }
1365 continue;
1366 }
a2f77c9f
JA
1367
1368 flags = WNOHANG;
1369 if (td->runstate == TD_EXITED)
1370 flags = 0;
1371
1372 /*
1373 * check if someone quit or got killed in an unusual way
1374 */
1375 ret = waitpid(td->pid, &status, flags);
3707f45b 1376 if (ret < 0) {
a2f77c9f 1377 if (errno == ECHILD) {
5921e80c
JA
1378 log_err("fio: pid=%d disappeared %d\n",
1379 (int) td->pid, td->runstate);
a2f77c9f
JA
1380 td_set_runstate(td, TD_REAPED);
1381 goto reaped;
1382 }
1383 perror("waitpid");
1384 } else if (ret == td->pid) {
1385 if (WIFSIGNALED(status)) {
fab6aa71
JA
1386 int sig = WTERMSIG(status);
1387
03e20d68 1388 if (sig != SIGTERM)
5ec10eaa 1389 log_err("fio: pid=%d, got signal=%d\n",
5921e80c 1390 (int) td->pid, sig);
fab6aa71
JA
1391 td_set_runstate(td, TD_REAPED);
1392 goto reaped;
1393 }
a2f77c9f
JA
1394 if (WIFEXITED(status)) {
1395 if (WEXITSTATUS(status) && !td->error)
1396 td->error = WEXITSTATUS(status);
a2f77c9f 1397
a2f77c9f
JA
1398 td_set_runstate(td, TD_REAPED);
1399 goto reaped;
a6418147
JA
1400 }
1401 }
ebac4655 1402
a2f77c9f
JA
1403 /*
1404 * thread is not dead, continue
1405 */
e48676ba 1406 pending++;
a2f77c9f 1407 continue;
fab6aa71 1408reaped:
ebac4655 1409 (*nr_running)--;
581e7141
JA
1410 (*m_rate) -= (td->o.ratemin[0] + td->o.ratemin[1]);
1411 (*t_rate) -= (td->o.rate[0] + td->o.rate[1]);
6f88bcb0
JA
1412 if (!td->pid)
1413 pending--;
a2f77c9f
JA
1414
1415 if (td->error)
1416 exit_value++;
d3eeeabc
JA
1417
1418 done_secs += mtime_since_now(&td->epoch) / 1000;
ebac4655 1419 }
b990b5c0 1420
1f809d15 1421 if (*nr_running == cputhreads && !pending && realthreads)
390c40e2 1422 terminate_threads(TERMINATE_ALL);
ebac4655
JA
1423}
1424
be4ecfdf
JA
1425static void *gtod_thread_main(void *data)
1426{
1427 fio_mutex_up(startup_mutex);
1428
1429 /*
1430 * As long as we have jobs around, update the clock. It would be nice
1431 * to have some way of NOT hammering that CPU with gettimeofday(),
1432 * but I'm not sure what to use outside of a simple CPU nop to relax
1433 * it - we don't want to lose precision.
1434 */
1435 while (threads) {
1436 fio_gtod_update();
1437 nop;
1438 }
1439
1440 return NULL;
1441}
1442
1443static int fio_start_gtod_thread(void)
1444{
304a47c7 1445 pthread_attr_t attr;
f42c153d
JA
1446 int ret;
1447
304a47c7
VA
1448 pthread_attr_init(&attr);
1449 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
1450 ret = pthread_create(&gtod_thread, &attr, gtod_thread_main, NULL);
1451 pthread_attr_destroy(&attr);
f42c153d
JA
1452 if (ret) {
1453 log_err("Can't create gtod thread: %s\n", strerror(ret));
be4ecfdf
JA
1454 return 1;
1455 }
f42c153d
JA
1456
1457 ret = pthread_detach(gtod_thread);
1458 if (ret) {
1459 log_err("Can't detatch gtod thread: %s\n", strerror(ret));
be4ecfdf
JA
1460 return 1;
1461 }
1462
29adda3c 1463 dprint(FD_MUTEX, "wait on startup_mutex\n");
be4ecfdf 1464 fio_mutex_down(startup_mutex);
29adda3c 1465 dprint(FD_MUTEX, "done waiting on startup_mutex\n");
be4ecfdf
JA
1466 return 0;
1467}
1468
906c8d75
JA
1469/*
1470 * Main function for kicking off and reaping jobs, as needed.
1471 */
ebac4655
JA
1472static void run_threads(void)
1473{
ebac4655
JA
1474 struct thread_data *td;
1475 unsigned long spent;
1476 int i, todo, nr_running, m_rate, t_rate, nr_started;
fcb6ade2 1477
2f9ade3c
JA
1478 if (fio_pin_memory())
1479 return;
ebac4655 1480
be4ecfdf
JA
1481 if (fio_gtod_offload && fio_start_gtod_thread())
1482 return;
1483
c6ae0a5b 1484 if (!terse_output) {
4ceb30d4 1485 log_info("Starting ");
9cedf167 1486 if (nr_thread)
4ceb30d4 1487 log_info("%d thread%s", nr_thread,
5ec10eaa 1488 nr_thread > 1 ? "s" : "");
9cedf167
JA
1489 if (nr_process) {
1490 if (nr_thread)
1491 printf(" and ");
4ceb30d4 1492 log_info("%d process%s", nr_process,
5ec10eaa 1493 nr_process > 1 ? "es" : "");
9cedf167 1494 }
4ceb30d4 1495 log_info("\n");
c6ae0a5b
JA
1496 fflush(stdout);
1497 }
c04f7ec3 1498
697a606c 1499 set_sig_handlers();
4efa970e 1500
ebac4655
JA
1501 todo = thread_number;
1502 nr_running = 0;
1503 nr_started = 0;
1504 m_rate = t_rate = 0;
1505
34572e28 1506 for_each_td(td, i) {
263e529f 1507 print_status_init(td->thread_number - 1);
ebac4655 1508
067430bb 1509 if (!td->o.create_serialize)
ebac4655
JA
1510 continue;
1511
1512 /*
1513 * do file setup here so it happens sequentially,
1514 * we don't want X number of threads getting their
1515 * client data interspersed on disk
1516 */
53cdc686 1517 if (setup_files(td)) {
5bf13a5a
JA
1518 exit_value++;
1519 if (td->error)
5921e80c
JA
1520 log_err("fio: pid=%d, err=%d/%s\n",
1521 (int) td->pid, td->error, td->verror);
ebac4655
JA
1522 td_set_runstate(td, TD_REAPED);
1523 todo--;
c69e8875
JA
1524 } else {
1525 struct fio_file *f;
2b13e716 1526 unsigned int j;
c69e8875
JA
1527
1528 /*
1529 * for sharing to work, each job must always open
1530 * its own files. so close them, if we opened them
1531 * for creation
1532 */
2b13e716 1533 for_each_file(td, f, j) {
22a57ba8
JA
1534 if (fio_file_open(f))
1535 td_io_close_file(td, f);
22a57ba8 1536 }
5ec10eaa 1537 }
ebac4655
JA
1538 }
1539
a2f77c9f
JA
1540 set_genesis_time();
1541
ebac4655 1542 while (todo) {
fca70358 1543 struct thread_data *map[REAL_MAX_JOBS];
75154845
JA
1544 struct timeval this_start;
1545 int this_jobs = 0, left;
1546
ebac4655
JA
1547 /*
1548 * create threads (TD_NOT_CREATED -> TD_CREATED)
1549 */
34572e28 1550 for_each_td(td, i) {
ebac4655
JA
1551 if (td->runstate != TD_NOT_CREATED)
1552 continue;
1553
1554 /*
1555 * never got a chance to start, killed by other
1556 * thread for some reason
1557 */
1558 if (td->terminate) {
1559 todo--;
1560 continue;
1561 }
1562
2dc1bbeb 1563 if (td->o.start_delay) {
263e529f 1564 spent = mtime_since_genesis();
ebac4655 1565
2dc1bbeb 1566 if (td->o.start_delay * 1000 > spent)
ebac4655
JA
1567 continue;
1568 }
1569
6f88bcb0
JA
1570 if (td->o.stonewall && (nr_started || nr_running)) {
1571 dprint(FD_PROCESS, "%s: stonewall wait\n",
1572 td->o.name);
ebac4655 1573 break;
6f88bcb0 1574 }
ebac4655 1575
067430bb
JA
1576 init_disk_util(td);
1577
75154845
JA
1578 /*
1579 * Set state to created. Thread will transition
1580 * to TD_INITIALIZED when it's done setting up.
1581 */
ebac4655 1582 td_set_runstate(td, TD_CREATED);
75154845 1583 map[this_jobs++] = td;
ebac4655
JA
1584 nr_started++;
1585
2dc1bbeb 1586 if (td->o.use_thread) {
f42c153d
JA
1587 int ret;
1588
ee56ad50 1589 dprint(FD_PROCESS, "will pthread_create\n");
f42c153d
JA
1590 ret = pthread_create(&td->thread, NULL,
1591 thread_main, td);
1592 if (ret) {
1593 log_err("pthread_create: %s\n",
1594 strerror(ret));
ebac4655 1595 nr_started--;
c8bb6faf 1596 break;
ebac4655 1597 }
f42c153d
JA
1598 ret = pthread_detach(td->thread);
1599 if (ret)
1600 log_err("pthread_detach: %s",
1601 strerror(ret));
ebac4655 1602 } else {
5e1d306e 1603 pid_t pid;
ee56ad50 1604 dprint(FD_PROCESS, "will fork\n");
5e1d306e
JA
1605 pid = fork();
1606 if (!pid) {
a6418147
JA
1607 int ret = fork_main(shm_id, i);
1608
5e1d306e
JA
1609 _exit(ret);
1610 } else if (i == fio_debug_jobno)
1611 *fio_debug_jobp = pid;
ebac4655 1612 }
29adda3c 1613 dprint(FD_MUTEX, "wait on startup_mutex\n");
656b1393
JA
1614 if (fio_mutex_down_timeout(startup_mutex, 10)) {
1615 log_err("fio: job startup hung? exiting.\n");
1616 terminate_threads(TERMINATE_ALL);
1617 fio_abort = 1;
1618 nr_started--;
1619 break;
1620 }
29adda3c 1621 dprint(FD_MUTEX, "done waiting on startup_mutex\n");
ebac4655
JA
1622 }
1623
1624 /*
75154845
JA
1625 * Wait for the started threads to transition to
1626 * TD_INITIALIZED.
ebac4655 1627 */
02bcaa8c 1628 fio_gettime(&this_start, NULL);
75154845 1629 left = this_jobs;
6ce15a32 1630 while (left && !fio_abort) {
75154845
JA
1631 if (mtime_since_now(&this_start) > JOB_START_TIMEOUT)
1632 break;
1633
1634 usleep(100000);
1635
1636 for (i = 0; i < this_jobs; i++) {
1637 td = map[i];
1638 if (!td)
1639 continue;
b6f4d880 1640 if (td->runstate == TD_INITIALIZED) {
75154845
JA
1641 map[i] = NULL;
1642 left--;
b6f4d880
JA
1643 } else if (td->runstate >= TD_EXITED) {
1644 map[i] = NULL;
1645 left--;
1646 todo--;
1647 nr_running++; /* work-around... */
75154845
JA
1648 }
1649 }
1650 }
1651
1652 if (left) {
3b70d7e5 1653 log_err("fio: %d jobs failed to start\n", left);
75154845
JA
1654 for (i = 0; i < this_jobs; i++) {
1655 td = map[i];
1656 if (!td)
1657 continue;
1658 kill(td->pid, SIGTERM);
1659 }
1660 break;
1661 }
1662
1663 /*
b6f4d880 1664 * start created threads (TD_INITIALIZED -> TD_RUNNING).
75154845 1665 */
34572e28 1666 for_each_td(td, i) {
75154845 1667 if (td->runstate != TD_INITIALIZED)
ebac4655
JA
1668 continue;
1669
b29ee5b3
JA
1670 if (in_ramp_time(td))
1671 td_set_runstate(td, TD_RAMP);
1672 else
1673 td_set_runstate(td, TD_RUNNING);
ebac4655
JA
1674 nr_running++;
1675 nr_started--;
581e7141
JA
1676 m_rate += td->o.ratemin[0] + td->o.ratemin[1];
1677 t_rate += td->o.rate[0] + td->o.rate[1];
75154845 1678 todo--;
cdd18ad8 1679 fio_mutex_up(td->mutex);
ebac4655
JA
1680 }
1681
1682 reap_threads(&nr_running, &t_rate, &m_rate);
1683
1684 if (todo)
1685 usleep(100000);
1686 }
1687
1688 while (nr_running) {
1689 reap_threads(&nr_running, &t_rate, &m_rate);
1690 usleep(10000);
1691 }
1692
1693 update_io_ticks();
2f9ade3c 1694 fio_unpin_memory();
ebac4655
JA
1695}
1696
50d16976 1697int exec_run(void)
ebac4655 1698{
a37f69b7 1699 if (nr_clients)
37db14fe 1700 return fio_handle_clients();
07b3232d
JA
1701 if (exec_profile && load_profile(exec_profile))
1702 return 1;
1703
74929ac2
JA
1704 if (!thread_number)
1705 return 0;
1706
bb3884d8
JA
1707 if (write_bw_log) {
1708 setup_log(&agg_io_log[DDIR_READ]);
1709 setup_log(&agg_io_log[DDIR_WRITE]);
1710 }
1711
cdd18ad8 1712 startup_mutex = fio_mutex_init(0);
03e20d68
BC
1713 if (startup_mutex == NULL)
1714 return 1;
d2bbb8d9 1715 writeout_mutex = fio_mutex_init(1);
03e20d68
BC
1716 if (writeout_mutex == NULL)
1717 return 1;
07739b57 1718
a2f77c9f 1719 set_genesis_time();
7985701b 1720 create_disk_util_thread();
ebac4655 1721
dae5341c
JA
1722 cgroup_list = smalloc(sizeof(*cgroup_list));
1723 INIT_FLIST_HEAD(cgroup_list);
1724
ebac4655 1725 run_threads();
6ce15a32 1726
bb3884d8 1727 if (!fio_abort) {
6ce15a32 1728 show_run_stats();
bb3884d8 1729 if (write_bw_log) {
5ec10eaa
JA
1730 __finish_log(agg_io_log[DDIR_READ], "agg-read_bw.log");
1731 __finish_log(agg_io_log[DDIR_WRITE],
1732 "agg-write_bw.log");
bb3884d8
JA
1733 }
1734 }
ebac4655 1735
dae5341c
JA
1736 cgroup_kill(cgroup_list);
1737 sfree(cgroup_list);
61eb313e 1738 sfree(cgroup_mnt);
39f22027 1739
cdd18ad8 1740 fio_mutex_remove(startup_mutex);
d2bbb8d9 1741 fio_mutex_remove(writeout_mutex);
437c9b71 1742 return exit_value;
ebac4655 1743}
50d16976
JA
1744
1745void reset_fio_state(void)
1746{
1747 groupid = 0;
1748 thread_number = 0;
1749 nr_process = 0;
1750 nr_thread = 0;
1751 done_secs = 0;
1752}
1753
1754int main(int argc, char *argv[], char *envp[])
1755{
1756 long ps;
1757
1758 arch_init(envp);
1759
1760 sinit();
1761
1762 /*
1763 * We need locale for number printing, if it isn't set then just
1764 * go with the US format.
1765 */
1766 if (!getenv("LC_NUMERIC"))
1767 setlocale(LC_NUMERIC, "en_US");
1768
1769 ps = sysconf(_SC_PAGESIZE);
1770 if (ps < 0) {
1771 log_err("Failed to get page size\n");
1772 return 1;
1773 }
1774
1775 page_size = ps;
1776 page_mask = ps - 1;
1777
1778 fio_keywords_init();
1779
1780 if (parse_options(argc, argv))
1781 return 1;
1782
1783 return exec_run();
1784}