Add write barriers
[fio.git] / io_u.c
CommitLineData
10ba535a
JA
1#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <signal.h>
5#include <time.h>
0c6e7517 6#include <assert.h>
10ba535a
JA
7
8#include "fio.h"
5973cafb 9#include "hash.h"
10ba535a 10
5945b9b4
JA
11/*
12 * Change this define to play with the timeout handling
13 */
14#undef FIO_USE_TIMEOUT
15
97601024
JA
16struct io_completion_data {
17 int nr; /* input */
97601024
JA
18
19 int error; /* output */
20 unsigned long bytes_done[2]; /* output */
21 struct timeval time; /* output */
22};
23
10ba535a
JA
24/*
25 * The ->file_map[] contains a map of blocks we have or have not done io
26 * to yet. Used to make sure we cover the entire range in a fair fashion.
27 */
aec2de20 28static int random_map_free(struct fio_file *f, const unsigned long long block)
10ba535a 29{
aec2de20
JA
30 unsigned int idx = RAND_MAP_IDX(f, block);
31 unsigned int bit = RAND_MAP_BIT(f, block);
10ba535a 32
84422acd
JA
33 dprint(FD_RANDOM, "free: b=%llu, idx=%u, bit=%u\n", block, idx, bit);
34
de605666 35 return (f->file_map[idx] & (1 << bit)) == 0;
10ba535a
JA
36}
37
df415585
JA
38/*
39 * Mark a given offset as used in the map.
40 */
9bf2061e 41static void mark_random_map(struct thread_data *td, struct io_u *io_u)
df415585 42{
2dc1bbeb 43 unsigned int min_bs = td->o.rw_min_bs;
9bf2061e 44 struct fio_file *f = io_u->file;
a00735e6 45 unsigned long long block;
3e3357b1 46 unsigned int blocks, nr_blocks;
df415585 47
b9c5b644 48 block = (io_u->offset - f->file_offset) / (unsigned long long) min_bs;
c685b5b2 49 nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
3e3357b1 50 blocks = 0;
c685b5b2 51
3e3357b1
JA
52 while (nr_blocks) {
53 unsigned int this_blocks, mask;
df415585
JA
54 unsigned int idx, bit;
55
1e3d53ac
JA
56 /*
57 * If we have a mixed random workload, we may
58 * encounter blocks we already did IO to.
59 */
8b113b4f
JA
60 if ((td->o.ddir_nr == 1) && !random_map_free(f, block)) {
61 if (!blocks)
62 blocks = 1;
df415585 63 break;
8b113b4f 64 }
df415585 65
aec2de20
JA
66 idx = RAND_MAP_IDX(f, block);
67 bit = RAND_MAP_BIT(f, block);
df415585 68
0032bf9f 69 fio_assert(td, idx < f->num_maps);
df415585 70
3e3357b1
JA
71 this_blocks = nr_blocks;
72 if (this_blocks + bit > BLOCKS_PER_MAP)
73 this_blocks = BLOCKS_PER_MAP - bit;
74
75 if (this_blocks == BLOCKS_PER_MAP)
76 mask = -1U;
77 else
78 mask = ((1U << this_blocks) - 1) << bit;
79
80 fio_assert(td, !(f->file_map[idx] & mask));
81 f->file_map[idx] |= mask;
82 nr_blocks -= this_blocks;
83 blocks += this_blocks;
c9dd34b2 84 block += this_blocks;
df415585
JA
85 }
86
a00735e6
JA
87 if ((blocks * min_bs) < io_u->buflen)
88 io_u->buflen = blocks * min_bs;
df415585
JA
89}
90
3e3357b1
JA
91static unsigned long long last_block(struct thread_data *td, struct fio_file *f,
92 enum fio_ddir ddir)
2ba1c290
JA
93{
94 unsigned long long max_blocks;
d9dd70f7 95 unsigned long long max_size;
2ba1c290 96
d9dd70f7
JA
97 /*
98 * Hmm, should we make sure that ->io_size <= ->real_file_size?
99 */
100 max_size = f->io_size;
101 if (max_size > f->real_file_size)
102 max_size = f->real_file_size;
103
104 max_blocks = max_size / (unsigned long long) td->o.min_bs[ddir];
2ba1c290
JA
105 if (!max_blocks)
106 return 0;
107
67778e88 108 return max_blocks;
2ba1c290
JA
109}
110
10ba535a
JA
111/*
112 * Return the next free block in the map.
113 */
114static int get_next_free_block(struct thread_data *td, struct fio_file *f,
4ba66134 115 enum fio_ddir ddir, unsigned long long *b)
10ba535a 116{
84422acd 117 unsigned long long min_bs = td->o.rw_min_bs;
10ba535a
JA
118 int i;
119
c685b5b2
JA
120 i = f->last_free_lookup;
121 *b = (i * BLOCKS_PER_MAP);
84422acd 122 while ((*b) * min_bs < f->real_file_size) {
de605666 123 if (f->file_map[i] != (unsigned int) -1) {
697a606c 124 *b += ffz(f->file_map[i]);
4ba66134 125 if (*b > last_block(td, f, ddir))
2ba1c290 126 break;
c685b5b2 127 f->last_free_lookup = i;
10ba535a
JA
128 return 0;
129 }
130
131 *b += BLOCKS_PER_MAP;
132 i++;
133 }
134
2ba1c290 135 dprint(FD_IO, "failed finding a free block\n");
10ba535a
JA
136 return 1;
137}
138
ec4015da 139static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
4ba66134 140 enum fio_ddir ddir, unsigned long long *b)
ec4015da 141{
84422acd 142 unsigned long long r;
ec4015da
JA
143 int loops = 5;
144
145 do {
146 r = os_random_long(&td->random_state);
84422acd 147 dprint(FD_RANDOM, "off rand %llu\n", r);
5ec10eaa
JA
148 *b = (last_block(td, f, ddir) - 1)
149 * (r / ((unsigned long long) RAND_MAX + 1.0));
2ba1c290 150
43c63a78
JA
151 /*
152 * if we are not maintaining a random map, we are done.
153 */
303032ae 154 if (!file_randommap(td, f))
43c63a78
JA
155 return 0;
156
157 /*
84422acd 158 * calculate map offset and check if it's free
43c63a78 159 */
aec2de20 160 if (random_map_free(f, *b))
43c63a78
JA
161 return 0;
162
84422acd
JA
163 dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
164 *b);
43c63a78 165 } while (--loops);
ec4015da
JA
166
167 /*
43c63a78
JA
168 * we get here, if we didn't suceed in looking up a block. generate
169 * a random start offset into the filemap, and find the first free
170 * block from there.
ec4015da 171 */
43c63a78
JA
172 loops = 10;
173 do {
174 f->last_free_lookup = (f->num_maps - 1) * (r / (RAND_MAX+1.0));
4ba66134 175 if (!get_next_free_block(td, f, ddir, b))
43c63a78 176 return 0;
ec4015da 177
43c63a78
JA
178 r = os_random_long(&td->random_state);
179 } while (--loops);
180
181 /*
182 * that didn't work either, try exhaustive search from the start
183 */
184 f->last_free_lookup = 0;
4ba66134 185 return get_next_free_block(td, f, ddir, b);
ec4015da
JA
186}
187
10ba535a
JA
188/*
189 * For random io, generate a random new block and see if it's used. Repeat
190 * until we find a free one. For sequential io, just return the end of
191 * the last io issued.
192 */
9bf2061e 193static int get_next_offset(struct thread_data *td, struct io_u *io_u)
10ba535a 194{
9bf2061e 195 struct fio_file *f = io_u->file;
ec4015da 196 unsigned long long b;
4ba66134 197 enum fio_ddir ddir = io_u->ddir;
10ba535a 198
ec4015da
JA
199 if (td_random(td) && (td->o.ddir_nr && !--td->ddir_nr)) {
200 td->ddir_nr = td->o.ddir_nr;
211097b2 201
4ba66134 202 if (get_next_rand_offset(td, f, ddir, &b))
bca4ed4d 203 return 1;
43063a1c 204 } else {
0c3d768a 205 if (f->last_pos >= f->real_file_size) {
4ba66134
JA
206 if (!td_random(td) ||
207 get_next_rand_offset(td, f, ddir, &b))
0c3d768a 208 return 1;
bcdedd0a 209 } else
4ba66134 210 b = (f->last_pos - f->file_offset) / td->o.min_bs[ddir];
43063a1c 211 }
10ba535a 212
009bd847
JA
213 io_u->offset = b * td->o.min_bs[ddir];
214 if (io_u->offset >= f->io_size) {
215 dprint(FD_IO, "get_next_offset: offset %llu >= io_size %llu\n",
216 io_u->offset, f->io_size);
217 return 1;
218 }
219
220 io_u->offset += f->file_offset;
2ba1c290
JA
221 if (io_u->offset >= f->real_file_size) {
222 dprint(FD_IO, "get_next_offset: offset %llu >= size %llu\n",
223 io_u->offset, f->real_file_size);
10ba535a 224 return 1;
2ba1c290 225 }
10ba535a
JA
226
227 return 0;
228}
229
9bf2061e 230static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u)
10ba535a 231{
bca4ed4d 232 const int ddir = io_u->ddir;
fc4398fd 233 unsigned int buflen = buflen; /* silence dumb gcc warning */
10ba535a
JA
234 long r;
235
2dc1bbeb
JA
236 if (td->o.min_bs[ddir] == td->o.max_bs[ddir])
237 buflen = td->o.min_bs[ddir];
10ba535a
JA
238 else {
239 r = os_random_long(&td->bsrange_state);
5ec10eaa
JA
240 if (!td->o.bssplit_nr) {
241 buflen = (unsigned int)
242 (1 + (double) (td->o.max_bs[ddir] - 1)
243 * r / (RAND_MAX + 1.0));
244 } else {
564ca972
JA
245 long perc = 0;
246 unsigned int i;
247
248 for (i = 0; i < td->o.bssplit_nr; i++) {
249 struct bssplit *bsp = &td->o.bssplit[i];
250
251 buflen = bsp->bs;
252 perc += bsp->perc;
253 if (r <= ((LONG_MAX / 100L) * perc))
254 break;
255 }
256 }
5ec10eaa
JA
257 if (!td->o.bs_unaligned) {
258 buflen = (buflen + td->o.min_bs[ddir] - 1)
259 & ~(td->o.min_bs[ddir] - 1);
260 }
10ba535a
JA
261 }
262
4ba66134
JA
263 if (io_u->offset + buflen > io_u->file->real_file_size) {
264 dprint(FD_IO, "lower buflen %u -> %u (ddir=%d)\n", buflen,
265 td->o.min_bs[ddir], ddir);
6a5e6884 266 buflen = td->o.min_bs[ddir];
4ba66134 267 }
6a5e6884 268
10ba535a
JA
269 return buflen;
270}
271
afe24a5a
JA
272static void set_rwmix_bytes(struct thread_data *td)
273{
afe24a5a
JA
274 unsigned int diff;
275
276 /*
277 * we do time or byte based switch. this is needed because
278 * buffered writes may issue a lot quicker than they complete,
279 * whereas reads do not.
280 */
e47f799f 281 diff = td->o.rwmix[td->rwmix_ddir ^ 1];
04c540d9 282 td->rwmix_issues = (td->io_issues[td->rwmix_ddir] * diff) / 100;
e47f799f
JA
283}
284
285static inline enum fio_ddir get_rand_ddir(struct thread_data *td)
286{
287 unsigned int v;
288 long r;
289
290 r = os_random_long(&td->rwmix_state);
291 v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0)));
04c540d9 292 if (v <= td->o.rwmix[DDIR_READ])
e47f799f
JA
293 return DDIR_READ;
294
295 return DDIR_WRITE;
afe24a5a
JA
296}
297
10ba535a
JA
298/*
299 * Return the data direction for the next io_u. If the job is a
300 * mixed read/write workload, check the rwmix cycle and switch if
301 * necessary.
302 */
1e97cce9 303static enum fio_ddir get_rw_ddir(struct thread_data *td)
10ba535a
JA
304{
305 if (td_rw(td)) {
10ba535a
JA
306 /*
307 * Check if it's time to seed a new data direction.
308 */
e4928662 309 if (td->io_issues[td->rwmix_ddir] >= td->rwmix_issues) {
e47f799f 310 unsigned long long max_bytes;
5ec10eaa 311 enum fio_ddir ddir;
e47f799f
JA
312
313 /*
314 * Put a top limit on how many bytes we do for
315 * one data direction, to avoid overflowing the
316 * ranges too much
317 */
318 ddir = get_rand_ddir(td);
319 max_bytes = td->this_io_bytes[ddir];
5ec10eaa
JA
320 if (max_bytes >=
321 (td->o.size * td->o.rwmix[ddir] / 100)) {
6162f524 322 if (!td->rw_end_set[ddir]) {
38d77cae 323 td->rw_end_set[ddir] = 1;
6162f524
JA
324 fio_gettime(&td->rw_end[ddir], NULL);
325 }
e4928662 326
e47f799f 327 ddir ^= 1;
38d77cae 328 }
e47f799f
JA
329
330 if (ddir != td->rwmix_ddir)
331 set_rwmix_bytes(td);
332
333 td->rwmix_ddir = ddir;
10ba535a
JA
334 }
335 return td->rwmix_ddir;
336 } else if (td_read(td))
337 return DDIR_READ;
338 else
339 return DDIR_WRITE;
340}
341
60f2c658
JA
342static void put_file_log(struct thread_data *td, struct fio_file *f)
343{
344 int ret = put_file(td, f);
345
346 if (ret)
347 td_verror(td, ret, "file close");
348}
349
10ba535a
JA
350void put_io_u(struct thread_data *td, struct io_u *io_u)
351{
0c6e7517
JA
352 assert((io_u->flags & IO_U_F_FREE) == 0);
353 io_u->flags |= IO_U_F_FREE;
354
60f2c658
JA
355 if (io_u->file)
356 put_file_log(td, io_u->file);
2dbdab7e 357
10ba535a 358 io_u->file = NULL;
01743ee1
JA
359 flist_del(&io_u->list);
360 flist_add(&io_u->list, &td->io_u_freelist);
10ba535a
JA
361 td->cur_depth--;
362}
363
755200a3
JA
364void requeue_io_u(struct thread_data *td, struct io_u **io_u)
365{
366 struct io_u *__io_u = *io_u;
367
465221b0
JA
368 dprint(FD_IO, "requeue %p\n", __io_u);
369
4d2e0f49 370 __io_u->flags |= IO_U_F_FREE;
e4f54adb
JA
371 if ((__io_u->flags & IO_U_F_FLIGHT) && (__io_u->ddir != DDIR_SYNC))
372 td->io_issues[__io_u->ddir]--;
5ec10eaa 373
4d2e0f49
JA
374 __io_u->flags &= ~IO_U_F_FLIGHT;
375
01743ee1
JA
376 flist_del(&__io_u->list);
377 flist_add_tail(&__io_u->list, &td->io_u_requeues);
755200a3
JA
378 td->cur_depth--;
379 *io_u = NULL;
380}
381
9bf2061e 382static int fill_io_u(struct thread_data *td, struct io_u *io_u)
10ba535a 383{
b4c5e1ac
JA
384 if (td->io_ops->flags & FIO_NOIO)
385 goto out;
386
87dc1ab1
JA
387 /*
388 * see if it's time to sync
389 */
2dc1bbeb
JA
390 if (td->o.fsync_blocks &&
391 !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) &&
392 td->io_issues[DDIR_WRITE] && should_fsync(td)) {
87dc1ab1 393 io_u->ddir = DDIR_SYNC;
c38e9468 394 goto out;
87dc1ab1
JA
395 }
396
a00735e6
JA
397 io_u->ddir = get_rw_ddir(td);
398
48f5abd3
JA
399 /*
400 * See if it's time to switch to a new zone
401 */
402 if (td->zone_bytes >= td->o.zone_size) {
403 td->zone_bytes = 0;
404 io_u->file->last_pos += td->o.zone_skip;
405 td->io_skip_bytes += td->o.zone_skip;
406 }
407
10ba535a 408 /*
c685b5b2
JA
409 * No log, let the seq/rand engine retrieve the next buflen and
410 * position.
10ba535a 411 */
2ba1c290
JA
412 if (get_next_offset(td, io_u)) {
413 dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
bca4ed4d 414 return 1;
2ba1c290 415 }
10ba535a 416
9bf2061e 417 io_u->buflen = get_next_buflen(td, io_u);
2ba1c290
JA
418 if (!io_u->buflen) {
419 dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
bca4ed4d 420 return 1;
2ba1c290 421 }
bca4ed4d 422
2ba1c290
JA
423 if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
424 dprint(FD_IO, "io_u %p, offset too large\n", io_u);
4ba66134
JA
425 dprint(FD_IO, " off=%llu/%lu > %llu\n", io_u->offset,
426 io_u->buflen, io_u->file->real_file_size);
6a5e6884 427 return 1;
2ba1c290 428 }
6a5e6884 429
bca4ed4d
JA
430 /*
431 * mark entry before potentially trimming io_u
432 */
303032ae 433 if (td_random(td) && file_randommap(td, io_u->file))
9bf2061e 434 mark_random_map(td, io_u);
bca4ed4d
JA
435
436 /*
437 * If using a write iolog, store this entry.
438 */
c38e9468 439out:
2ba1c290 440 dprint_io_u(io_u, "fill_io_u");
d9d91e39 441 td->zone_bytes += io_u->buflen;
f29b25a3 442 log_io_u(td, io_u);
bca4ed4d 443 return 0;
10ba535a
JA
444}
445
838bc709
JA
446static void __io_u_mark_map(unsigned int *map, unsigned int nr)
447{
448 int index = 0;
449
450 switch (nr) {
451 default:
452 index = 6;
453 break;
454 case 33 ... 64:
455 index = 5;
456 break;
457 case 17 ... 32:
458 index = 4;
459 break;
460 case 9 ... 16:
461 index = 3;
462 break;
463 case 5 ... 8:
464 index = 2;
465 break;
466 case 1 ... 4:
467 index = 1;
468 case 0:
469 break;
470 }
471
472 map[index]++;
473}
474
475void io_u_mark_submit(struct thread_data *td, unsigned int nr)
476{
477 __io_u_mark_map(td->ts.io_u_submit, nr);
478 td->ts.total_submit++;
479}
480
481void io_u_mark_complete(struct thread_data *td, unsigned int nr)
482{
483 __io_u_mark_map(td->ts.io_u_complete, nr);
484 td->ts.total_complete++;
485}
486
d8005759 487void io_u_mark_depth(struct thread_data *td, unsigned int nr)
71619dc2
JA
488{
489 int index = 0;
490
491 switch (td->cur_depth) {
492 default:
a783e61a
JA
493 index = 6;
494 break;
71619dc2 495 case 32 ... 63:
a783e61a
JA
496 index = 5;
497 break;
71619dc2 498 case 16 ... 31:
a783e61a
JA
499 index = 4;
500 break;
71619dc2 501 case 8 ... 15:
a783e61a
JA
502 index = 3;
503 break;
71619dc2 504 case 4 ... 7:
a783e61a
JA
505 index = 2;
506 break;
71619dc2 507 case 2 ... 3:
a783e61a 508 index = 1;
71619dc2
JA
509 case 1:
510 break;
511 }
512
3bec7ae0 513 td->ts.io_u_map[index] += nr;
71619dc2
JA
514}
515
04a0feae
JA
516static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec)
517{
518 int index = 0;
519
520 assert(usec < 1000);
521
522 switch (usec) {
523 case 750 ... 999:
524 index = 9;
525 break;
526 case 500 ... 749:
527 index = 8;
528 break;
529 case 250 ... 499:
530 index = 7;
531 break;
532 case 100 ... 249:
533 index = 6;
534 break;
535 case 50 ... 99:
536 index = 5;
537 break;
538 case 20 ... 49:
539 index = 4;
540 break;
541 case 10 ... 19:
542 index = 3;
543 break;
544 case 4 ... 9:
545 index = 2;
546 break;
547 case 2 ... 3:
548 index = 1;
549 case 0 ... 1:
550 break;
551 }
552
553 assert(index < FIO_IO_U_LAT_U_NR);
554 td->ts.io_u_lat_u[index]++;
555}
556
557static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec)
ec118304
JA
558{
559 int index = 0;
560
561 switch (msec) {
562 default:
04a0feae
JA
563 index = 11;
564 break;
8abdce66 565 case 1000 ... 1999:
04a0feae
JA
566 index = 10;
567 break;
8abdce66 568 case 750 ... 999:
04a0feae
JA
569 index = 9;
570 break;
8abdce66 571 case 500 ... 749:
04a0feae
JA
572 index = 8;
573 break;
8abdce66 574 case 250 ... 499:
04a0feae
JA
575 index = 7;
576 break;
8abdce66 577 case 100 ... 249:
04a0feae
JA
578 index = 6;
579 break;
8abdce66 580 case 50 ... 99:
04a0feae
JA
581 index = 5;
582 break;
8abdce66 583 case 20 ... 49:
04a0feae
JA
584 index = 4;
585 break;
8abdce66 586 case 10 ... 19:
04a0feae
JA
587 index = 3;
588 break;
8abdce66 589 case 4 ... 9:
04a0feae
JA
590 index = 2;
591 break;
ec118304 592 case 2 ... 3:
04a0feae 593 index = 1;
ec118304
JA
594 case 0 ... 1:
595 break;
596 }
597
04a0feae
JA
598 assert(index < FIO_IO_U_LAT_M_NR);
599 td->ts.io_u_lat_m[index]++;
600}
601
602static void io_u_mark_latency(struct thread_data *td, unsigned long usec)
603{
604 if (usec < 1000)
605 io_u_mark_lat_usec(td, usec);
606 else
607 io_u_mark_lat_msec(td, usec / 1000);
ec118304
JA
608}
609
0aabe160
JA
610/*
611 * Get next file to service by choosing one at random
612 */
1c178180
JA
613static struct fio_file *get_next_file_rand(struct thread_data *td, int goodf,
614 int badf)
0aabe160 615{
0aabe160 616 struct fio_file *f;
1c178180 617 int fno;
0aabe160
JA
618
619 do {
7c83c089
JA
620 long r = os_random_long(&td->next_file_state);
621
5ec10eaa
JA
622 fno = (unsigned int) ((double) td->o.nr_files
623 * (r / (RAND_MAX + 1.0)));
126d65c6 624 f = td->files[fno];
059e63c0
JA
625 if (f->flags & FIO_FILE_DONE)
626 continue;
1c178180 627
2ba1c290
JA
628 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) {
629 dprint(FD_FILE, "get_next_file_rand: %p\n", f);
0aabe160 630 return f;
2ba1c290 631 }
0aabe160
JA
632 } while (1);
633}
634
635/*
636 * Get next file to service by doing round robin between all available ones
637 */
1c178180
JA
638static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
639 int badf)
3d7c391d
JA
640{
641 unsigned int old_next_file = td->next_file;
642 struct fio_file *f;
643
644 do {
126d65c6 645 f = td->files[td->next_file];
3d7c391d
JA
646
647 td->next_file++;
2dc1bbeb 648 if (td->next_file >= td->o.nr_files)
3d7c391d
JA
649 td->next_file = 0;
650
d5ed68ea
JA
651 if (f->flags & FIO_FILE_DONE) {
652 f = NULL;
059e63c0 653 continue;
d5ed68ea 654 }
059e63c0 655
1c178180 656 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
3d7c391d
JA
657 break;
658
659 f = NULL;
660 } while (td->next_file != old_next_file);
661
2ba1c290 662 dprint(FD_FILE, "get_next_file_rr: %p\n", f);
3d7c391d
JA
663 return f;
664}
665
bdb4e2e9
JA
666static struct fio_file *get_next_file(struct thread_data *td)
667{
1907dbc6
JA
668 struct fio_file *f;
669
2dc1bbeb 670 assert(td->o.nr_files <= td->files_index);
1c178180 671
2ba1c290 672 if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files) {
5ec10eaa
JA
673 dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d,"
674 " nr_files=%d\n", td->nr_open_files,
675 td->nr_done_files,
676 td->o.nr_files);
bdb4e2e9 677 return NULL;
2ba1c290 678 }
bdb4e2e9 679
1907dbc6 680 f = td->file_service_file;
f11bd94d 681 if (f && (f->flags & FIO_FILE_OPEN) && td->file_service_left--)
2ba1c290 682 goto out;
1907dbc6 683
2dc1bbeb 684 if (td->o.file_service_type == FIO_FSERVICE_RR)
1c178180 685 f = get_next_file_rr(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
bdb4e2e9 686 else
1c178180 687 f = get_next_file_rand(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
1907dbc6
JA
688
689 td->file_service_file = f;
690 td->file_service_left = td->file_service_nr - 1;
2ba1c290
JA
691out:
692 dprint(FD_FILE, "get_next_file: %p\n", f);
1907dbc6 693 return f;
bdb4e2e9
JA
694}
695
1c178180
JA
696static struct fio_file *find_next_new_file(struct thread_data *td)
697{
698 struct fio_file *f;
699
1020a139
JA
700 if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files)
701 return NULL;
702
2dc1bbeb 703 if (td->o.file_service_type == FIO_FSERVICE_RR)
1c178180
JA
704 f = get_next_file_rr(td, 0, FIO_FILE_OPEN);
705 else
706 f = get_next_file_rand(td, 0, FIO_FILE_OPEN);
707
708 return f;
709}
710
429f6675
JA
711static int set_io_u_file(struct thread_data *td, struct io_u *io_u)
712{
713 struct fio_file *f;
714
715 do {
716 f = get_next_file(td);
717 if (!f)
718 return 1;
719
720set_file:
721 io_u->file = f;
722 get_file(f);
723
724 if (!fill_io_u(td, io_u))
725 break;
726
009bd847
JA
727 /*
728 * optimization to prevent close/open of the same file. This
729 * way we preserve queueing etc.
730 */
731 if (td->o.nr_files == 1 && td->o.time_based) {
60f2c658 732 put_file_log(td, f);
009bd847
JA
733 fio_file_reset(f);
734 goto set_file;
735 }
736
429f6675
JA
737 /*
738 * td_io_close() does a put_file() as well, so no need to
739 * do that here.
740 */
741 io_u->file = NULL;
742 td_io_close_file(td, f);
743 f->flags |= FIO_FILE_DONE;
744 td->nr_done_files++;
745
746 /*
747 * probably not the right place to do this, but see
748 * if we need to open a new file
749 */
750 if (td->nr_open_files < td->o.open_files &&
751 td->o.open_files != td->o.nr_files) {
752 f = find_next_new_file(td);
753
754 if (!f || td_io_open_file(td, f))
755 return 1;
756
757 goto set_file;
758 }
759 } while (1);
760
761 return 0;
762}
763
764
10ba535a
JA
765struct io_u *__get_io_u(struct thread_data *td)
766{
767 struct io_u *io_u = NULL;
768
01743ee1
JA
769 if (!flist_empty(&td->io_u_requeues))
770 io_u = flist_entry(td->io_u_requeues.next, struct io_u, list);
755200a3 771 else if (!queue_full(td)) {
01743ee1 772 io_u = flist_entry(td->io_u_freelist.next, struct io_u, list);
10ba535a 773
6040dabc 774 io_u->buflen = 0;
10ba535a 775 io_u->resid = 0;
755200a3 776 io_u->file = NULL;
d7762cf8 777 io_u->end_io = NULL;
755200a3
JA
778 }
779
780 if (io_u) {
0c6e7517
JA
781 assert(io_u->flags & IO_U_F_FREE);
782 io_u->flags &= ~IO_U_F_FREE;
783
755200a3 784 io_u->error = 0;
01743ee1
JA
785 flist_del(&io_u->list);
786 flist_add(&io_u->list, &td->io_u_busylist);
10ba535a
JA
787 td->cur_depth++;
788 }
789
790 return io_u;
791}
792
793/*
794 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
795 * etc. The returned io_u is fully ready to be prepped and submitted.
796 */
3d7c391d 797struct io_u *get_io_u(struct thread_data *td)
10ba535a 798{
3d7c391d 799 struct fio_file *f;
10ba535a
JA
800 struct io_u *io_u;
801
802 io_u = __get_io_u(td);
2ba1c290
JA
803 if (!io_u) {
804 dprint(FD_IO, "__get_io_u failed\n");
10ba535a 805 return NULL;
2ba1c290 806 }
10ba535a 807
755200a3
JA
808 /*
809 * from a requeue, io_u already setup
810 */
811 if (io_u->file)
77f392bf 812 goto out;
755200a3 813
429f6675
JA
814 /*
815 * If using an iolog, grab next piece if any available.
816 */
817 if (td->o.read_iolog_file) {
818 if (read_iolog_get(td, io_u))
819 goto err_put;
2ba1c290
JA
820 } else if (set_io_u_file(td, io_u)) {
821 dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
429f6675 822 goto err_put;
2ba1c290 823 }
5ec10eaa 824
429f6675
JA
825 f = io_u->file;
826 assert(f->flags & FIO_FILE_OPEN);
97af62ce 827
87dc1ab1 828 if (io_u->ddir != DDIR_SYNC) {
d0656a93 829 if (!io_u->buflen && !(td->io_ops->flags & FIO_NOIO)) {
2ba1c290 830 dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u);
429f6675 831 goto err_put;
2ba1c290 832 }
10ba535a 833
36167d82 834 f->last_pos = io_u->offset + io_u->buflen;
10ba535a 835
2dc1bbeb 836 if (td->o.verify != VERIFY_NONE)
87dc1ab1 837 populate_verify_io_u(td, io_u);
e4dad9c6
JA
838 else if (td->o.refill_buffers && io_u->ddir == DDIR_WRITE)
839 io_u_fill_buffer(td, io_u, io_u->xfer_buflen);
87dc1ab1 840 }
10ba535a 841
165faf16
JA
842 /*
843 * Set io data pointers.
844 */
d460eb31 845 io_u->endpos = io_u->offset + io_u->buflen;
cec6b55d
JA
846 io_u->xfer_buf = io_u->buf;
847 io_u->xfer_buflen = io_u->buflen;
5973cafb 848
6ac7a331 849out:
429f6675
JA
850 if (!td_io_prep(td, io_u)) {
851 fio_gettime(&io_u->start_time, NULL);
852 return io_u;
36167d82 853 }
429f6675 854err_put:
2ba1c290 855 dprint(FD_IO, "get_io_u failed\n");
429f6675
JA
856 put_io_u(td, io_u);
857 return NULL;
10ba535a
JA
858}
859
5451792e
JA
860void io_u_log_error(struct thread_data *td, struct io_u *io_u)
861{
862 const char *msg[] = { "read", "write", "sync" };
863
864 log_err("fio: io_u error");
865
866 if (io_u->file)
867 log_err(" on file %s", io_u->file->file_name);
868
869 log_err(": %s\n", strerror(io_u->error));
870
5ec10eaa
JA
871 log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir],
872 io_u->offset, io_u->xfer_buflen);
5451792e
JA
873
874 if (!td->error)
875 td_verror(td, io_u->error, "io_u error");
876}
877
97601024
JA
878static void io_completed(struct thread_data *td, struct io_u *io_u,
879 struct io_completion_data *icd)
10ba535a 880{
d85f5118 881 unsigned long usec;
10ba535a 882
2ba1c290
JA
883 dprint_io_u(io_u, "io complete");
884
0c6e7517
JA
885 assert(io_u->flags & IO_U_F_FLIGHT);
886 io_u->flags &= ~IO_U_F_FLIGHT;
887
87dc1ab1
JA
888 if (io_u->ddir == DDIR_SYNC) {
889 td->last_was_sync = 1;
890 return;
891 }
892
893 td->last_was_sync = 0;
894
10ba535a
JA
895 if (!io_u->error) {
896 unsigned int bytes = io_u->buflen - io_u->resid;
1e97cce9 897 const enum fio_ddir idx = io_u->ddir;
3af6ef39 898 int ret;
10ba535a
JA
899
900 td->io_blocks[idx]++;
901 td->io_bytes[idx] += bytes;
10ba535a
JA
902 td->this_io_bytes[idx] += bytes;
903
d85f5118 904 usec = utime_since(&io_u->issue_time, &icd->time);
10ba535a 905
d85f5118 906 add_clat_sample(td, idx, usec);
02bcaa8c 907 add_bw_sample(td, idx, &icd->time);
04a0feae 908 io_u_mark_latency(td, usec);
10ba535a 909
660a1cb5 910 if (td_write(td) && idx == DDIR_WRITE &&
8670579e 911 td->o.do_verify &&
41128405 912 td->o.verify != VERIFY_NONE)
10ba535a
JA
913 log_io_piece(td, io_u);
914
915 icd->bytes_done[idx] += bytes;
3af6ef39 916
d7762cf8 917 if (io_u->end_io) {
36690c9b 918 ret = io_u->end_io(td, io_u);
3af6ef39
JA
919 if (ret && !icd->error)
920 icd->error = ret;
921 }
5451792e 922 } else {
10ba535a 923 icd->error = io_u->error;
5451792e
JA
924 io_u_log_error(td, io_u);
925 }
10ba535a
JA
926}
927
d7762cf8 928static void init_icd(struct io_completion_data *icd, int nr)
10ba535a 929{
02bcaa8c
JA
930 fio_gettime(&icd->time, NULL);
931
3af6ef39
JA
932 icd->nr = nr;
933
10ba535a
JA
934 icd->error = 0;
935 icd->bytes_done[0] = icd->bytes_done[1] = 0;
36167d82
JA
936}
937
97601024
JA
938static void ios_completed(struct thread_data *td,
939 struct io_completion_data *icd)
36167d82
JA
940{
941 struct io_u *io_u;
942 int i;
943
10ba535a
JA
944 for (i = 0; i < icd->nr; i++) {
945 io_u = td->io_ops->event(td, i);
946
947 io_completed(td, io_u, icd);
948 put_io_u(td, io_u);
949 }
950}
97601024 951
e7e6cfb4
JA
952/*
953 * Complete a single io_u for the sync engines.
954 */
d7762cf8 955long io_u_sync_complete(struct thread_data *td, struct io_u *io_u)
97601024
JA
956{
957 struct io_completion_data icd;
958
d7762cf8 959 init_icd(&icd, 1);
97601024
JA
960 io_completed(td, io_u, &icd);
961 put_io_u(td, io_u);
962
963 if (!icd.error)
964 return icd.bytes_done[0] + icd.bytes_done[1];
965
37e974a5 966 td_verror(td, icd.error, "io_u_sync_complete");
97601024
JA
967 return -1;
968}
969
e7e6cfb4
JA
970/*
971 * Called to complete min_events number of io for the async engines.
972 */
d7762cf8 973long io_u_queued_complete(struct thread_data *td, int min_events)
97601024 974{
97601024 975 struct io_completion_data icd;
00de55ef 976 struct timespec *tvp = NULL;
97601024 977 int ret;
4d06a338 978 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
97601024 979
b271fe62
JA
980 dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_events);
981
4d06a338 982 if (!min_events)
00de55ef 983 tvp = &ts;
97601024 984
00de55ef 985 ret = td_io_getevents(td, min_events, td->cur_depth, tvp);
97601024 986 if (ret < 0) {
e1161c32 987 td_verror(td, -ret, "td_io_getevents");
97601024
JA
988 return ret;
989 } else if (!ret)
990 return ret;
991
d7762cf8 992 init_icd(&icd, ret);
97601024
JA
993 ios_completed(td, &icd);
994 if (!icd.error)
995 return icd.bytes_done[0] + icd.bytes_done[1];
996
37e974a5 997 td_verror(td, icd.error, "io_u_queued_complete");
97601024
JA
998 return -1;
999}
7e77dd02
JA
1000
1001/*
1002 * Call when io_u is really queued, to update the submission latency.
1003 */
1004void io_u_queued(struct thread_data *td, struct io_u *io_u)
1005{
1006 unsigned long slat_time;
1007
d85f5118 1008 slat_time = utime_since(&io_u->start_time, &io_u->issue_time);
7e77dd02
JA
1009 add_slat_sample(td, io_u->ddir, slat_time);
1010}
433afcb4 1011
5973cafb
JA
1012/*
1013 * "randomly" fill the buffer contents
1014 */
1015void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
1016 unsigned int max_bs)
1017{
1018 long *ptr = io_u->buf;
1019
1020 if (!td->o.zero_buffers) {
1021 while ((void *) ptr - io_u->buf < max_bs) {
1022 *ptr = rand() * GOLDEN_RATIO_PRIME;
1023 ptr++;
1024 }
1025 } else
1026 memset(ptr, 0, max_bs);
1027}
1028
55bc9728 1029#ifdef FIO_USE_TIMEOUT
433afcb4
JA
1030void io_u_set_timeout(struct thread_data *td)
1031{
1032 assert(td->cur_depth);
1033
1034 td->timer.it_interval.tv_sec = 0;
1035 td->timer.it_interval.tv_usec = 0;
1036 td->timer.it_value.tv_sec = IO_U_TIMEOUT + IO_U_TIMEOUT_INC;
1037 td->timer.it_value.tv_usec = 0;
1038 setitimer(ITIMER_REAL, &td->timer, NULL);
1039 fio_gettime(&td->timeout_end, NULL);
1040}
5945b9b4
JA
1041
1042static void io_u_dump(struct io_u *io_u)
1043{
1044 unsigned long t_start = mtime_since_now(&io_u->start_time);
1045 unsigned long t_issue = mtime_since_now(&io_u->issue_time);
1046
1047 log_err("io_u=%p, t_start=%lu, t_issue=%lu\n", io_u, t_start, t_issue);
5ec10eaa
JA
1048 log_err(" buf=%p/%p, len=%lu/%lu, offset=%llu\n", io_u->buf,
1049 io_u->xfer_buf, io_u->buflen,
1050 io_u->xfer_buflen,
1051 io_u->offset);
5945b9b4
JA
1052 log_err(" ddir=%d, fname=%s\n", io_u->ddir, io_u->file->file_name);
1053}
55bc9728
JA
1054#else
1055void io_u_set_timeout(struct thread_data fio_unused *td)
1056{
1057}
1058#endif
433afcb4 1059
55bc9728 1060#ifdef FIO_USE_TIMEOUT
433afcb4
JA
1061static void io_u_timeout_handler(int fio_unused sig)
1062{
1063 struct thread_data *td, *__td;
1064 pid_t pid = getpid();
01743ee1 1065 struct flist_head *entry;
5945b9b4 1066 struct io_u *io_u;
433afcb4
JA
1067 int i;
1068
1069 log_err("fio: io_u timeout\n");
1070
1071 /*
1072 * TLS would be nice...
1073 */
1074 td = NULL;
1075 for_each_td(__td, i) {
1076 if (__td->pid == pid) {
1077 td = __td;
1078 break;
1079 }
1080 }
1081
1082 if (!td) {
1083 log_err("fio: io_u timeout, can't find job\n");
1084 exit(1);
1085 }
1086
1087 if (!td->cur_depth) {
1088 log_err("fio: timeout without pending work?\n");
1089 return;
1090 }
1091
15506d09 1092 log_err("fio: io_u timeout: job=%s, pid=%d\n", td->o.name, td->pid);
5945b9b4 1093
01743ee1
JA
1094 flist_for_each(entry, &td->io_u_busylist) {
1095 io_u = flist_entry(entry, struct io_u, list);
5945b9b4
JA
1096
1097 io_u_dump(io_u);
1098 }
1099
1100 td_verror(td, ETIMEDOUT, "io_u timeout");
433afcb4
JA
1101 exit(1);
1102}
55bc9728 1103#endif
433afcb4
JA
1104
1105void io_u_init_timeout(void)
1106{
55bc9728 1107#ifdef FIO_USE_TIMEOUT
433afcb4 1108 signal(SIGALRM, io_u_timeout_handler);
55bc9728 1109#endif
433afcb4 1110}