Allow randommap allocation to fail
[fio.git] / io_u.c
CommitLineData
10ba535a
JA
1#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <signal.h>
5#include <time.h>
0c6e7517 6#include <assert.h>
10ba535a
JA
7
8#include "fio.h"
10ba535a 9
5945b9b4
JA
10/*
11 * Change this define to play with the timeout handling
12 */
13#undef FIO_USE_TIMEOUT
14
97601024
JA
15struct io_completion_data {
16 int nr; /* input */
97601024
JA
17
18 int error; /* output */
19 unsigned long bytes_done[2]; /* output */
20 struct timeval time; /* output */
21};
22
10ba535a
JA
23/*
24 * The ->file_map[] contains a map of blocks we have or have not done io
25 * to yet. Used to make sure we cover the entire range in a fair fashion.
26 */
27static int random_map_free(struct thread_data *td, struct fio_file *f,
84422acd 28 const unsigned long long block)
10ba535a
JA
29{
30 unsigned int idx = RAND_MAP_IDX(td, f, block);
31 unsigned int bit = RAND_MAP_BIT(td, f, block);
32
84422acd
JA
33 dprint(FD_RANDOM, "free: b=%llu, idx=%u, bit=%u\n", block, idx, bit);
34
10ba535a
JA
35 return (f->file_map[idx] & (1UL << bit)) == 0;
36}
37
df415585
JA
38/*
39 * Mark a given offset as used in the map.
40 */
9bf2061e 41static void mark_random_map(struct thread_data *td, struct io_u *io_u)
df415585 42{
2dc1bbeb 43 unsigned int min_bs = td->o.rw_min_bs;
9bf2061e 44 struct fio_file *f = io_u->file;
a00735e6
JA
45 unsigned long long block;
46 unsigned int blocks;
c685b5b2 47 unsigned int nr_blocks;
df415585 48
b9c5b644 49 block = (io_u->offset - f->file_offset) / (unsigned long long) min_bs;
a00735e6 50 blocks = 0;
c685b5b2
JA
51 nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
52
53 while (blocks < nr_blocks) {
df415585
JA
54 unsigned int idx, bit;
55
1e3d53ac
JA
56 /*
57 * If we have a mixed random workload, we may
58 * encounter blocks we already did IO to.
59 */
51b6996e 60 if ((td->o.ddir_nr == 1) && !random_map_free(td, f, block))
df415585
JA
61 break;
62
63 idx = RAND_MAP_IDX(td, f, block);
64 bit = RAND_MAP_BIT(td, f, block);
65
0032bf9f 66 fio_assert(td, idx < f->num_maps);
df415585
JA
67
68 f->file_map[idx] |= (1UL << bit);
69 block++;
70 blocks++;
71 }
72
a00735e6
JA
73 if ((blocks * min_bs) < io_u->buflen)
74 io_u->buflen = blocks * min_bs;
df415585
JA
75}
76
2ba1c290 77static inline unsigned long long last_block(struct thread_data *td,
4ba66134
JA
78 struct fio_file *f,
79 enum fio_ddir ddir)
2ba1c290
JA
80{
81 unsigned long long max_blocks;
82
84422acd 83 max_blocks = f->io_size / (unsigned long long) td->o.min_bs[ddir];
2ba1c290
JA
84 if (!max_blocks)
85 return 0;
86
87 return max_blocks - 1;
88}
89
10ba535a
JA
90/*
91 * Return the next free block in the map.
92 */
93static int get_next_free_block(struct thread_data *td, struct fio_file *f,
4ba66134 94 enum fio_ddir ddir, unsigned long long *b)
10ba535a 95{
84422acd 96 unsigned long long min_bs = td->o.rw_min_bs;
10ba535a
JA
97 int i;
98
c685b5b2
JA
99 i = f->last_free_lookup;
100 *b = (i * BLOCKS_PER_MAP);
84422acd 101 while ((*b) * min_bs < f->real_file_size) {
10ba535a 102 if (f->file_map[i] != -1UL) {
b12ebc65 103 *b += fio_ffz(f->file_map[i]);
4ba66134 104 if (*b > last_block(td, f, ddir))
2ba1c290 105 break;
c685b5b2 106 f->last_free_lookup = i;
10ba535a
JA
107 return 0;
108 }
109
110 *b += BLOCKS_PER_MAP;
111 i++;
112 }
113
2ba1c290 114 dprint(FD_IO, "failed finding a free block\n");
10ba535a
JA
115 return 1;
116}
117
ec4015da 118static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
4ba66134 119 enum fio_ddir ddir, unsigned long long *b)
ec4015da 120{
84422acd 121 unsigned long long r;
ec4015da
JA
122 int loops = 5;
123
124 do {
125 r = os_random_long(&td->random_state);
84422acd 126 dprint(FD_RANDOM, "off rand %llu\n", r);
5ec10eaa
JA
127 *b = (last_block(td, f, ddir) - 1)
128 * (r / ((unsigned long long) RAND_MAX + 1.0));
2ba1c290 129
43c63a78
JA
130 /*
131 * if we are not maintaining a random map, we are done.
132 */
303032ae 133 if (!file_randommap(td, f))
43c63a78
JA
134 return 0;
135
136 /*
84422acd 137 * calculate map offset and check if it's free
43c63a78 138 */
84422acd 139 if (random_map_free(td, f, *b))
43c63a78
JA
140 return 0;
141
84422acd
JA
142 dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
143 *b);
43c63a78 144 } while (--loops);
ec4015da
JA
145
146 /*
43c63a78
JA
147 * we get here, if we didn't suceed in looking up a block. generate
148 * a random start offset into the filemap, and find the first free
149 * block from there.
ec4015da 150 */
43c63a78
JA
151 loops = 10;
152 do {
153 f->last_free_lookup = (f->num_maps - 1) * (r / (RAND_MAX+1.0));
4ba66134 154 if (!get_next_free_block(td, f, ddir, b))
43c63a78 155 return 0;
ec4015da 156
43c63a78
JA
157 r = os_random_long(&td->random_state);
158 } while (--loops);
159
160 /*
161 * that didn't work either, try exhaustive search from the start
162 */
163 f->last_free_lookup = 0;
4ba66134 164 return get_next_free_block(td, f, ddir, b);
ec4015da
JA
165}
166
10ba535a
JA
167/*
168 * For random io, generate a random new block and see if it's used. Repeat
169 * until we find a free one. For sequential io, just return the end of
170 * the last io issued.
171 */
9bf2061e 172static int get_next_offset(struct thread_data *td, struct io_u *io_u)
10ba535a 173{
9bf2061e 174 struct fio_file *f = io_u->file;
ec4015da 175 unsigned long long b;
4ba66134 176 enum fio_ddir ddir = io_u->ddir;
10ba535a 177
ec4015da
JA
178 if (td_random(td) && (td->o.ddir_nr && !--td->ddir_nr)) {
179 td->ddir_nr = td->o.ddir_nr;
211097b2 180
4ba66134 181 if (get_next_rand_offset(td, f, ddir, &b))
bca4ed4d 182 return 1;
43063a1c 183 } else {
0c3d768a 184 if (f->last_pos >= f->real_file_size) {
4ba66134
JA
185 if (!td_random(td) ||
186 get_next_rand_offset(td, f, ddir, &b))
0c3d768a 187 return 1;
bcdedd0a 188 } else
4ba66134 189 b = (f->last_pos - f->file_offset) / td->o.min_bs[ddir];
43063a1c 190 }
10ba535a 191
4ba66134 192 io_u->offset = (b * td->o.min_bs[ddir]) + f->file_offset;
2ba1c290
JA
193 if (io_u->offset >= f->real_file_size) {
194 dprint(FD_IO, "get_next_offset: offset %llu >= size %llu\n",
195 io_u->offset, f->real_file_size);
10ba535a 196 return 1;
2ba1c290 197 }
10ba535a
JA
198
199 return 0;
200}
201
9bf2061e 202static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u)
10ba535a 203{
bca4ed4d 204 const int ddir = io_u->ddir;
10ba535a
JA
205 unsigned int buflen;
206 long r;
207
2dc1bbeb
JA
208 if (td->o.min_bs[ddir] == td->o.max_bs[ddir])
209 buflen = td->o.min_bs[ddir];
10ba535a
JA
210 else {
211 r = os_random_long(&td->bsrange_state);
5ec10eaa
JA
212 if (!td->o.bssplit_nr) {
213 buflen = (unsigned int)
214 (1 + (double) (td->o.max_bs[ddir] - 1)
215 * r / (RAND_MAX + 1.0));
216 } else {
564ca972
JA
217 long perc = 0;
218 unsigned int i;
219
220 for (i = 0; i < td->o.bssplit_nr; i++) {
221 struct bssplit *bsp = &td->o.bssplit[i];
222
223 buflen = bsp->bs;
224 perc += bsp->perc;
225 if (r <= ((LONG_MAX / 100L) * perc))
226 break;
227 }
228 }
5ec10eaa
JA
229 if (!td->o.bs_unaligned) {
230 buflen = (buflen + td->o.min_bs[ddir] - 1)
231 & ~(td->o.min_bs[ddir] - 1);
232 }
10ba535a
JA
233 }
234
4ba66134
JA
235 if (io_u->offset + buflen > io_u->file->real_file_size) {
236 dprint(FD_IO, "lower buflen %u -> %u (ddir=%d)\n", buflen,
237 td->o.min_bs[ddir], ddir);
6a5e6884 238 buflen = td->o.min_bs[ddir];
4ba66134 239 }
6a5e6884 240
10ba535a
JA
241 return buflen;
242}
243
afe24a5a
JA
244static void set_rwmix_bytes(struct thread_data *td)
245{
246 unsigned long long rbytes;
247 unsigned int diff;
248
249 /*
250 * we do time or byte based switch. this is needed because
251 * buffered writes may issue a lot quicker than they complete,
252 * whereas reads do not.
253 */
254 rbytes = td->io_bytes[td->rwmix_ddir] - td->rwmix_bytes;
e47f799f 255 diff = td->o.rwmix[td->rwmix_ddir ^ 1];
afe24a5a 256
5ec10eaa
JA
257 td->rwmix_bytes = td->io_bytes[td->rwmix_ddir]
258 + (rbytes * ((100 - diff)) / diff);
e47f799f
JA
259}
260
261static inline enum fio_ddir get_rand_ddir(struct thread_data *td)
262{
263 unsigned int v;
264 long r;
265
266 r = os_random_long(&td->rwmix_state);
267 v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0)));
268 if (v < td->o.rwmix[DDIR_READ])
269 return DDIR_READ;
270
271 return DDIR_WRITE;
afe24a5a
JA
272}
273
10ba535a
JA
274/*
275 * Return the data direction for the next io_u. If the job is a
276 * mixed read/write workload, check the rwmix cycle and switch if
277 * necessary.
278 */
1e97cce9 279static enum fio_ddir get_rw_ddir(struct thread_data *td)
10ba535a
JA
280{
281 if (td_rw(td)) {
282 struct timeval now;
283 unsigned long elapsed;
afe24a5a 284 unsigned int cycle;
10ba535a 285
02bcaa8c 286 fio_gettime(&now, NULL);
5ec10eaa 287 elapsed = mtime_since_now(&td->rwmix_switch);
10ba535a 288
e47f799f
JA
289 /*
290 * if this is the first cycle, make it shorter
291 */
afe24a5a
JA
292 cycle = td->o.rwmixcycle;
293 if (!td->rwmix_bytes)
294 cycle /= 10;
295
10ba535a
JA
296 /*
297 * Check if it's time to seed a new data direction.
298 */
e47f799f 299 if (elapsed >= cycle ||
afe24a5a 300 td->io_bytes[td->rwmix_ddir] >= td->rwmix_bytes) {
e47f799f 301 unsigned long long max_bytes;
5ec10eaa 302 enum fio_ddir ddir;
e47f799f
JA
303
304 /*
305 * Put a top limit on how many bytes we do for
306 * one data direction, to avoid overflowing the
307 * ranges too much
308 */
309 ddir = get_rand_ddir(td);
310 max_bytes = td->this_io_bytes[ddir];
5ec10eaa
JA
311 if (max_bytes >=
312 (td->o.size * td->o.rwmix[ddir] / 100)) {
38d77cae
JA
313 if (!td->rw_end_set[ddir]) {
314 td->rw_end_set[ddir] = 1;
5ec10eaa
JA
315 memcpy(&td->rw_end[ddir], &now,
316 sizeof(now));
38d77cae 317 }
e47f799f 318 ddir ^= 1;
38d77cae 319 }
e47f799f
JA
320
321 if (ddir != td->rwmix_ddir)
322 set_rwmix_bytes(td);
323
324 td->rwmix_ddir = ddir;
10ba535a
JA
325 memcpy(&td->rwmix_switch, &now, sizeof(now));
326 }
327 return td->rwmix_ddir;
328 } else if (td_read(td))
329 return DDIR_READ;
330 else
331 return DDIR_WRITE;
332}
333
10ba535a
JA
334void put_io_u(struct thread_data *td, struct io_u *io_u)
335{
0c6e7517
JA
336 assert((io_u->flags & IO_U_F_FREE) == 0);
337 io_u->flags |= IO_U_F_FREE;
338
6977bcd0
JA
339 if (io_u->file) {
340 int ret = put_file(td, io_u->file);
341
342 if (ret)
343 td_verror(td, ret, "file close");
344 }
2dbdab7e 345
10ba535a
JA
346 io_u->file = NULL;
347 list_del(&io_u->list);
348 list_add(&io_u->list, &td->io_u_freelist);
349 td->cur_depth--;
350}
351
755200a3
JA
352void requeue_io_u(struct thread_data *td, struct io_u **io_u)
353{
354 struct io_u *__io_u = *io_u;
355
4d2e0f49 356 __io_u->flags |= IO_U_F_FREE;
e4f54adb
JA
357 if ((__io_u->flags & IO_U_F_FLIGHT) && (__io_u->ddir != DDIR_SYNC))
358 td->io_issues[__io_u->ddir]--;
5ec10eaa 359
4d2e0f49
JA
360 __io_u->flags &= ~IO_U_F_FLIGHT;
361
755200a3
JA
362 list_del(&__io_u->list);
363 list_add_tail(&__io_u->list, &td->io_u_requeues);
364 td->cur_depth--;
365 *io_u = NULL;
366}
367
9bf2061e 368static int fill_io_u(struct thread_data *td, struct io_u *io_u)
10ba535a 369{
b4c5e1ac
JA
370 if (td->io_ops->flags & FIO_NOIO)
371 goto out;
372
87dc1ab1
JA
373 /*
374 * see if it's time to sync
375 */
2dc1bbeb
JA
376 if (td->o.fsync_blocks &&
377 !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) &&
378 td->io_issues[DDIR_WRITE] && should_fsync(td)) {
87dc1ab1 379 io_u->ddir = DDIR_SYNC;
c38e9468 380 goto out;
87dc1ab1
JA
381 }
382
a00735e6
JA
383 io_u->ddir = get_rw_ddir(td);
384
48f5abd3
JA
385 /*
386 * See if it's time to switch to a new zone
387 */
388 if (td->zone_bytes >= td->o.zone_size) {
389 td->zone_bytes = 0;
390 io_u->file->last_pos += td->o.zone_skip;
391 td->io_skip_bytes += td->o.zone_skip;
392 }
393
10ba535a 394 /*
c685b5b2
JA
395 * No log, let the seq/rand engine retrieve the next buflen and
396 * position.
10ba535a 397 */
2ba1c290
JA
398 if (get_next_offset(td, io_u)) {
399 dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
bca4ed4d 400 return 1;
2ba1c290 401 }
10ba535a 402
9bf2061e 403 io_u->buflen = get_next_buflen(td, io_u);
2ba1c290
JA
404 if (!io_u->buflen) {
405 dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
bca4ed4d 406 return 1;
2ba1c290 407 }
bca4ed4d 408
2ba1c290
JA
409 if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
410 dprint(FD_IO, "io_u %p, offset too large\n", io_u);
4ba66134
JA
411 dprint(FD_IO, " off=%llu/%lu > %llu\n", io_u->offset,
412 io_u->buflen, io_u->file->real_file_size);
6a5e6884 413 return 1;
2ba1c290 414 }
6a5e6884 415
bca4ed4d
JA
416 /*
417 * mark entry before potentially trimming io_u
418 */
303032ae 419 if (td_random(td) && file_randommap(td, io_u->file))
9bf2061e 420 mark_random_map(td, io_u);
bca4ed4d
JA
421
422 /*
423 * If using a write iolog, store this entry.
424 */
c38e9468 425out:
2ba1c290 426 dprint_io_u(io_u, "fill_io_u");
d9d91e39 427 td->zone_bytes += io_u->buflen;
f29b25a3 428 log_io_u(td, io_u);
bca4ed4d 429 return 0;
10ba535a
JA
430}
431
b3605062 432void io_u_mark_depth(struct thread_data *td, struct io_u *io_u)
71619dc2
JA
433{
434 int index = 0;
435
b3605062
JA
436 if (io_u->ddir == DDIR_SYNC)
437 return;
438
71619dc2
JA
439 switch (td->cur_depth) {
440 default:
a783e61a
JA
441 index = 6;
442 break;
71619dc2 443 case 32 ... 63:
a783e61a
JA
444 index = 5;
445 break;
71619dc2 446 case 16 ... 31:
a783e61a
JA
447 index = 4;
448 break;
71619dc2 449 case 8 ... 15:
a783e61a
JA
450 index = 3;
451 break;
71619dc2 452 case 4 ... 7:
a783e61a
JA
453 index = 2;
454 break;
71619dc2 455 case 2 ... 3:
a783e61a 456 index = 1;
71619dc2
JA
457 case 1:
458 break;
459 }
460
756867bd 461 td->ts.io_u_map[index]++;
b3605062 462 td->ts.total_io_u[io_u->ddir]++;
71619dc2
JA
463}
464
04a0feae
JA
465static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec)
466{
467 int index = 0;
468
469 assert(usec < 1000);
470
471 switch (usec) {
472 case 750 ... 999:
473 index = 9;
474 break;
475 case 500 ... 749:
476 index = 8;
477 break;
478 case 250 ... 499:
479 index = 7;
480 break;
481 case 100 ... 249:
482 index = 6;
483 break;
484 case 50 ... 99:
485 index = 5;
486 break;
487 case 20 ... 49:
488 index = 4;
489 break;
490 case 10 ... 19:
491 index = 3;
492 break;
493 case 4 ... 9:
494 index = 2;
495 break;
496 case 2 ... 3:
497 index = 1;
498 case 0 ... 1:
499 break;
500 }
501
502 assert(index < FIO_IO_U_LAT_U_NR);
503 td->ts.io_u_lat_u[index]++;
504}
505
506static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec)
ec118304
JA
507{
508 int index = 0;
509
510 switch (msec) {
511 default:
04a0feae
JA
512 index = 11;
513 break;
8abdce66 514 case 1000 ... 1999:
04a0feae
JA
515 index = 10;
516 break;
8abdce66 517 case 750 ... 999:
04a0feae
JA
518 index = 9;
519 break;
8abdce66 520 case 500 ... 749:
04a0feae
JA
521 index = 8;
522 break;
8abdce66 523 case 250 ... 499:
04a0feae
JA
524 index = 7;
525 break;
8abdce66 526 case 100 ... 249:
04a0feae
JA
527 index = 6;
528 break;
8abdce66 529 case 50 ... 99:
04a0feae
JA
530 index = 5;
531 break;
8abdce66 532 case 20 ... 49:
04a0feae
JA
533 index = 4;
534 break;
8abdce66 535 case 10 ... 19:
04a0feae
JA
536 index = 3;
537 break;
8abdce66 538 case 4 ... 9:
04a0feae
JA
539 index = 2;
540 break;
ec118304 541 case 2 ... 3:
04a0feae 542 index = 1;
ec118304
JA
543 case 0 ... 1:
544 break;
545 }
546
04a0feae
JA
547 assert(index < FIO_IO_U_LAT_M_NR);
548 td->ts.io_u_lat_m[index]++;
549}
550
551static void io_u_mark_latency(struct thread_data *td, unsigned long usec)
552{
553 if (usec < 1000)
554 io_u_mark_lat_usec(td, usec);
555 else
556 io_u_mark_lat_msec(td, usec / 1000);
ec118304
JA
557}
558
0aabe160
JA
559/*
560 * Get next file to service by choosing one at random
561 */
1c178180
JA
562static struct fio_file *get_next_file_rand(struct thread_data *td, int goodf,
563 int badf)
0aabe160 564{
0aabe160 565 struct fio_file *f;
1c178180 566 int fno;
0aabe160
JA
567
568 do {
7c83c089
JA
569 long r = os_random_long(&td->next_file_state);
570
5ec10eaa
JA
571 fno = (unsigned int) ((double) td->o.nr_files
572 * (r / (RAND_MAX + 1.0)));
126d65c6 573 f = td->files[fno];
059e63c0
JA
574 if (f->flags & FIO_FILE_DONE)
575 continue;
1c178180 576
2ba1c290
JA
577 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) {
578 dprint(FD_FILE, "get_next_file_rand: %p\n", f);
0aabe160 579 return f;
2ba1c290 580 }
0aabe160
JA
581 } while (1);
582}
583
584/*
585 * Get next file to service by doing round robin between all available ones
586 */
1c178180
JA
587static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
588 int badf)
3d7c391d
JA
589{
590 unsigned int old_next_file = td->next_file;
591 struct fio_file *f;
592
593 do {
126d65c6 594 f = td->files[td->next_file];
3d7c391d
JA
595
596 td->next_file++;
2dc1bbeb 597 if (td->next_file >= td->o.nr_files)
3d7c391d
JA
598 td->next_file = 0;
599
d5ed68ea
JA
600 if (f->flags & FIO_FILE_DONE) {
601 f = NULL;
059e63c0 602 continue;
d5ed68ea 603 }
059e63c0 604
1c178180 605 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
3d7c391d
JA
606 break;
607
608 f = NULL;
609 } while (td->next_file != old_next_file);
610
2ba1c290 611 dprint(FD_FILE, "get_next_file_rr: %p\n", f);
3d7c391d
JA
612 return f;
613}
614
bdb4e2e9
JA
615static struct fio_file *get_next_file(struct thread_data *td)
616{
1907dbc6
JA
617 struct fio_file *f;
618
2dc1bbeb 619 assert(td->o.nr_files <= td->files_index);
1c178180 620
2ba1c290 621 if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files) {
5ec10eaa
JA
622 dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d,"
623 " nr_files=%d\n", td->nr_open_files,
624 td->nr_done_files,
625 td->o.nr_files);
bdb4e2e9 626 return NULL;
2ba1c290 627 }
bdb4e2e9 628
1907dbc6 629 f = td->file_service_file;
f11bd94d 630 if (f && (f->flags & FIO_FILE_OPEN) && td->file_service_left--)
2ba1c290 631 goto out;
1907dbc6 632
2dc1bbeb 633 if (td->o.file_service_type == FIO_FSERVICE_RR)
1c178180 634 f = get_next_file_rr(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
bdb4e2e9 635 else
1c178180 636 f = get_next_file_rand(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
1907dbc6
JA
637
638 td->file_service_file = f;
639 td->file_service_left = td->file_service_nr - 1;
2ba1c290
JA
640out:
641 dprint(FD_FILE, "get_next_file: %p\n", f);
1907dbc6 642 return f;
bdb4e2e9
JA
643}
644
1c178180
JA
645static struct fio_file *find_next_new_file(struct thread_data *td)
646{
647 struct fio_file *f;
648
1020a139
JA
649 if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files)
650 return NULL;
651
2dc1bbeb 652 if (td->o.file_service_type == FIO_FSERVICE_RR)
1c178180
JA
653 f = get_next_file_rr(td, 0, FIO_FILE_OPEN);
654 else
655 f = get_next_file_rand(td, 0, FIO_FILE_OPEN);
656
657 return f;
658}
659
429f6675
JA
660static int set_io_u_file(struct thread_data *td, struct io_u *io_u)
661{
662 struct fio_file *f;
663
664 do {
665 f = get_next_file(td);
666 if (!f)
667 return 1;
668
669set_file:
670 io_u->file = f;
671 get_file(f);
672
673 if (!fill_io_u(td, io_u))
674 break;
675
676 /*
677 * td_io_close() does a put_file() as well, so no need to
678 * do that here.
679 */
680 io_u->file = NULL;
681 td_io_close_file(td, f);
682 f->flags |= FIO_FILE_DONE;
683 td->nr_done_files++;
684
685 /*
686 * probably not the right place to do this, but see
687 * if we need to open a new file
688 */
689 if (td->nr_open_files < td->o.open_files &&
690 td->o.open_files != td->o.nr_files) {
691 f = find_next_new_file(td);
692
693 if (!f || td_io_open_file(td, f))
694 return 1;
695
696 goto set_file;
697 }
698 } while (1);
699
700 return 0;
701}
702
703
10ba535a
JA
704struct io_u *__get_io_u(struct thread_data *td)
705{
706 struct io_u *io_u = NULL;
707
755200a3
JA
708 if (!list_empty(&td->io_u_requeues))
709 io_u = list_entry(td->io_u_requeues.next, struct io_u, list);
710 else if (!queue_full(td)) {
10ba535a
JA
711 io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
712
6040dabc 713 io_u->buflen = 0;
10ba535a 714 io_u->resid = 0;
755200a3 715 io_u->file = NULL;
d7762cf8 716 io_u->end_io = NULL;
755200a3
JA
717 }
718
719 if (io_u) {
0c6e7517
JA
720 assert(io_u->flags & IO_U_F_FREE);
721 io_u->flags &= ~IO_U_F_FREE;
722
755200a3 723 io_u->error = 0;
10ba535a
JA
724 list_del(&io_u->list);
725 list_add(&io_u->list, &td->io_u_busylist);
726 td->cur_depth++;
727 }
728
729 return io_u;
730}
731
732/*
733 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
734 * etc. The returned io_u is fully ready to be prepped and submitted.
735 */
3d7c391d 736struct io_u *get_io_u(struct thread_data *td)
10ba535a 737{
3d7c391d 738 struct fio_file *f;
10ba535a
JA
739 struct io_u *io_u;
740
741 io_u = __get_io_u(td);
2ba1c290
JA
742 if (!io_u) {
743 dprint(FD_IO, "__get_io_u failed\n");
10ba535a 744 return NULL;
2ba1c290 745 }
10ba535a 746
755200a3
JA
747 /*
748 * from a requeue, io_u already setup
749 */
750 if (io_u->file)
77f392bf 751 goto out;
755200a3 752
429f6675
JA
753 /*
754 * If using an iolog, grab next piece if any available.
755 */
756 if (td->o.read_iolog_file) {
757 if (read_iolog_get(td, io_u))
758 goto err_put;
2ba1c290
JA
759 } else if (set_io_u_file(td, io_u)) {
760 dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
429f6675 761 goto err_put;
2ba1c290 762 }
5ec10eaa 763
429f6675
JA
764 f = io_u->file;
765 assert(f->flags & FIO_FILE_OPEN);
97af62ce 766
87dc1ab1 767 if (io_u->ddir != DDIR_SYNC) {
d0656a93 768 if (!io_u->buflen && !(td->io_ops->flags & FIO_NOIO)) {
2ba1c290 769 dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u);
429f6675 770 goto err_put;
2ba1c290 771 }
10ba535a 772
36167d82 773 f->last_pos = io_u->offset + io_u->buflen;
10ba535a 774
2dc1bbeb 775 if (td->o.verify != VERIFY_NONE)
87dc1ab1
JA
776 populate_verify_io_u(td, io_u);
777 }
10ba535a 778
165faf16
JA
779 /*
780 * Set io data pointers.
781 */
d460eb31 782 io_u->endpos = io_u->offset + io_u->buflen;
cec6b55d
JA
783 io_u->xfer_buf = io_u->buf;
784 io_u->xfer_buflen = io_u->buflen;
6ac7a331 785out:
429f6675
JA
786 if (!td_io_prep(td, io_u)) {
787 fio_gettime(&io_u->start_time, NULL);
788 return io_u;
36167d82 789 }
429f6675 790err_put:
2ba1c290 791 dprint(FD_IO, "get_io_u failed\n");
429f6675
JA
792 put_io_u(td, io_u);
793 return NULL;
10ba535a
JA
794}
795
5451792e
JA
796void io_u_log_error(struct thread_data *td, struct io_u *io_u)
797{
798 const char *msg[] = { "read", "write", "sync" };
799
800 log_err("fio: io_u error");
801
802 if (io_u->file)
803 log_err(" on file %s", io_u->file->file_name);
804
805 log_err(": %s\n", strerror(io_u->error));
806
5ec10eaa
JA
807 log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir],
808 io_u->offset, io_u->xfer_buflen);
5451792e
JA
809
810 if (!td->error)
811 td_verror(td, io_u->error, "io_u error");
812}
813
97601024
JA
814static void io_completed(struct thread_data *td, struct io_u *io_u,
815 struct io_completion_data *icd)
10ba535a 816{
d85f5118 817 unsigned long usec;
10ba535a 818
2ba1c290
JA
819 dprint_io_u(io_u, "io complete");
820
0c6e7517
JA
821 assert(io_u->flags & IO_U_F_FLIGHT);
822 io_u->flags &= ~IO_U_F_FLIGHT;
823
87dc1ab1
JA
824 if (io_u->ddir == DDIR_SYNC) {
825 td->last_was_sync = 1;
826 return;
827 }
828
829 td->last_was_sync = 0;
830
10ba535a
JA
831 if (!io_u->error) {
832 unsigned int bytes = io_u->buflen - io_u->resid;
1e97cce9 833 const enum fio_ddir idx = io_u->ddir;
3af6ef39 834 int ret;
10ba535a
JA
835
836 td->io_blocks[idx]++;
837 td->io_bytes[idx] += bytes;
10ba535a
JA
838 td->this_io_bytes[idx] += bytes;
839
d85f5118 840 usec = utime_since(&io_u->issue_time, &icd->time);
10ba535a 841
d85f5118 842 add_clat_sample(td, idx, usec);
02bcaa8c 843 add_bw_sample(td, idx, &icd->time);
04a0feae 844 io_u_mark_latency(td, usec);
10ba535a 845
660a1cb5 846 if (td_write(td) && idx == DDIR_WRITE &&
8670579e 847 td->o.do_verify &&
41128405 848 td->o.verify != VERIFY_NONE)
10ba535a
JA
849 log_io_piece(td, io_u);
850
851 icd->bytes_done[idx] += bytes;
3af6ef39 852
d7762cf8 853 if (io_u->end_io) {
36690c9b 854 ret = io_u->end_io(td, io_u);
3af6ef39
JA
855 if (ret && !icd->error)
856 icd->error = ret;
857 }
5451792e 858 } else {
10ba535a 859 icd->error = io_u->error;
5451792e
JA
860 io_u_log_error(td, io_u);
861 }
10ba535a
JA
862}
863
d7762cf8 864static void init_icd(struct io_completion_data *icd, int nr)
10ba535a 865{
02bcaa8c
JA
866 fio_gettime(&icd->time, NULL);
867
3af6ef39
JA
868 icd->nr = nr;
869
10ba535a
JA
870 icd->error = 0;
871 icd->bytes_done[0] = icd->bytes_done[1] = 0;
36167d82
JA
872}
873
97601024
JA
874static void ios_completed(struct thread_data *td,
875 struct io_completion_data *icd)
36167d82
JA
876{
877 struct io_u *io_u;
878 int i;
879
10ba535a
JA
880 for (i = 0; i < icd->nr; i++) {
881 io_u = td->io_ops->event(td, i);
882
883 io_completed(td, io_u, icd);
884 put_io_u(td, io_u);
885 }
886}
97601024 887
e7e6cfb4
JA
888/*
889 * Complete a single io_u for the sync engines.
890 */
d7762cf8 891long io_u_sync_complete(struct thread_data *td, struct io_u *io_u)
97601024
JA
892{
893 struct io_completion_data icd;
894
d7762cf8 895 init_icd(&icd, 1);
97601024
JA
896 io_completed(td, io_u, &icd);
897 put_io_u(td, io_u);
898
899 if (!icd.error)
900 return icd.bytes_done[0] + icd.bytes_done[1];
901
37e974a5 902 td_verror(td, icd.error, "io_u_sync_complete");
97601024
JA
903 return -1;
904}
905
e7e6cfb4
JA
906/*
907 * Called to complete min_events number of io for the async engines.
908 */
d7762cf8 909long io_u_queued_complete(struct thread_data *td, int min_events)
97601024 910{
97601024 911 struct io_completion_data icd;
00de55ef 912 struct timespec *tvp = NULL;
97601024 913 int ret;
4d06a338 914 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
97601024 915
b271fe62
JA
916 dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_events);
917
4d06a338 918 if (!min_events)
00de55ef 919 tvp = &ts;
97601024 920
00de55ef 921 ret = td_io_getevents(td, min_events, td->cur_depth, tvp);
97601024 922 if (ret < 0) {
e1161c32 923 td_verror(td, -ret, "td_io_getevents");
97601024
JA
924 return ret;
925 } else if (!ret)
926 return ret;
927
d7762cf8 928 init_icd(&icd, ret);
97601024
JA
929 ios_completed(td, &icd);
930 if (!icd.error)
931 return icd.bytes_done[0] + icd.bytes_done[1];
932
37e974a5 933 td_verror(td, icd.error, "io_u_queued_complete");
97601024
JA
934 return -1;
935}
7e77dd02
JA
936
937/*
938 * Call when io_u is really queued, to update the submission latency.
939 */
940void io_u_queued(struct thread_data *td, struct io_u *io_u)
941{
942 unsigned long slat_time;
943
d85f5118 944 slat_time = utime_since(&io_u->start_time, &io_u->issue_time);
7e77dd02
JA
945 add_slat_sample(td, io_u->ddir, slat_time);
946}
433afcb4 947
55bc9728 948#ifdef FIO_USE_TIMEOUT
433afcb4
JA
949void io_u_set_timeout(struct thread_data *td)
950{
951 assert(td->cur_depth);
952
953 td->timer.it_interval.tv_sec = 0;
954 td->timer.it_interval.tv_usec = 0;
955 td->timer.it_value.tv_sec = IO_U_TIMEOUT + IO_U_TIMEOUT_INC;
956 td->timer.it_value.tv_usec = 0;
957 setitimer(ITIMER_REAL, &td->timer, NULL);
958 fio_gettime(&td->timeout_end, NULL);
959}
5945b9b4
JA
960
961static void io_u_dump(struct io_u *io_u)
962{
963 unsigned long t_start = mtime_since_now(&io_u->start_time);
964 unsigned long t_issue = mtime_since_now(&io_u->issue_time);
965
966 log_err("io_u=%p, t_start=%lu, t_issue=%lu\n", io_u, t_start, t_issue);
5ec10eaa
JA
967 log_err(" buf=%p/%p, len=%lu/%lu, offset=%llu\n", io_u->buf,
968 io_u->xfer_buf, io_u->buflen,
969 io_u->xfer_buflen,
970 io_u->offset);
5945b9b4
JA
971 log_err(" ddir=%d, fname=%s\n", io_u->ddir, io_u->file->file_name);
972}
55bc9728
JA
973#else
974void io_u_set_timeout(struct thread_data fio_unused *td)
975{
976}
977#endif
433afcb4 978
55bc9728 979#ifdef FIO_USE_TIMEOUT
433afcb4
JA
980static void io_u_timeout_handler(int fio_unused sig)
981{
982 struct thread_data *td, *__td;
983 pid_t pid = getpid();
5945b9b4
JA
984 struct list_head *entry;
985 struct io_u *io_u;
433afcb4
JA
986 int i;
987
988 log_err("fio: io_u timeout\n");
989
990 /*
991 * TLS would be nice...
992 */
993 td = NULL;
994 for_each_td(__td, i) {
995 if (__td->pid == pid) {
996 td = __td;
997 break;
998 }
999 }
1000
1001 if (!td) {
1002 log_err("fio: io_u timeout, can't find job\n");
1003 exit(1);
1004 }
1005
1006 if (!td->cur_depth) {
1007 log_err("fio: timeout without pending work?\n");
1008 return;
1009 }
1010
15506d09 1011 log_err("fio: io_u timeout: job=%s, pid=%d\n", td->o.name, td->pid);
5945b9b4
JA
1012
1013 list_for_each(entry, &td->io_u_busylist) {
1014 io_u = list_entry(entry, struct io_u, list);
1015
1016 io_u_dump(io_u);
1017 }
1018
1019 td_verror(td, ETIMEDOUT, "io_u timeout");
433afcb4
JA
1020 exit(1);
1021}
55bc9728 1022#endif
433afcb4
JA
1023
1024void io_u_init_timeout(void)
1025{
55bc9728 1026#ifdef FIO_USE_TIMEOUT
433afcb4 1027 signal(SIGALRM, io_u_timeout_handler);
55bc9728 1028#endif
433afcb4 1029}