Fix file open leak
[fio.git] / io_u.c
CommitLineData
10ba535a
JA
1#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <signal.h>
5#include <time.h>
0c6e7517 6#include <assert.h>
10ba535a
JA
7
8#include "fio.h"
5973cafb 9#include "hash.h"
10ba535a 10
97601024
JA
11struct io_completion_data {
12 int nr; /* input */
97601024
JA
13
14 int error; /* output */
15 unsigned long bytes_done[2]; /* output */
16 struct timeval time; /* output */
17};
18
10ba535a
JA
19/*
20 * The ->file_map[] contains a map of blocks we have or have not done io
21 * to yet. Used to make sure we cover the entire range in a fair fashion.
22 */
aec2de20 23static int random_map_free(struct fio_file *f, const unsigned long long block)
10ba535a 24{
aec2de20
JA
25 unsigned int idx = RAND_MAP_IDX(f, block);
26 unsigned int bit = RAND_MAP_BIT(f, block);
10ba535a 27
84422acd
JA
28 dprint(FD_RANDOM, "free: b=%llu, idx=%u, bit=%u\n", block, idx, bit);
29
de605666 30 return (f->file_map[idx] & (1 << bit)) == 0;
10ba535a
JA
31}
32
df415585
JA
33/*
34 * Mark a given offset as used in the map.
35 */
9bf2061e 36static void mark_random_map(struct thread_data *td, struct io_u *io_u)
df415585 37{
2dc1bbeb 38 unsigned int min_bs = td->o.rw_min_bs;
9bf2061e 39 struct fio_file *f = io_u->file;
a00735e6 40 unsigned long long block;
3e3357b1 41 unsigned int blocks, nr_blocks;
df415585 42
b9c5b644 43 block = (io_u->offset - f->file_offset) / (unsigned long long) min_bs;
c685b5b2 44 nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
3e3357b1 45 blocks = 0;
c685b5b2 46
3e3357b1
JA
47 while (nr_blocks) {
48 unsigned int this_blocks, mask;
df415585
JA
49 unsigned int idx, bit;
50
1e3d53ac
JA
51 /*
52 * If we have a mixed random workload, we may
53 * encounter blocks we already did IO to.
54 */
8b113b4f
JA
55 if ((td->o.ddir_nr == 1) && !random_map_free(f, block)) {
56 if (!blocks)
57 blocks = 1;
df415585 58 break;
8b113b4f 59 }
df415585 60
aec2de20
JA
61 idx = RAND_MAP_IDX(f, block);
62 bit = RAND_MAP_BIT(f, block);
df415585 63
0032bf9f 64 fio_assert(td, idx < f->num_maps);
df415585 65
3e3357b1
JA
66 this_blocks = nr_blocks;
67 if (this_blocks + bit > BLOCKS_PER_MAP)
68 this_blocks = BLOCKS_PER_MAP - bit;
69
70 if (this_blocks == BLOCKS_PER_MAP)
71 mask = -1U;
72 else
73 mask = ((1U << this_blocks) - 1) << bit;
74
3e3357b1
JA
75 f->file_map[idx] |= mask;
76 nr_blocks -= this_blocks;
77 blocks += this_blocks;
c9dd34b2 78 block += this_blocks;
df415585
JA
79 }
80
a00735e6
JA
81 if ((blocks * min_bs) < io_u->buflen)
82 io_u->buflen = blocks * min_bs;
df415585
JA
83}
84
3e3357b1
JA
85static unsigned long long last_block(struct thread_data *td, struct fio_file *f,
86 enum fio_ddir ddir)
2ba1c290
JA
87{
88 unsigned long long max_blocks;
d9dd70f7 89 unsigned long long max_size;
2ba1c290 90
d9dd70f7
JA
91 /*
92 * Hmm, should we make sure that ->io_size <= ->real_file_size?
93 */
94 max_size = f->io_size;
95 if (max_size > f->real_file_size)
96 max_size = f->real_file_size;
97
98 max_blocks = max_size / (unsigned long long) td->o.min_bs[ddir];
2ba1c290
JA
99 if (!max_blocks)
100 return 0;
101
67778e88 102 return max_blocks;
2ba1c290
JA
103}
104
10ba535a
JA
105/*
106 * Return the next free block in the map.
107 */
108static int get_next_free_block(struct thread_data *td, struct fio_file *f,
4ba66134 109 enum fio_ddir ddir, unsigned long long *b)
10ba535a 110{
84422acd 111 unsigned long long min_bs = td->o.rw_min_bs;
10ba535a
JA
112 int i;
113
c685b5b2
JA
114 i = f->last_free_lookup;
115 *b = (i * BLOCKS_PER_MAP);
84422acd 116 while ((*b) * min_bs < f->real_file_size) {
de605666 117 if (f->file_map[i] != (unsigned int) -1) {
697a606c 118 *b += ffz(f->file_map[i]);
4ba66134 119 if (*b > last_block(td, f, ddir))
2ba1c290 120 break;
c685b5b2 121 f->last_free_lookup = i;
10ba535a
JA
122 return 0;
123 }
124
125 *b += BLOCKS_PER_MAP;
126 i++;
127 }
128
2ba1c290 129 dprint(FD_IO, "failed finding a free block\n");
10ba535a
JA
130 return 1;
131}
132
ec4015da 133static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
4ba66134 134 enum fio_ddir ddir, unsigned long long *b)
ec4015da 135{
84422acd 136 unsigned long long r;
ec4015da
JA
137 int loops = 5;
138
139 do {
140 r = os_random_long(&td->random_state);
84422acd 141 dprint(FD_RANDOM, "off rand %llu\n", r);
5ec10eaa 142 *b = (last_block(td, f, ddir) - 1)
dc873b6f 143 * (r / ((unsigned long long) OS_RAND_MAX + 1.0));
2ba1c290 144
43c63a78
JA
145 /*
146 * if we are not maintaining a random map, we are done.
147 */
303032ae 148 if (!file_randommap(td, f))
43c63a78
JA
149 return 0;
150
151 /*
84422acd 152 * calculate map offset and check if it's free
43c63a78 153 */
aec2de20 154 if (random_map_free(f, *b))
43c63a78
JA
155 return 0;
156
84422acd
JA
157 dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
158 *b);
43c63a78 159 } while (--loops);
ec4015da
JA
160
161 /*
43c63a78
JA
162 * we get here, if we didn't suceed in looking up a block. generate
163 * a random start offset into the filemap, and find the first free
164 * block from there.
ec4015da 165 */
43c63a78
JA
166 loops = 10;
167 do {
dc873b6f
JA
168 f->last_free_lookup = (f->num_maps - 1) *
169 (r / (OS_RAND_MAX + 1.0));
4ba66134 170 if (!get_next_free_block(td, f, ddir, b))
43c63a78 171 return 0;
ec4015da 172
43c63a78
JA
173 r = os_random_long(&td->random_state);
174 } while (--loops);
175
176 /*
177 * that didn't work either, try exhaustive search from the start
178 */
179 f->last_free_lookup = 0;
4ba66134 180 return get_next_free_block(td, f, ddir, b);
ec4015da
JA
181}
182
10ba535a
JA
183/*
184 * For random io, generate a random new block and see if it's used. Repeat
185 * until we find a free one. For sequential io, just return the end of
186 * the last io issued.
187 */
9bf2061e 188static int get_next_offset(struct thread_data *td, struct io_u *io_u)
10ba535a 189{
9bf2061e 190 struct fio_file *f = io_u->file;
ec4015da 191 unsigned long long b;
4ba66134 192 enum fio_ddir ddir = io_u->ddir;
10ba535a 193
ec4015da
JA
194 if (td_random(td) && (td->o.ddir_nr && !--td->ddir_nr)) {
195 td->ddir_nr = td->o.ddir_nr;
211097b2 196
683023e8
JA
197 if (get_next_rand_offset(td, f, ddir, &b)) {
198 dprint(FD_IO, "%s: getting rand offset failed\n",
199 f->file_name);
bca4ed4d 200 return 1;
683023e8 201 }
43063a1c 202 } else {
0c3d768a 203 if (f->last_pos >= f->real_file_size) {
4ba66134 204 if (!td_random(td) ||
683023e8
JA
205 get_next_rand_offset(td, f, ddir, &b)) {
206 dprint(FD_IO, "%s: pos %llu > size %llu\n",
207 f->file_name, f->last_pos,
208 f->real_file_size);
0c3d768a 209 return 1;
683023e8 210 }
bcdedd0a 211 } else
4ba66134 212 b = (f->last_pos - f->file_offset) / td->o.min_bs[ddir];
43063a1c 213 }
10ba535a 214
009bd847
JA
215 io_u->offset = b * td->o.min_bs[ddir];
216 if (io_u->offset >= f->io_size) {
217 dprint(FD_IO, "get_next_offset: offset %llu >= io_size %llu\n",
218 io_u->offset, f->io_size);
219 return 1;
220 }
221
222 io_u->offset += f->file_offset;
2ba1c290
JA
223 if (io_u->offset >= f->real_file_size) {
224 dprint(FD_IO, "get_next_offset: offset %llu >= size %llu\n",
225 io_u->offset, f->real_file_size);
10ba535a 226 return 1;
2ba1c290 227 }
10ba535a
JA
228
229 return 0;
230}
231
f3059de1
JA
232static inline int is_power_of_2(unsigned int val)
233{
234 return (val != 0 && ((val & (val - 1)) == 0));
235}
236
9bf2061e 237static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u)
10ba535a 238{
bca4ed4d 239 const int ddir = io_u->ddir;
f3f552b9 240 unsigned int uninitialized_var(buflen);
f3059de1 241 unsigned int minbs, maxbs;
10ba535a
JA
242 long r;
243
f3059de1
JA
244 minbs = td->o.min_bs[ddir];
245 maxbs = td->o.max_bs[ddir];
246
247 if (minbs == maxbs)
248 buflen = minbs;
10ba535a
JA
249 else {
250 r = os_random_long(&td->bsrange_state);
5ec10eaa 251 if (!td->o.bssplit_nr) {
f3059de1
JA
252 buflen = 1 + (unsigned int) ((double) maxbs *
253 (r / (OS_RAND_MAX + 1.0)));
254 if (buflen < minbs)
255 buflen = minbs;
5ec10eaa 256 } else {
564ca972
JA
257 long perc = 0;
258 unsigned int i;
259
260 for (i = 0; i < td->o.bssplit_nr; i++) {
261 struct bssplit *bsp = &td->o.bssplit[i];
262
263 buflen = bsp->bs;
264 perc += bsp->perc;
f3059de1 265 if (r <= ((OS_RAND_MAX / 100L) * perc))
564ca972
JA
266 break;
267 }
268 }
f3059de1
JA
269 if (!td->o.bs_unaligned && is_power_of_2(minbs))
270 buflen = (buflen + minbs - 1) & ~(minbs - 1);
10ba535a
JA
271 }
272
4ba66134
JA
273 if (io_u->offset + buflen > io_u->file->real_file_size) {
274 dprint(FD_IO, "lower buflen %u -> %u (ddir=%d)\n", buflen,
f3059de1
JA
275 minbs, ddir);
276 buflen = minbs;
4ba66134 277 }
6a5e6884 278
10ba535a
JA
279 return buflen;
280}
281
afe24a5a
JA
282static void set_rwmix_bytes(struct thread_data *td)
283{
afe24a5a
JA
284 unsigned int diff;
285
286 /*
287 * we do time or byte based switch. this is needed because
288 * buffered writes may issue a lot quicker than they complete,
289 * whereas reads do not.
290 */
e47f799f 291 diff = td->o.rwmix[td->rwmix_ddir ^ 1];
04c540d9 292 td->rwmix_issues = (td->io_issues[td->rwmix_ddir] * diff) / 100;
e47f799f
JA
293}
294
295static inline enum fio_ddir get_rand_ddir(struct thread_data *td)
296{
297 unsigned int v;
298 long r;
299
300 r = os_random_long(&td->rwmix_state);
dc873b6f 301 v = 1 + (int) (100.0 * (r / (OS_RAND_MAX + 1.0)));
04c540d9 302 if (v <= td->o.rwmix[DDIR_READ])
e47f799f
JA
303 return DDIR_READ;
304
305 return DDIR_WRITE;
afe24a5a
JA
306}
307
10ba535a
JA
308/*
309 * Return the data direction for the next io_u. If the job is a
310 * mixed read/write workload, check the rwmix cycle and switch if
311 * necessary.
312 */
1e97cce9 313static enum fio_ddir get_rw_ddir(struct thread_data *td)
10ba535a
JA
314{
315 if (td_rw(td)) {
10ba535a
JA
316 /*
317 * Check if it's time to seed a new data direction.
318 */
e4928662 319 if (td->io_issues[td->rwmix_ddir] >= td->rwmix_issues) {
e47f799f 320 unsigned long long max_bytes;
5ec10eaa 321 enum fio_ddir ddir;
e47f799f
JA
322
323 /*
324 * Put a top limit on how many bytes we do for
325 * one data direction, to avoid overflowing the
326 * ranges too much
327 */
328 ddir = get_rand_ddir(td);
329 max_bytes = td->this_io_bytes[ddir];
5ec10eaa
JA
330 if (max_bytes >=
331 (td->o.size * td->o.rwmix[ddir] / 100)) {
6162f524 332 if (!td->rw_end_set[ddir]) {
38d77cae 333 td->rw_end_set[ddir] = 1;
6162f524
JA
334 fio_gettime(&td->rw_end[ddir], NULL);
335 }
e4928662 336
e47f799f 337 ddir ^= 1;
38d77cae 338 }
e47f799f
JA
339
340 if (ddir != td->rwmix_ddir)
341 set_rwmix_bytes(td);
342
343 td->rwmix_ddir = ddir;
10ba535a
JA
344 }
345 return td->rwmix_ddir;
346 } else if (td_read(td))
347 return DDIR_READ;
348 else
349 return DDIR_WRITE;
350}
351
60f2c658
JA
352static void put_file_log(struct thread_data *td, struct fio_file *f)
353{
354 int ret = put_file(td, f);
355
356 if (ret)
357 td_verror(td, ret, "file close");
358}
359
10ba535a
JA
360void put_io_u(struct thread_data *td, struct io_u *io_u)
361{
0c6e7517
JA
362 assert((io_u->flags & IO_U_F_FREE) == 0);
363 io_u->flags |= IO_U_F_FREE;
364
60f2c658
JA
365 if (io_u->file)
366 put_file_log(td, io_u->file);
2dbdab7e 367
10ba535a 368 io_u->file = NULL;
01743ee1
JA
369 flist_del(&io_u->list);
370 flist_add(&io_u->list, &td->io_u_freelist);
10ba535a
JA
371 td->cur_depth--;
372}
373
755200a3
JA
374void requeue_io_u(struct thread_data *td, struct io_u **io_u)
375{
376 struct io_u *__io_u = *io_u;
377
465221b0
JA
378 dprint(FD_IO, "requeue %p\n", __io_u);
379
4d2e0f49 380 __io_u->flags |= IO_U_F_FREE;
e4f54adb
JA
381 if ((__io_u->flags & IO_U_F_FLIGHT) && (__io_u->ddir != DDIR_SYNC))
382 td->io_issues[__io_u->ddir]--;
5ec10eaa 383
4d2e0f49
JA
384 __io_u->flags &= ~IO_U_F_FLIGHT;
385
01743ee1
JA
386 flist_del(&__io_u->list);
387 flist_add_tail(&__io_u->list, &td->io_u_requeues);
755200a3
JA
388 td->cur_depth--;
389 *io_u = NULL;
390}
391
9bf2061e 392static int fill_io_u(struct thread_data *td, struct io_u *io_u)
10ba535a 393{
b4c5e1ac
JA
394 if (td->io_ops->flags & FIO_NOIO)
395 goto out;
396
87dc1ab1
JA
397 /*
398 * see if it's time to sync
399 */
2dc1bbeb
JA
400 if (td->o.fsync_blocks &&
401 !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) &&
402 td->io_issues[DDIR_WRITE] && should_fsync(td)) {
87dc1ab1 403 io_u->ddir = DDIR_SYNC;
c38e9468 404 goto out;
87dc1ab1
JA
405 }
406
a00735e6
JA
407 io_u->ddir = get_rw_ddir(td);
408
48f5abd3
JA
409 /*
410 * See if it's time to switch to a new zone
411 */
412 if (td->zone_bytes >= td->o.zone_size) {
413 td->zone_bytes = 0;
414 io_u->file->last_pos += td->o.zone_skip;
415 td->io_skip_bytes += td->o.zone_skip;
416 }
417
10ba535a 418 /*
c685b5b2
JA
419 * No log, let the seq/rand engine retrieve the next buflen and
420 * position.
10ba535a 421 */
2ba1c290
JA
422 if (get_next_offset(td, io_u)) {
423 dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
bca4ed4d 424 return 1;
2ba1c290 425 }
10ba535a 426
9bf2061e 427 io_u->buflen = get_next_buflen(td, io_u);
2ba1c290
JA
428 if (!io_u->buflen) {
429 dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
bca4ed4d 430 return 1;
2ba1c290 431 }
bca4ed4d 432
2ba1c290
JA
433 if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
434 dprint(FD_IO, "io_u %p, offset too large\n", io_u);
4ba66134
JA
435 dprint(FD_IO, " off=%llu/%lu > %llu\n", io_u->offset,
436 io_u->buflen, io_u->file->real_file_size);
6a5e6884 437 return 1;
2ba1c290 438 }
6a5e6884 439
bca4ed4d
JA
440 /*
441 * mark entry before potentially trimming io_u
442 */
303032ae 443 if (td_random(td) && file_randommap(td, io_u->file))
9bf2061e 444 mark_random_map(td, io_u);
bca4ed4d
JA
445
446 /*
447 * If using a write iolog, store this entry.
448 */
c38e9468 449out:
2ba1c290 450 dprint_io_u(io_u, "fill_io_u");
d9d91e39 451 td->zone_bytes += io_u->buflen;
f29b25a3 452 log_io_u(td, io_u);
bca4ed4d 453 return 0;
10ba535a
JA
454}
455
838bc709
JA
456static void __io_u_mark_map(unsigned int *map, unsigned int nr)
457{
458 int index = 0;
459
460 switch (nr) {
461 default:
462 index = 6;
463 break;
464 case 33 ... 64:
465 index = 5;
466 break;
467 case 17 ... 32:
468 index = 4;
469 break;
470 case 9 ... 16:
471 index = 3;
472 break;
473 case 5 ... 8:
474 index = 2;
475 break;
476 case 1 ... 4:
477 index = 1;
478 case 0:
479 break;
480 }
481
482 map[index]++;
483}
484
485void io_u_mark_submit(struct thread_data *td, unsigned int nr)
486{
487 __io_u_mark_map(td->ts.io_u_submit, nr);
488 td->ts.total_submit++;
489}
490
491void io_u_mark_complete(struct thread_data *td, unsigned int nr)
492{
493 __io_u_mark_map(td->ts.io_u_complete, nr);
494 td->ts.total_complete++;
495}
496
d8005759 497void io_u_mark_depth(struct thread_data *td, unsigned int nr)
71619dc2
JA
498{
499 int index = 0;
500
501 switch (td->cur_depth) {
502 default:
a783e61a
JA
503 index = 6;
504 break;
71619dc2 505 case 32 ... 63:
a783e61a
JA
506 index = 5;
507 break;
71619dc2 508 case 16 ... 31:
a783e61a
JA
509 index = 4;
510 break;
71619dc2 511 case 8 ... 15:
a783e61a
JA
512 index = 3;
513 break;
71619dc2 514 case 4 ... 7:
a783e61a
JA
515 index = 2;
516 break;
71619dc2 517 case 2 ... 3:
a783e61a 518 index = 1;
71619dc2
JA
519 case 1:
520 break;
521 }
522
3bec7ae0 523 td->ts.io_u_map[index] += nr;
71619dc2
JA
524}
525
04a0feae
JA
526static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec)
527{
528 int index = 0;
529
530 assert(usec < 1000);
531
532 switch (usec) {
533 case 750 ... 999:
534 index = 9;
535 break;
536 case 500 ... 749:
537 index = 8;
538 break;
539 case 250 ... 499:
540 index = 7;
541 break;
542 case 100 ... 249:
543 index = 6;
544 break;
545 case 50 ... 99:
546 index = 5;
547 break;
548 case 20 ... 49:
549 index = 4;
550 break;
551 case 10 ... 19:
552 index = 3;
553 break;
554 case 4 ... 9:
555 index = 2;
556 break;
557 case 2 ... 3:
558 index = 1;
559 case 0 ... 1:
560 break;
561 }
562
563 assert(index < FIO_IO_U_LAT_U_NR);
564 td->ts.io_u_lat_u[index]++;
565}
566
567static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec)
ec118304
JA
568{
569 int index = 0;
570
571 switch (msec) {
572 default:
04a0feae
JA
573 index = 11;
574 break;
8abdce66 575 case 1000 ... 1999:
04a0feae
JA
576 index = 10;
577 break;
8abdce66 578 case 750 ... 999:
04a0feae
JA
579 index = 9;
580 break;
8abdce66 581 case 500 ... 749:
04a0feae
JA
582 index = 8;
583 break;
8abdce66 584 case 250 ... 499:
04a0feae
JA
585 index = 7;
586 break;
8abdce66 587 case 100 ... 249:
04a0feae
JA
588 index = 6;
589 break;
8abdce66 590 case 50 ... 99:
04a0feae
JA
591 index = 5;
592 break;
8abdce66 593 case 20 ... 49:
04a0feae
JA
594 index = 4;
595 break;
8abdce66 596 case 10 ... 19:
04a0feae
JA
597 index = 3;
598 break;
8abdce66 599 case 4 ... 9:
04a0feae
JA
600 index = 2;
601 break;
ec118304 602 case 2 ... 3:
04a0feae 603 index = 1;
ec118304
JA
604 case 0 ... 1:
605 break;
606 }
607
04a0feae
JA
608 assert(index < FIO_IO_U_LAT_M_NR);
609 td->ts.io_u_lat_m[index]++;
610}
611
612static void io_u_mark_latency(struct thread_data *td, unsigned long usec)
613{
614 if (usec < 1000)
615 io_u_mark_lat_usec(td, usec);
616 else
617 io_u_mark_lat_msec(td, usec / 1000);
ec118304
JA
618}
619
0aabe160
JA
620/*
621 * Get next file to service by choosing one at random
622 */
1c178180
JA
623static struct fio_file *get_next_file_rand(struct thread_data *td, int goodf,
624 int badf)
0aabe160 625{
0aabe160 626 struct fio_file *f;
1c178180 627 int fno;
0aabe160
JA
628
629 do {
7c83c089 630 long r = os_random_long(&td->next_file_state);
87b10676 631 int opened = 0;
7c83c089 632
5ec10eaa 633 fno = (unsigned int) ((double) td->o.nr_files
dc873b6f 634 * (r / (OS_RAND_MAX + 1.0)));
126d65c6 635 f = td->files[fno];
059e63c0
JA
636 if (f->flags & FIO_FILE_DONE)
637 continue;
1c178180 638
87b10676
JA
639 if (!(f->flags & FIO_FILE_OPEN)) {
640 int err;
641
642 err = td_io_open_file(td, f);
643 if (err)
644 continue;
645 opened = 1;
646 }
647
2ba1c290
JA
648 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) {
649 dprint(FD_FILE, "get_next_file_rand: %p\n", f);
0aabe160 650 return f;
2ba1c290 651 }
87b10676
JA
652 if (opened)
653 td_io_close_file(td, f);
0aabe160
JA
654 } while (1);
655}
656
657/*
658 * Get next file to service by doing round robin between all available ones
659 */
1c178180
JA
660static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
661 int badf)
3d7c391d
JA
662{
663 unsigned int old_next_file = td->next_file;
664 struct fio_file *f;
665
666 do {
87b10676
JA
667 int opened = 0;
668
126d65c6 669 f = td->files[td->next_file];
3d7c391d
JA
670
671 td->next_file++;
2dc1bbeb 672 if (td->next_file >= td->o.nr_files)
3d7c391d
JA
673 td->next_file = 0;
674
87b10676 675 dprint(FD_FILE, "trying file %s %x\n", f->file_name, f->flags);
d5ed68ea
JA
676 if (f->flags & FIO_FILE_DONE) {
677 f = NULL;
059e63c0 678 continue;
d5ed68ea 679 }
059e63c0 680
87b10676
JA
681 if (!(f->flags & FIO_FILE_OPEN)) {
682 int err;
683
684 err = td_io_open_file(td, f);
685 if (err)
686 continue;
687 opened = 1;
688 }
689
1c178180 690 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
3d7c391d
JA
691 break;
692
87b10676
JA
693 if (opened)
694 td_io_close_file(td, f);
695
3d7c391d
JA
696 f = NULL;
697 } while (td->next_file != old_next_file);
698
2ba1c290 699 dprint(FD_FILE, "get_next_file_rr: %p\n", f);
3d7c391d
JA
700 return f;
701}
702
bdb4e2e9
JA
703static struct fio_file *get_next_file(struct thread_data *td)
704{
1907dbc6
JA
705 struct fio_file *f;
706
2dc1bbeb 707 assert(td->o.nr_files <= td->files_index);
1c178180 708
2ba1c290 709 if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files) {
5ec10eaa
JA
710 dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d,"
711 " nr_files=%d\n", td->nr_open_files,
712 td->nr_done_files,
713 td->o.nr_files);
bdb4e2e9 714 return NULL;
2ba1c290 715 }
bdb4e2e9 716
1907dbc6 717 f = td->file_service_file;
683023e8 718 if (f && (f->flags & FIO_FILE_OPEN) && !(f->flags & FIO_FILE_CLOSING)) {
a086c257
JA
719 if (td->o.file_service_type == FIO_FSERVICE_SEQ)
720 goto out;
721 if (td->file_service_left--)
722 goto out;
723 }
1907dbc6 724
a086c257
JA
725 if (td->o.file_service_type == FIO_FSERVICE_RR ||
726 td->o.file_service_type == FIO_FSERVICE_SEQ)
1c178180 727 f = get_next_file_rr(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
bdb4e2e9 728 else
1c178180 729 f = get_next_file_rand(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
1907dbc6
JA
730
731 td->file_service_file = f;
732 td->file_service_left = td->file_service_nr - 1;
2ba1c290 733out:
683023e8 734 dprint(FD_FILE, "get_next_file: %p [%s]\n", f, f->file_name);
1907dbc6 735 return f;
bdb4e2e9
JA
736}
737
1c178180
JA
738static struct fio_file *find_next_new_file(struct thread_data *td)
739{
740 struct fio_file *f;
741
1020a139
JA
742 if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files)
743 return NULL;
744
2dc1bbeb 745 if (td->o.file_service_type == FIO_FSERVICE_RR)
1c178180
JA
746 f = get_next_file_rr(td, 0, FIO_FILE_OPEN);
747 else
748 f = get_next_file_rand(td, 0, FIO_FILE_OPEN);
749
750 return f;
751}
752
429f6675
JA
753static int set_io_u_file(struct thread_data *td, struct io_u *io_u)
754{
755 struct fio_file *f;
756
757 do {
758 f = get_next_file(td);
759 if (!f)
760 return 1;
761
762set_file:
763 io_u->file = f;
764 get_file(f);
765
766 if (!fill_io_u(td, io_u))
767 break;
768
009bd847
JA
769 /*
770 * optimization to prevent close/open of the same file. This
771 * way we preserve queueing etc.
772 */
773 if (td->o.nr_files == 1 && td->o.time_based) {
60f2c658 774 put_file_log(td, f);
009bd847
JA
775 fio_file_reset(f);
776 goto set_file;
777 }
778
429f6675
JA
779 /*
780 * td_io_close() does a put_file() as well, so no need to
781 * do that here.
782 */
783 io_u->file = NULL;
784 td_io_close_file(td, f);
785 f->flags |= FIO_FILE_DONE;
786 td->nr_done_files++;
87b10676 787 dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name, td->nr_done_files, td->o.nr_files);
429f6675
JA
788
789 /*
790 * probably not the right place to do this, but see
791 * if we need to open a new file
792 */
793 if (td->nr_open_files < td->o.open_files &&
794 td->o.open_files != td->o.nr_files) {
795 f = find_next_new_file(td);
4bb997ce 796 if (!f)
429f6675
JA
797 return 1;
798
799 goto set_file;
800 }
801 } while (1);
802
803 return 0;
804}
805
806
10ba535a
JA
807struct io_u *__get_io_u(struct thread_data *td)
808{
809 struct io_u *io_u = NULL;
810
01743ee1
JA
811 if (!flist_empty(&td->io_u_requeues))
812 io_u = flist_entry(td->io_u_requeues.next, struct io_u, list);
755200a3 813 else if (!queue_full(td)) {
01743ee1 814 io_u = flist_entry(td->io_u_freelist.next, struct io_u, list);
10ba535a 815
6040dabc 816 io_u->buflen = 0;
10ba535a 817 io_u->resid = 0;
755200a3 818 io_u->file = NULL;
d7762cf8 819 io_u->end_io = NULL;
755200a3
JA
820 }
821
822 if (io_u) {
0c6e7517
JA
823 assert(io_u->flags & IO_U_F_FREE);
824 io_u->flags &= ~IO_U_F_FREE;
825
755200a3 826 io_u->error = 0;
01743ee1
JA
827 flist_del(&io_u->list);
828 flist_add(&io_u->list, &td->io_u_busylist);
10ba535a
JA
829 td->cur_depth++;
830 }
831
832 return io_u;
833}
834
835/*
836 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
837 * etc. The returned io_u is fully ready to be prepped and submitted.
838 */
3d7c391d 839struct io_u *get_io_u(struct thread_data *td)
10ba535a 840{
3d7c391d 841 struct fio_file *f;
10ba535a
JA
842 struct io_u *io_u;
843
844 io_u = __get_io_u(td);
2ba1c290
JA
845 if (!io_u) {
846 dprint(FD_IO, "__get_io_u failed\n");
10ba535a 847 return NULL;
2ba1c290 848 }
10ba535a 849
755200a3
JA
850 /*
851 * from a requeue, io_u already setup
852 */
853 if (io_u->file)
77f392bf 854 goto out;
755200a3 855
429f6675
JA
856 /*
857 * If using an iolog, grab next piece if any available.
858 */
859 if (td->o.read_iolog_file) {
860 if (read_iolog_get(td, io_u))
861 goto err_put;
2ba1c290
JA
862 } else if (set_io_u_file(td, io_u)) {
863 dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
429f6675 864 goto err_put;
2ba1c290 865 }
5ec10eaa 866
429f6675
JA
867 f = io_u->file;
868 assert(f->flags & FIO_FILE_OPEN);
97af62ce 869
87dc1ab1 870 if (io_u->ddir != DDIR_SYNC) {
d0656a93 871 if (!io_u->buflen && !(td->io_ops->flags & FIO_NOIO)) {
2ba1c290 872 dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u);
429f6675 873 goto err_put;
2ba1c290 874 }
10ba535a 875
36167d82 876 f->last_pos = io_u->offset + io_u->buflen;
10ba535a 877
2dc1bbeb 878 if (td->o.verify != VERIFY_NONE)
87dc1ab1 879 populate_verify_io_u(td, io_u);
e4dad9c6
JA
880 else if (td->o.refill_buffers && io_u->ddir == DDIR_WRITE)
881 io_u_fill_buffer(td, io_u, io_u->xfer_buflen);
87dc1ab1 882 }
10ba535a 883
165faf16
JA
884 /*
885 * Set io data pointers.
886 */
d460eb31 887 io_u->endpos = io_u->offset + io_u->buflen;
cec6b55d
JA
888 io_u->xfer_buf = io_u->buf;
889 io_u->xfer_buflen = io_u->buflen;
5973cafb 890
6ac7a331 891out:
429f6675 892 if (!td_io_prep(td, io_u)) {
993bf48b
JA
893 if (!td->o.disable_slat)
894 fio_gettime(&io_u->start_time, NULL);
429f6675 895 return io_u;
36167d82 896 }
429f6675 897err_put:
2ba1c290 898 dprint(FD_IO, "get_io_u failed\n");
429f6675
JA
899 put_io_u(td, io_u);
900 return NULL;
10ba535a
JA
901}
902
5451792e
JA
903void io_u_log_error(struct thread_data *td, struct io_u *io_u)
904{
905 const char *msg[] = { "read", "write", "sync" };
906
907 log_err("fio: io_u error");
908
909 if (io_u->file)
910 log_err(" on file %s", io_u->file->file_name);
911
912 log_err(": %s\n", strerror(io_u->error));
913
5ec10eaa
JA
914 log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir],
915 io_u->offset, io_u->xfer_buflen);
5451792e
JA
916
917 if (!td->error)
918 td_verror(td, io_u->error, "io_u error");
919}
920
97601024
JA
921static void io_completed(struct thread_data *td, struct io_u *io_u,
922 struct io_completion_data *icd)
10ba535a 923{
dbad30b0
JA
924 /*
925 * Older gcc's are too dumb to realize that usec is always used
926 * initialized, silence that warning.
927 */
928 unsigned long uninitialized_var(usec);
10ba535a 929
2ba1c290
JA
930 dprint_io_u(io_u, "io complete");
931
0c6e7517
JA
932 assert(io_u->flags & IO_U_F_FLIGHT);
933 io_u->flags &= ~IO_U_F_FLIGHT;
934
87dc1ab1
JA
935 if (io_u->ddir == DDIR_SYNC) {
936 td->last_was_sync = 1;
937 return;
938 }
939
940 td->last_was_sync = 0;
941
10ba535a
JA
942 if (!io_u->error) {
943 unsigned int bytes = io_u->buflen - io_u->resid;
1e97cce9 944 const enum fio_ddir idx = io_u->ddir;
b29ee5b3 945 int ret;
10ba535a 946
b29ee5b3
JA
947 td->io_blocks[idx]++;
948 td->io_bytes[idx] += bytes;
949 td->this_io_bytes[idx] += bytes;
10ba535a 950
b29ee5b3 951 if (ramp_time_over(td)) {
9520ebb9
JA
952 if (!td->o.disable_clat || !td->o.disable_bw)
953 usec = utime_since(&io_u->issue_time,
954 &icd->time);
721938ae 955
9520ebb9
JA
956 if (!td->o.disable_clat) {
957 add_clat_sample(td, idx, usec);
958 io_u_mark_latency(td, usec);
959 }
960 if (!td->o.disable_bw)
961 add_bw_sample(td, idx, &icd->time);
721938ae 962 }
10ba535a 963
660a1cb5 964 if (td_write(td) && idx == DDIR_WRITE &&
8670579e 965 td->o.do_verify &&
41128405 966 td->o.verify != VERIFY_NONE)
10ba535a
JA
967 log_io_piece(td, io_u);
968
b29ee5b3 969 icd->bytes_done[idx] += bytes;
3af6ef39 970
d7762cf8 971 if (io_u->end_io) {
36690c9b 972 ret = io_u->end_io(td, io_u);
3af6ef39
JA
973 if (ret && !icd->error)
974 icd->error = ret;
975 }
5451792e 976 } else {
10ba535a 977 icd->error = io_u->error;
5451792e
JA
978 io_u_log_error(td, io_u);
979 }
10ba535a
JA
980}
981
9520ebb9
JA
982static void init_icd(struct thread_data *td, struct io_completion_data *icd,
983 int nr)
10ba535a 984{
9520ebb9
JA
985 if (!td->o.disable_clat || !td->o.disable_bw)
986 fio_gettime(&icd->time, NULL);
02bcaa8c 987
3af6ef39
JA
988 icd->nr = nr;
989
10ba535a
JA
990 icd->error = 0;
991 icd->bytes_done[0] = icd->bytes_done[1] = 0;
36167d82
JA
992}
993
97601024
JA
994static void ios_completed(struct thread_data *td,
995 struct io_completion_data *icd)
36167d82
JA
996{
997 struct io_u *io_u;
998 int i;
999
10ba535a
JA
1000 for (i = 0; i < icd->nr; i++) {
1001 io_u = td->io_ops->event(td, i);
1002
1003 io_completed(td, io_u, icd);
1004 put_io_u(td, io_u);
1005 }
1006}
97601024 1007
e7e6cfb4
JA
1008/*
1009 * Complete a single io_u for the sync engines.
1010 */
d7762cf8 1011long io_u_sync_complete(struct thread_data *td, struct io_u *io_u)
97601024
JA
1012{
1013 struct io_completion_data icd;
1014
9520ebb9 1015 init_icd(td, &icd, 1);
97601024
JA
1016 io_completed(td, io_u, &icd);
1017 put_io_u(td, io_u);
1018
1019 if (!icd.error)
1020 return icd.bytes_done[0] + icd.bytes_done[1];
1021
37e974a5 1022 td_verror(td, icd.error, "io_u_sync_complete");
97601024
JA
1023 return -1;
1024}
1025
e7e6cfb4
JA
1026/*
1027 * Called to complete min_events number of io for the async engines.
1028 */
4950421a 1029long io_u_queued_complete(struct thread_data *td, int min_evts)
97601024 1030{
97601024 1031 struct io_completion_data icd;
00de55ef 1032 struct timespec *tvp = NULL;
97601024 1033 int ret;
4d06a338 1034 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
97601024 1035
4950421a 1036 dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_evts);
b271fe62 1037
4950421a 1038 if (!min_evts)
00de55ef 1039 tvp = &ts;
97601024 1040
4950421a 1041 ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete, tvp);
97601024 1042 if (ret < 0) {
e1161c32 1043 td_verror(td, -ret, "td_io_getevents");
97601024
JA
1044 return ret;
1045 } else if (!ret)
1046 return ret;
1047
9520ebb9 1048 init_icd(td, &icd, ret);
97601024
JA
1049 ios_completed(td, &icd);
1050 if (!icd.error)
1051 return icd.bytes_done[0] + icd.bytes_done[1];
1052
37e974a5 1053 td_verror(td, icd.error, "io_u_queued_complete");
97601024
JA
1054 return -1;
1055}
7e77dd02
JA
1056
1057/*
1058 * Call when io_u is really queued, to update the submission latency.
1059 */
1060void io_u_queued(struct thread_data *td, struct io_u *io_u)
1061{
9520ebb9
JA
1062 if (!td->o.disable_slat) {
1063 unsigned long slat_time;
7e77dd02 1064
9520ebb9
JA
1065 slat_time = utime_since(&io_u->start_time, &io_u->issue_time);
1066 add_slat_sample(td, io_u->ddir, slat_time);
1067 }
7e77dd02 1068}
433afcb4 1069
5973cafb
JA
1070/*
1071 * "randomly" fill the buffer contents
1072 */
1073void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
1074 unsigned int max_bs)
1075{
1076 long *ptr = io_u->buf;
1077
1078 if (!td->o.zero_buffers) {
1079 while ((void *) ptr - io_u->buf < max_bs) {
1080 *ptr = rand() * GOLDEN_RATIO_PRIME;
1081 ptr++;
1082 }
1083 } else
1084 memset(ptr, 0, max_bs);
1085}