Merge branch 'master' of https://github.com/bvanassche/fio
[fio.git] / io_u.c
CommitLineData
10ba535a 1#include <unistd.h>
10ba535a 2#include <string.h>
0c6e7517 3#include <assert.h>
10ba535a
JA
4
5#include "fio.h"
4f5af7b2 6#include "verify.h"
0d29de83 7#include "trim.h"
1fbbf72e 8#include "lib/rand.h"
7ebd796f 9#include "lib/axmap.h"
002fe734 10#include "err.h"
0f38bbef 11#include "lib/pow2.h"
4b157ac6 12#include "minmax.h"
bfbdd35b 13#include "zbd.h"
10ba535a 14
97601024
JA
15struct io_completion_data {
16 int nr; /* input */
97601024
JA
17
18 int error; /* output */
100f49f1 19 uint64_t bytes_done[DDIR_RWDIR_CNT]; /* output */
8b6a404c 20 struct timespec time; /* output */
97601024
JA
21};
22
10ba535a 23/*
7ebd796f 24 * The ->io_axmap contains a map of blocks we have or have not done io
10ba535a
JA
25 * to yet. Used to make sure we cover the entire range in a fair fashion.
26 */
e39c0676 27static bool random_map_free(struct fio_file *f, const uint64_t block)
10ba535a 28{
7ebd796f 29 return !axmap_isset(f->io_axmap, block);
10ba535a
JA
30}
31
df415585
JA
32/*
33 * Mark a given offset as used in the map.
34 */
6cc1a3d1
BVA
35static uint64_t mark_random_map(struct thread_data *td, struct io_u *io_u,
36 uint64_t offset, uint64_t buflen)
df415585 37{
5fff9543 38 unsigned long long min_bs = td->o.min_bs[io_u->ddir];
9bf2061e 39 struct fio_file *f = io_u->file;
5fff9543 40 unsigned long long nr_blocks;
1ae83d45 41 uint64_t block;
df415585 42
6cc1a3d1
BVA
43 block = (offset - f->file_offset) / (uint64_t) min_bs;
44 nr_blocks = (buflen + min_bs - 1) / min_bs;
bd6b959a 45 assert(nr_blocks > 0);
df415585 46
bd6b959a 47 if (!(io_u->flags & IO_U_F_BUSY_OK)) {
7ebd796f 48 nr_blocks = axmap_set_nr(f->io_axmap, block, nr_blocks);
bd6b959a
BVA
49 assert(nr_blocks > 0);
50 }
df415585 51
6cc1a3d1
BVA
52 if ((nr_blocks * min_bs) < buflen)
53 buflen = nr_blocks * min_bs;
54
55 return buflen;
df415585
JA
56}
57
74776733
JA
58static uint64_t last_block(struct thread_data *td, struct fio_file *f,
59 enum fio_ddir ddir)
2ba1c290 60{
74776733
JA
61 uint64_t max_blocks;
62 uint64_t max_size;
2ba1c290 63
ff58fced
JA
64 assert(ddir_rw(ddir));
65
d9dd70f7
JA
66 /*
67 * Hmm, should we make sure that ->io_size <= ->real_file_size?
79591fa9 68 * -> not for now since there is code assuming it could go either.
d9dd70f7
JA
69 */
70 max_size = f->io_size;
71 if (max_size > f->real_file_size)
72 max_size = f->real_file_size;
73
7b865a2f 74 if (td->o.zone_mode == ZONE_MODE_STRIDED && td->o.zone_range)
ed335855
SN
75 max_size = td->o.zone_range;
76
0412d12e
JE
77 if (td->o.min_bs[ddir] > td->o.ba[ddir])
78 max_size -= td->o.min_bs[ddir] - td->o.ba[ddir];
79
1ae83d45 80 max_blocks = max_size / (uint64_t) td->o.ba[ddir];
2ba1c290
JA
81 if (!max_blocks)
82 return 0;
83
67778e88 84 return max_blocks;
2ba1c290
JA
85}
86
e25839d4 87static int __get_next_rand_offset(struct thread_data *td, struct fio_file *f,
e0a04ac1
JA
88 enum fio_ddir ddir, uint64_t *b,
89 uint64_t lastb)
ec4015da 90{
6f49f8bc 91 uint64_t r;
5e0baa7f 92
c3546b53
JA
93 if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE ||
94 td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64) {
6f49f8bc 95
d6b72507 96 r = __rand(&td->random_state);
8055e41d 97
4b91ee8f 98 dprint(FD_RANDOM, "off rand %llu\n", (unsigned long long) r);
8055e41d 99
e0a04ac1 100 *b = lastb * (r / (rand_max(&td->random_state) + 1.0));
51ede0b1 101 } else {
8055e41d 102 uint64_t off = 0;
43c63a78 103
967d1b63
JA
104 assert(fio_file_lfsr(f));
105
6f49f8bc 106 if (lfsr_next(&f->lfsr, &off))
8055e41d 107 return 1;
ec4015da 108
8055e41d
JA
109 *b = off;
110 }
0ce8b119 111
ec4015da 112 /*
51ede0b1 113 * if we are not maintaining a random map, we are done.
ec4015da 114 */
51ede0b1
JA
115 if (!file_randommap(td, f))
116 goto ret;
43c63a78
JA
117
118 /*
51ede0b1 119 * calculate map offset and check if it's free
43c63a78 120 */
51ede0b1
JA
121 if (random_map_free(f, *b))
122 goto ret;
123
4b91ee8f
JA
124 dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
125 (unsigned long long) *b);
51ede0b1 126
7ebd796f 127 *b = axmap_next_free(f->io_axmap, *b);
51ede0b1
JA
128 if (*b == (uint64_t) -1ULL)
129 return 1;
0ce8b119
JA
130ret:
131 return 0;
ec4015da
JA
132}
133
925fee33
JA
134static int __get_next_rand_offset_zipf(struct thread_data *td,
135 struct fio_file *f, enum fio_ddir ddir,
1ae83d45 136 uint64_t *b)
e25839d4 137{
9c6f6316 138 *b = zipf_next(&f->zipf);
e25839d4
JA
139 return 0;
140}
141
925fee33
JA
142static int __get_next_rand_offset_pareto(struct thread_data *td,
143 struct fio_file *f, enum fio_ddir ddir,
1ae83d45 144 uint64_t *b)
925fee33 145{
9c6f6316 146 *b = pareto_next(&f->zipf);
925fee33
JA
147 return 0;
148}
149
56d9fa4b
JA
150static int __get_next_rand_offset_gauss(struct thread_data *td,
151 struct fio_file *f, enum fio_ddir ddir,
152 uint64_t *b)
153{
154 *b = gauss_next(&f->gauss);
155 return 0;
156}
157
59466396
JA
158static int __get_next_rand_offset_zoned_abs(struct thread_data *td,
159 struct fio_file *f,
160 enum fio_ddir ddir, uint64_t *b)
161{
162 struct zone_split_index *zsi;
e3c8c108 163 uint64_t lastb, send, stotal;
59466396
JA
164 unsigned int v;
165
166 lastb = last_block(td, f, ddir);
167 if (!lastb)
168 return 1;
169
170 if (!td->o.zone_split_nr[ddir]) {
171bail:
172 return __get_next_rand_offset(td, f, ddir, b, lastb);
173 }
174
175 /*
176 * Generate a value, v, between 1 and 100, both inclusive
177 */
1bd5d213 178 v = rand_between(&td->zone_state, 1, 100);
59466396 179
e3c8c108
JA
180 /*
181 * Find our generated table. 'send' is the end block of this zone,
182 * 'stotal' is our start offset.
183 */
59466396
JA
184 zsi = &td->zone_state_index[ddir][v - 1];
185 stotal = zsi->size_prev / td->o.ba[ddir];
186 send = zsi->size / td->o.ba[ddir];
187
188 /*
189 * Should never happen
190 */
191 if (send == -1U) {
264e3d30 192 if (!fio_did_warn(FIO_WARN_ZONED_BUG))
59466396 193 log_err("fio: bug in zoned generation\n");
59466396
JA
194 goto bail;
195 } else if (send > lastb) {
196 /*
197 * This happens if the user specifies ranges that exceed
198 * the file/device size. We can't handle that gracefully,
199 * so error and exit.
200 */
201 log_err("fio: zoned_abs sizes exceed file size\n");
202 return 1;
203 }
204
205 /*
e3c8c108 206 * Generate index from 0..send-stotal
59466396 207 */
e3c8c108 208 if (__get_next_rand_offset(td, f, ddir, b, send - stotal) == 1)
59466396
JA
209 return 1;
210
e3c8c108 211 *b += stotal;
59466396
JA
212 return 0;
213}
214
e0a04ac1
JA
215static int __get_next_rand_offset_zoned(struct thread_data *td,
216 struct fio_file *f, enum fio_ddir ddir,
217 uint64_t *b)
218{
219 unsigned int v, send, stotal;
220 uint64_t offset, lastb;
e0a04ac1
JA
221 struct zone_split_index *zsi;
222
223 lastb = last_block(td, f, ddir);
224 if (!lastb)
225 return 1;
226
227 if (!td->o.zone_split_nr[ddir]) {
228bail:
229 return __get_next_rand_offset(td, f, ddir, b, lastb);
230 }
231
232 /*
233 * Generate a value, v, between 1 and 100, both inclusive
234 */
1bd5d213 235 v = rand_between(&td->zone_state, 1, 100);
e0a04ac1
JA
236
237 zsi = &td->zone_state_index[ddir][v - 1];
238 stotal = zsi->size_perc_prev;
239 send = zsi->size_perc;
240
241 /*
242 * Should never happen
243 */
244 if (send == -1U) {
264e3d30 245 if (!fio_did_warn(FIO_WARN_ZONED_BUG))
e0a04ac1 246 log_err("fio: bug in zoned generation\n");
e0a04ac1
JA
247 goto bail;
248 }
249
250 /*
251 * 'send' is some percentage below or equal to 100 that
252 * marks the end of the current IO range. 'stotal' marks
253 * the start, in percent.
254 */
255 if (stotal)
256 offset = stotal * lastb / 100ULL;
257 else
258 offset = 0;
259
260 lastb = lastb * (send - stotal) / 100ULL;
261
262 /*
263 * Generate index from 0..send-of-lastb
264 */
265 if (__get_next_rand_offset(td, f, ddir, b, lastb) == 1)
266 return 1;
267
268 /*
269 * Add our start offset, if any
270 */
271 if (offset)
272 *b += offset;
273
274 return 0;
275}
56d9fa4b 276
f31feaa2
JA
277static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
278 enum fio_ddir ddir, uint64_t *b)
e25839d4 279{
e0a04ac1
JA
280 if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) {
281 uint64_t lastb;
282
283 lastb = last_block(td, f, ddir);
284 if (!lastb)
285 return 1;
286
287 return __get_next_rand_offset(td, f, ddir, b, lastb);
288 } else if (td->o.random_distribution == FIO_RAND_DIST_ZIPF)
e25839d4 289 return __get_next_rand_offset_zipf(td, f, ddir, b);
925fee33
JA
290 else if (td->o.random_distribution == FIO_RAND_DIST_PARETO)
291 return __get_next_rand_offset_pareto(td, f, ddir, b);
56d9fa4b
JA
292 else if (td->o.random_distribution == FIO_RAND_DIST_GAUSS)
293 return __get_next_rand_offset_gauss(td, f, ddir, b);
e0a04ac1
JA
294 else if (td->o.random_distribution == FIO_RAND_DIST_ZONED)
295 return __get_next_rand_offset_zoned(td, f, ddir, b);
59466396
JA
296 else if (td->o.random_distribution == FIO_RAND_DIST_ZONED_ABS)
297 return __get_next_rand_offset_zoned_abs(td, f, ddir, b);
e25839d4
JA
298
299 log_err("fio: unknown random distribution: %d\n", td->o.random_distribution);
300 return 1;
301}
302
e39c0676 303static bool should_do_random(struct thread_data *td, enum fio_ddir ddir)
211c9b89
JA
304{
305 unsigned int v;
211c9b89 306
d9472271 307 if (td->o.perc_rand[ddir] == 100)
e39c0676 308 return true;
211c9b89 309
1bd5d213 310 v = rand_between(&td->seq_rand_state[ddir], 1, 100);
211c9b89 311
d9472271 312 return v <= td->o.perc_rand[ddir];
211c9b89
JA
313}
314
0bcf41cd
JA
315static void loop_cache_invalidate(struct thread_data *td, struct fio_file *f)
316{
317 struct thread_options *o = &td->o;
318
319 if (o->invalidate_cache && !o->odirect) {
320 int fio_unused ret;
321
322 ret = file_invalidate_cache(td, f);
323 }
324}
325
38dad62d 326static int get_next_rand_block(struct thread_data *td, struct fio_file *f,
1ae83d45 327 enum fio_ddir ddir, uint64_t *b)
38dad62d 328{
c04e4661
DE
329 if (!get_next_rand_offset(td, f, ddir, b))
330 return 0;
331
8c07860d
JA
332 if (td->o.time_based ||
333 (td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM)) {
33c48814 334 fio_file_reset(td, f);
80f02150 335 loop_cache_invalidate(td, f);
c04e4661
DE
336 if (!get_next_rand_offset(td, f, ddir, b))
337 return 0;
38dad62d
JA
338 }
339
c04e4661 340 dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n",
f1dfb668 341 f->file_name, (unsigned long long) f->last_pos[ddir],
4b91ee8f 342 (unsigned long long) f->real_file_size);
c04e4661 343 return 1;
38dad62d
JA
344}
345
37cf9e3c 346static int get_next_seq_offset(struct thread_data *td, struct fio_file *f,
1ae83d45 347 enum fio_ddir ddir, uint64_t *offset)
38dad62d 348{
ac002339
JA
349 struct thread_options *o = &td->o;
350
ff58fced
JA
351 assert(ddir_rw(ddir));
352
17373ce2
JA
353 /*
354 * If we reach the end for a time based run, reset us back to 0
355 * and invalidate the cache, if we need to.
356 */
f1dfb668 357 if (f->last_pos[ddir] >= f->io_size + get_start_offset(td, f) &&
19ddc35b 358 o->time_based) {
c89daa4a 359 f->last_pos[ddir] = f->file_offset;
0bcf41cd 360 loop_cache_invalidate(td, f);
19ddc35b 361 }
c04e4661 362
f1dfb668 363 if (f->last_pos[ddir] < f->real_file_size) {
1ae83d45 364 uint64_t pos;
059b0802 365
69b98f11
JA
366 /*
367 * Only rewind if we already hit the end
368 */
369 if (f->last_pos[ddir] == f->file_offset &&
370 f->file_offset && o->ddir_seq_add < 0) {
c22825bb
JA
371 if (f->real_file_size > f->io_size)
372 f->last_pos[ddir] = f->io_size;
373 else
374 f->last_pos[ddir] = f->real_file_size;
375 }
a66da7a2 376
f1dfb668 377 pos = f->last_pos[ddir] - f->file_offset;
ac002339
JA
378 if (pos && o->ddir_seq_add) {
379 pos += o->ddir_seq_add;
380
381 /*
382 * If we reach beyond the end of the file
383 * with holed IO, wrap around to the
b0a84f48
JA
384 * beginning again. If we're doing backwards IO,
385 * wrap to the end.
ac002339 386 */
b0a84f48
JA
387 if (pos >= f->real_file_size) {
388 if (o->ddir_seq_add > 0)
389 pos = f->file_offset;
c22825bb
JA
390 else {
391 if (f->real_file_size > f->io_size)
392 pos = f->io_size;
393 else
394 pos = f->real_file_size;
395
396 pos += o->ddir_seq_add;
397 }
b0a84f48 398 }
ac002339 399 }
059b0802 400
37cf9e3c 401 *offset = pos;
38dad62d
JA
402 return 0;
403 }
404
405 return 1;
406}
407
408static int get_next_block(struct thread_data *td, struct io_u *io_u,
6aca9b3d 409 enum fio_ddir ddir, int rw_seq,
ec370c22 410 bool *is_random)
38dad62d
JA
411{
412 struct fio_file *f = io_u->file;
1ae83d45 413 uint64_t b, offset;
38dad62d
JA
414 int ret;
415
ff58fced
JA
416 assert(ddir_rw(ddir));
417
37cf9e3c
JA
418 b = offset = -1ULL;
419
38dad62d 420 if (rw_seq) {
211c9b89 421 if (td_random(td)) {
6aca9b3d 422 if (should_do_random(td, ddir)) {
211c9b89 423 ret = get_next_rand_block(td, f, ddir, &b);
ec370c22 424 *is_random = true;
6aca9b3d 425 } else {
ec370c22 426 *is_random = false;
1651e431 427 io_u_set(td, io_u, IO_U_F_BUSY_OK);
211c9b89
JA
428 ret = get_next_seq_offset(td, f, ddir, &offset);
429 if (ret)
430 ret = get_next_rand_block(td, f, ddir, &b);
431 }
6aca9b3d 432 } else {
ec370c22 433 *is_random = false;
37cf9e3c 434 ret = get_next_seq_offset(td, f, ddir, &offset);
6aca9b3d 435 }
38dad62d 436 } else {
1651e431 437 io_u_set(td, io_u, IO_U_F_BUSY_OK);
ec370c22 438 *is_random = false;
38dad62d
JA
439
440 if (td->o.rw_seq == RW_SEQ_SEQ) {
37cf9e3c 441 ret = get_next_seq_offset(td, f, ddir, &offset);
6aca9b3d 442 if (ret) {
37cf9e3c 443 ret = get_next_rand_block(td, f, ddir, &b);
ec370c22 444 *is_random = false;
6aca9b3d 445 }
38dad62d 446 } else if (td->o.rw_seq == RW_SEQ_IDENT) {
f1dfb668
JA
447 if (f->last_start[ddir] != -1ULL)
448 offset = f->last_start[ddir] - f->file_offset;
38dad62d 449 else
37cf9e3c 450 offset = 0;
38dad62d
JA
451 ret = 0;
452 } else {
453 log_err("fio: unknown rw_seq=%d\n", td->o.rw_seq);
454 ret = 1;
455 }
456 }
6d68b997 457
37cf9e3c
JA
458 if (!ret) {
459 if (offset != -1ULL)
460 io_u->offset = offset;
461 else if (b != -1ULL)
462 io_u->offset = b * td->o.ba[ddir];
463 else {
4e0a8fa2 464 log_err("fio: bug in offset generation: offset=%llu, b=%llu\n", (unsigned long long) offset, (unsigned long long) b);
37cf9e3c
JA
465 ret = 1;
466 }
4fff54cc 467 io_u->verify_offset = io_u->offset;
37cf9e3c
JA
468 }
469
38dad62d
JA
470 return ret;
471}
472
10ba535a
JA
473/*
474 * For random io, generate a random new block and see if it's used. Repeat
475 * until we find a free one. For sequential io, just return the end of
476 * the last io issued.
477 */
e8fb335e 478static int get_next_offset(struct thread_data *td, struct io_u *io_u,
ec370c22 479 bool *is_random)
10ba535a 480{
9bf2061e 481 struct fio_file *f = io_u->file;
4ba66134 482 enum fio_ddir ddir = io_u->ddir;
38dad62d 483 int rw_seq_hit = 0;
10ba535a 484
ff58fced
JA
485 assert(ddir_rw(ddir));
486
38dad62d
JA
487 if (td->o.ddir_seq_nr && !--td->ddir_seq_nr) {
488 rw_seq_hit = 1;
5736c10d 489 td->ddir_seq_nr = td->o.ddir_seq_nr;
38dad62d 490 }
211097b2 491
6aca9b3d 492 if (get_next_block(td, io_u, ddir, rw_seq_hit, is_random))
38dad62d 493 return 1;
10ba535a 494
009bd847
JA
495 if (io_u->offset >= f->io_size) {
496 dprint(FD_IO, "get_next_offset: offset %llu >= io_size %llu\n",
4b91ee8f
JA
497 (unsigned long long) io_u->offset,
498 (unsigned long long) f->io_size);
009bd847
JA
499 return 1;
500 }
501
502 io_u->offset += f->file_offset;
2ba1c290
JA
503 if (io_u->offset >= f->real_file_size) {
504 dprint(FD_IO, "get_next_offset: offset %llu >= size %llu\n",
4b91ee8f
JA
505 (unsigned long long) io_u->offset,
506 (unsigned long long) f->real_file_size);
10ba535a 507 return 1;
2ba1c290 508 }
10ba535a 509
9bf46532 510 io_u->verify_offset = io_u->offset;
10ba535a
JA
511 return 0;
512}
513
e39c0676 514static inline bool io_u_fits(struct thread_data *td, struct io_u *io_u,
5fff9543 515 unsigned long long buflen)
79944128
JA
516{
517 struct fio_file *f = io_u->file;
518
bedc9dc2 519 return io_u->offset + buflen <= f->io_size + get_start_offset(td, f);
79944128
JA
520}
521
5fff9543 522static unsigned long long get_next_buflen(struct thread_data *td, struct io_u *io_u,
ec370c22 523 bool is_random)
10ba535a 524{
6aca9b3d 525 int ddir = io_u->ddir;
5fff9543
JF
526 unsigned long long buflen = 0;
527 unsigned long long minbs, maxbs;
3dd29f7c 528 uint64_t frand_max, r;
7c961359 529 bool power_2;
10ba535a 530
9ee1c647 531 assert(ddir_rw(ddir));
6aca9b3d
JA
532
533 if (td->o.bs_is_seq_rand)
ec370c22 534 ddir = is_random ? DDIR_WRITE : DDIR_READ;
ff58fced 535
f3059de1
JA
536 minbs = td->o.min_bs[ddir];
537 maxbs = td->o.max_bs[ddir];
538
79944128
JA
539 if (minbs == maxbs)
540 return minbs;
541
52c58027
JA
542 /*
543 * If we can't satisfy the min block size from here, then fail
544 */
545 if (!io_u_fits(td, io_u, minbs))
546 return 0;
547
2f282cec 548 frand_max = rand_max(&td->bsrange_state[ddir]);
79944128 549 do {
2f282cec 550 r = __rand(&td->bsrange_state[ddir]);
4c07ad86 551
720e84ad 552 if (!td->o.bssplit_nr[ddir]) {
5fff9543 553 buflen = minbs + (unsigned long long) ((double) maxbs *
c3546b53 554 (r / (frand_max + 1.0)));
5ec10eaa 555 } else {
3dd29f7c 556 long long perc = 0;
564ca972
JA
557 unsigned int i;
558
720e84ad
JA
559 for (i = 0; i < td->o.bssplit_nr[ddir]; i++) {
560 struct bssplit *bsp = &td->o.bssplit[ddir][i];
564ca972 561
cc5e4cb0
DLM
562 if (!bsp->perc)
563 continue;
564ca972
JA
564 buflen = bsp->bs;
565 perc += bsp->perc;
3dd29f7c 566 if ((r / perc <= frand_max / 100ULL) &&
79944128 567 io_u_fits(td, io_u, buflen))
564ca972
JA
568 break;
569 }
570 }
79944128 571
17a6b702
PL
572 power_2 = is_power_of_2(minbs);
573 if (!td->o.bs_unaligned && power_2)
7c306bb1 574 buflen &= ~(minbs - 1);
8135fe4d
JA
575 else if (!td->o.bs_unaligned && !power_2)
576 buflen -= buflen % minbs;
577 if (buflen > maxbs)
578 buflen = maxbs;
79944128 579 } while (!io_u_fits(td, io_u, buflen));
6a5e6884 580
10ba535a
JA
581 return buflen;
582}
583
afe24a5a
JA
584static void set_rwmix_bytes(struct thread_data *td)
585{
afe24a5a
JA
586 unsigned int diff;
587
588 /*
589 * we do time or byte based switch. this is needed because
590 * buffered writes may issue a lot quicker than they complete,
591 * whereas reads do not.
592 */
e47f799f 593 diff = td->o.rwmix[td->rwmix_ddir ^ 1];
04c540d9 594 td->rwmix_issues = (td->io_issues[td->rwmix_ddir] * diff) / 100;
e47f799f
JA
595}
596
597static inline enum fio_ddir get_rand_ddir(struct thread_data *td)
598{
599 unsigned int v;
e47f799f 600
1bd5d213 601 v = rand_between(&td->rwmix_state, 1, 100);
4c07ad86 602
04c540d9 603 if (v <= td->o.rwmix[DDIR_READ])
e47f799f
JA
604 return DDIR_READ;
605
606 return DDIR_WRITE;
afe24a5a
JA
607}
608
0d593542 609int io_u_quiesce(struct thread_data *td)
002e7183 610{
2dcfdeb9 611 int ret = 0, completed = 0, err = 0;
0d593542 612
002e7183
JA
613 /*
614 * We are going to sleep, ensure that we flush anything pending as
615 * not to skew our latency numbers.
616 *
617 * Changed to only monitor 'in flight' requests here instead of the
618 * td->cur_depth, b/c td->cur_depth does not accurately represent
619 * io's that have been actually submitted to an async engine,
620 * and cur_depth is meaningless for sync engines.
621 */
a80cb54b
BVA
622 if (td->io_u_queued || td->cur_depth)
623 td_io_commit(td);
9cc80b6d 624
002e7183 625 while (td->io_u_in_flight) {
55312f9f 626 ret = io_u_queued_complete(td, 1);
0d593542
JA
627 if (ret > 0)
628 completed += ret;
d28174f0 629 else if (ret < 0)
2dcfdeb9 630 err = ret;
002e7183 631 }
0d593542 632
6be06c46
JA
633 if (td->flags & TD_F_REGROW_LOGS)
634 regrow_logs(td);
635
d28174f0
JA
636 if (completed)
637 return completed;
638
2dcfdeb9 639 return err;
002e7183
JA
640}
641
581e7141
JA
642static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)
643{
644 enum fio_ddir odir = ddir ^ 1;
b5407f8b 645 uint64_t usec;
90eff1c9 646 uint64_t now;
581e7141 647
ff58fced 648 assert(ddir_rw(ddir));
970d9000 649 now = utime_since_now(&td->epoch);
ff58fced 650
50a8ce86
D
651 /*
652 * if rate_next_io_time is in the past, need to catch up to rate
653 */
654 if (td->rate_next_io_time[ddir] <= now)
581e7141
JA
655 return ddir;
656
657 /*
50a8ce86 658 * We are ahead of rate in this direction. See if we
581e7141
JA
659 * should switch.
660 */
315fcfec 661 if (td_rw(td) && td->o.rwmix[odir]) {
581e7141 662 /*
50a8ce86 663 * Other direction is behind rate, switch
581e7141 664 */
50a8ce86 665 if (td->rate_next_io_time[odir] <= now)
581e7141
JA
666 return odir;
667
668 /*
395feabb
JA
669 * Both directions are ahead of rate. sleep the min,
670 * switch if necessary
581e7141 671 */
50a8ce86 672 if (td->rate_next_io_time[ddir] <=
395feabb 673 td->rate_next_io_time[odir]) {
50a8ce86 674 usec = td->rate_next_io_time[ddir] - now;
581e7141 675 } else {
50a8ce86 676 usec = td->rate_next_io_time[odir] - now;
581e7141
JA
677 ddir = odir;
678 }
679 } else
50a8ce86 680 usec = td->rate_next_io_time[ddir] - now;
581e7141 681
a9da8ab2
JA
682 if (td->o.io_submit_mode == IO_MODE_INLINE)
683 io_u_quiesce(td);
78c1eda5 684
df06a036 685 if (td->o.timeout && ((usec + now) > td->o.timeout)) {
686 /*
687 * check if the usec is capable of taking negative values
688 */
689 if (now > td->o.timeout) {
690 ddir = DDIR_INVAL;
691 return ddir;
692 }
693 usec = td->o.timeout - now;
694 }
1a9bf814 695 usec_sleep(td, usec);
df06a036 696
697 now = utime_since_now(&td->epoch);
698 if ((td->o.timeout && (now > td->o.timeout)) || td->terminate)
699 ddir = DDIR_INVAL;
700
581e7141
JA
701 return ddir;
702}
703
10ba535a
JA
704/*
705 * Return the data direction for the next io_u. If the job is a
706 * mixed read/write workload, check the rwmix cycle and switch if
707 * necessary.
708 */
1e97cce9 709static enum fio_ddir get_rw_ddir(struct thread_data *td)
10ba535a 710{
581e7141
JA
711 enum fio_ddir ddir;
712
5f9099ea 713 /*
f6860972
TK
714 * See if it's time to fsync/fdatasync/sync_file_range first,
715 * and if not then move on to check regular I/Os.
5f9099ea 716 */
f6860972
TK
717 if (should_fsync(td)) {
718 if (td->o.fsync_blocks && td->io_issues[DDIR_WRITE] &&
719 !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks))
720 return DDIR_SYNC;
721
722 if (td->o.fdatasync_blocks && td->io_issues[DDIR_WRITE] &&
723 !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks))
724 return DDIR_DATASYNC;
725
726 if (td->sync_file_range_nr && td->io_issues[DDIR_WRITE] &&
727 !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr))
728 return DDIR_SYNC_FILE_RANGE;
729 }
44f29692 730
10ba535a 731 if (td_rw(td)) {
10ba535a
JA
732 /*
733 * Check if it's time to seed a new data direction.
734 */
e4928662 735 if (td->io_issues[td->rwmix_ddir] >= td->rwmix_issues) {
e47f799f
JA
736 /*
737 * Put a top limit on how many bytes we do for
738 * one data direction, to avoid overflowing the
739 * ranges too much
740 */
741 ddir = get_rand_ddir(td);
e47f799f
JA
742
743 if (ddir != td->rwmix_ddir)
744 set_rwmix_bytes(td);
745
746 td->rwmix_ddir = ddir;
10ba535a 747 }
581e7141 748 ddir = td->rwmix_ddir;
10ba535a 749 } else if (td_read(td))
581e7141 750 ddir = DDIR_READ;
6eaf09d6 751 else if (td_write(td))
581e7141 752 ddir = DDIR_WRITE;
5c8e84ca 753 else if (td_trim(td))
6eaf09d6 754 ddir = DDIR_TRIM;
5c8e84ca
TK
755 else
756 ddir = DDIR_INVAL;
581e7141
JA
757
758 td->rwmix_ddir = rate_ddir(td, ddir);
759 return td->rwmix_ddir;
10ba535a
JA
760}
761
1ef2b6be
JA
762static void set_rw_ddir(struct thread_data *td, struct io_u *io_u)
763{
0e4dd95c
DE
764 enum fio_ddir ddir = get_rw_ddir(td);
765
731461cc
DLM
766 if (td->o.zone_mode == ZONE_MODE_ZBD)
767 ddir = zbd_adjust_ddir(td, io_u, ddir);
c65057f9 768
82a90686 769 if (td_trimwrite(td)) {
0e4dd95c
DE
770 struct fio_file *f = io_u->file;
771 if (f->last_pos[DDIR_WRITE] == f->last_pos[DDIR_TRIM])
772 ddir = DDIR_TRIM;
773 else
774 ddir = DDIR_WRITE;
775 }
776
777 io_u->ddir = io_u->acct_ddir = ddir;
1ef2b6be 778
9b87f09b 779 if (io_u->ddir == DDIR_WRITE && td_ioengine_flagged(td, FIO_BARRIER) &&
1ef2b6be
JA
780 td->o.barrier_blocks &&
781 !(td->io_issues[DDIR_WRITE] % td->o.barrier_blocks) &&
782 td->io_issues[DDIR_WRITE])
1651e431 783 io_u_set(td, io_u, IO_U_F_BARRIER);
1ef2b6be
JA
784}
785
e8462bd8 786void put_file_log(struct thread_data *td, struct fio_file *f)
60f2c658 787{
71b84caa 788 unsigned int ret = put_file(td, f);
60f2c658
JA
789
790 if (ret)
791 td_verror(td, ret, "file close");
792}
793
10ba535a
JA
794void put_io_u(struct thread_data *td, struct io_u *io_u)
795{
26b3a188
JA
796 const bool needs_lock = td_async_processing(td);
797
b2da58c4 798 zbd_put_io_u(td, io_u);
99952ca7 799
a9da8ab2
JA
800 if (td->parent)
801 td = td->parent;
802
26b3a188
JA
803 if (needs_lock)
804 __td_io_u_lock(td);
e8462bd8 805
f8b0bd10 806 if (io_u->file && !(io_u->flags & IO_U_F_NO_FILE_PUT))
60f2c658 807 put_file_log(td, io_u->file);
f8b0bd10 808
10ba535a 809 io_u->file = NULL;
1651e431 810 io_u_set(td, io_u, IO_U_F_FREE);
d7ee2a7d 811
a9da8ab2 812 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
0c41214f 813 td->cur_depth--;
a9da8ab2
JA
814 assert(!(td->flags & TD_F_CHILD));
815 }
2ae0b204 816 io_u_qpush(&td->io_u_freelist, io_u);
e8462bd8 817 td_io_u_free_notify(td);
26b3a188
JA
818
819 if (needs_lock)
820 __td_io_u_unlock(td);
10ba535a
JA
821}
822
f2bba182
RR
823void clear_io_u(struct thread_data *td, struct io_u *io_u)
824{
1651e431 825 io_u_clear(td, io_u, IO_U_F_FLIGHT);
f2bba182
RR
826 put_io_u(td, io_u);
827}
828
755200a3
JA
829void requeue_io_u(struct thread_data *td, struct io_u **io_u)
830{
26b3a188 831 const bool needs_lock = td_async_processing(td);
755200a3 832 struct io_u *__io_u = *io_u;
bcd5abfa 833 enum fio_ddir ddir = acct_ddir(__io_u);
755200a3 834
465221b0
JA
835 dprint(FD_IO, "requeue %p\n", __io_u);
836
a9da8ab2
JA
837 if (td->parent)
838 td = td->parent;
839
26b3a188
JA
840 if (needs_lock)
841 __td_io_u_lock(td);
e8462bd8 842
1651e431 843 io_u_set(td, __io_u, IO_U_F_FREE);
bcd5abfa
JA
844 if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir))
845 td->io_issues[ddir]--;
5ec10eaa 846
1651e431 847 io_u_clear(td, __io_u, IO_U_F_FLIGHT);
a9da8ab2 848 if (__io_u->flags & IO_U_F_IN_CUR_DEPTH) {
0c41214f 849 td->cur_depth--;
a9da8ab2
JA
850 assert(!(td->flags & TD_F_CHILD));
851 }
2ae0b204
JA
852
853 io_u_rpush(&td->io_u_requeues, __io_u);
a9da8ab2 854 td_io_u_free_notify(td);
26b3a188
JA
855
856 if (needs_lock)
857 __td_io_u_unlock(td);
858
755200a3
JA
859 *io_u = NULL;
860}
861
7b865a2f 862static void setup_strided_zone_mode(struct thread_data *td, struct io_u *io_u)
224b3093 863{
864 struct fio_file *f = io_u->file;
865
7b865a2f
BVA
866 assert(td->o.zone_mode == ZONE_MODE_STRIDED);
867 assert(td->o.zone_size);
868 assert(td->o.zone_range);
869
224b3093 870 /*
871 * See if it's time to switch to a new zone
872 */
fa9fd914 873 if (td->zone_bytes >= td->o.zone_size) {
224b3093 874 td->zone_bytes = 0;
875 f->file_offset += td->o.zone_range + td->o.zone_skip;
876
877 /*
878 * Wrap from the beginning, if we exceed the file size
879 */
880 if (f->file_offset >= f->real_file_size)
04bc85a1
JA
881 f->file_offset = get_start_offset(td, f);
882
224b3093 883 f->last_pos[io_u->ddir] = f->file_offset;
884 td->io_skip_bytes += td->o.zone_skip;
885 }
886
887 /*
04bc85a1
JA
888 * If zone_size > zone_range, then maintain the same zone until
889 * zone_bytes >= zone_size.
890 */
224b3093 891 if (f->last_pos[io_u->ddir] >= (f->file_offset + td->o.zone_range)) {
892 dprint(FD_IO, "io_u maintain zone offset=%" PRIu64 "/last_pos=%" PRIu64 "\n",
893 f->file_offset, f->last_pos[io_u->ddir]);
894 f->last_pos[io_u->ddir] = f->file_offset;
895 }
896
897 /*
898 * For random: if 'norandommap' is not set and zone_size > zone_range,
899 * map needs to be reset as it's done with zone_range everytime.
900 */
04bc85a1 901 if ((td->zone_bytes % td->o.zone_range) == 0)
224b3093 902 fio_file_reset(td, f);
224b3093 903}
904
9bf2061e 905static int fill_io_u(struct thread_data *td, struct io_u *io_u)
10ba535a 906{
ec370c22 907 bool is_random;
bfbdd35b
BVA
908 uint64_t offset;
909 enum io_u_action ret;
6aca9b3d 910
9b87f09b 911 if (td_ioengine_flagged(td, FIO_NOIO))
b4c5e1ac
JA
912 goto out;
913
1ef2b6be 914 set_rw_ddir(td, io_u);
5f9099ea 915
df06a036 916 if (io_u->ddir == DDIR_INVAL) {
917 dprint(FD_IO, "invalid direction received ddir = %d", io_u->ddir);
918 return 1;
919 }
87dc1ab1 920 /*
ff58fced 921 * fsync() or fdatasync() or trim etc, we are done
87dc1ab1 922 */
ff58fced 923 if (!ddir_rw(io_u->ddir))
c38e9468 924 goto out;
a00735e6 925
7b865a2f
BVA
926 if (td->o.zone_mode == ZONE_MODE_STRIDED)
927 setup_strided_zone_mode(td, io_u);
4d37720a
DLM
928 else if (td->o.zone_mode == ZONE_MODE_ZBD)
929 setup_zbd_zone_mode(td, io_u);
48f5abd3 930
10ba535a 931 /*
c685b5b2
JA
932 * No log, let the seq/rand engine retrieve the next buflen and
933 * position.
10ba535a 934 */
6aca9b3d 935 if (get_next_offset(td, io_u, &is_random)) {
2ba1c290 936 dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
bca4ed4d 937 return 1;
2ba1c290 938 }
10ba535a 939
6aca9b3d 940 io_u->buflen = get_next_buflen(td, io_u, is_random);
2ba1c290
JA
941 if (!io_u->buflen) {
942 dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
bca4ed4d 943 return 1;
2ba1c290 944 }
bca4ed4d 945
bfbdd35b
BVA
946 offset = io_u->offset;
947 if (td->o.zone_mode == ZONE_MODE_ZBD) {
948 ret = zbd_adjust_block(td, io_u);
949 if (ret == io_u_eof)
950 return 1;
951 }
952
2ba1c290 953 if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
5fff9543 954 dprint(FD_IO, "io_u %p, off=0x%llx + len=0x%llx exceeds file size=0x%llx\n",
e5f9a813 955 io_u,
4b91ee8f
JA
956 (unsigned long long) io_u->offset, io_u->buflen,
957 (unsigned long long) io_u->file->real_file_size);
6a5e6884 958 return 1;
2ba1c290 959 }
6a5e6884 960
bca4ed4d
JA
961 /*
962 * mark entry before potentially trimming io_u
963 */
303032ae 964 if (td_random(td) && file_randommap(td, io_u->file))
bfbdd35b 965 io_u->buflen = mark_random_map(td, io_u, offset, io_u->buflen);
bca4ed4d 966
c38e9468 967out:
e5f9a813 968 dprint_io_u(io_u, "fill");
cb7d7abb 969 io_u->verify_offset = io_u->offset;
d9d91e39 970 td->zone_bytes += io_u->buflen;
bca4ed4d 971 return 0;
10ba535a
JA
972}
973
6cc0e5aa 974static void __io_u_mark_map(uint64_t *map, unsigned int nr)
838bc709 975{
2b13e716 976 int idx = 0;
838bc709
JA
977
978 switch (nr) {
979 default:
2b13e716 980 idx = 6;
838bc709
JA
981 break;
982 case 33 ... 64:
2b13e716 983 idx = 5;
838bc709
JA
984 break;
985 case 17 ... 32:
2b13e716 986 idx = 4;
838bc709
JA
987 break;
988 case 9 ... 16:
2b13e716 989 idx = 3;
838bc709
JA
990 break;
991 case 5 ... 8:
2b13e716 992 idx = 2;
838bc709
JA
993 break;
994 case 1 ... 4:
2b13e716 995 idx = 1;
db83b0ab 996 fallthrough;
838bc709
JA
997 case 0:
998 break;
999 }
1000
2b13e716 1001 map[idx]++;
838bc709
JA
1002}
1003
1004void io_u_mark_submit(struct thread_data *td, unsigned int nr)
1005{
1006 __io_u_mark_map(td->ts.io_u_submit, nr);
1007 td->ts.total_submit++;
1008}
1009
1010void io_u_mark_complete(struct thread_data *td, unsigned int nr)
1011{
1012 __io_u_mark_map(td->ts.io_u_complete, nr);
1013 td->ts.total_complete++;
1014}
1015
d8005759 1016void io_u_mark_depth(struct thread_data *td, unsigned int nr)
71619dc2 1017{
2b13e716 1018 int idx = 0;
71619dc2
JA
1019
1020 switch (td->cur_depth) {
1021 default:
2b13e716 1022 idx = 6;
a783e61a 1023 break;
71619dc2 1024 case 32 ... 63:
2b13e716 1025 idx = 5;
a783e61a 1026 break;
71619dc2 1027 case 16 ... 31:
2b13e716 1028 idx = 4;
a783e61a 1029 break;
71619dc2 1030 case 8 ... 15:
2b13e716 1031 idx = 3;
a783e61a 1032 break;
71619dc2 1033 case 4 ... 7:
2b13e716 1034 idx = 2;
a783e61a 1035 break;
71619dc2 1036 case 2 ... 3:
2b13e716 1037 idx = 1;
db83b0ab 1038 fallthrough;
71619dc2
JA
1039 case 1:
1040 break;
1041 }
1042
2b13e716 1043 td->ts.io_u_map[idx] += nr;
71619dc2
JA
1044}
1045
d6bb626e 1046static void io_u_mark_lat_nsec(struct thread_data *td, unsigned long long nsec)
04a0feae 1047{
2b13e716 1048 int idx = 0;
04a0feae 1049
d6bb626e
VF
1050 assert(nsec < 1000);
1051
1052 switch (nsec) {
1053 case 750 ... 999:
1054 idx = 9;
1055 break;
1056 case 500 ... 749:
1057 idx = 8;
1058 break;
1059 case 250 ... 499:
1060 idx = 7;
1061 break;
1062 case 100 ... 249:
1063 idx = 6;
1064 break;
1065 case 50 ... 99:
1066 idx = 5;
1067 break;
1068 case 20 ... 49:
1069 idx = 4;
1070 break;
1071 case 10 ... 19:
1072 idx = 3;
1073 break;
1074 case 4 ... 9:
1075 idx = 2;
1076 break;
1077 case 2 ... 3:
1078 idx = 1;
db83b0ab 1079 fallthrough;
d6bb626e
VF
1080 case 0 ... 1:
1081 break;
1082 }
1083
1084 assert(idx < FIO_IO_U_LAT_N_NR);
1085 td->ts.io_u_lat_n[idx]++;
1086}
1087
1088static void io_u_mark_lat_usec(struct thread_data *td, unsigned long long usec)
1089{
1090 int idx = 0;
1091
1092 assert(usec < 1000 && usec >= 1);
04a0feae
JA
1093
1094 switch (usec) {
1095 case 750 ... 999:
2b13e716 1096 idx = 9;
04a0feae
JA
1097 break;
1098 case 500 ... 749:
2b13e716 1099 idx = 8;
04a0feae
JA
1100 break;
1101 case 250 ... 499:
2b13e716 1102 idx = 7;
04a0feae
JA
1103 break;
1104 case 100 ... 249:
2b13e716 1105 idx = 6;
04a0feae
JA
1106 break;
1107 case 50 ... 99:
2b13e716 1108 idx = 5;
04a0feae
JA
1109 break;
1110 case 20 ... 49:
2b13e716 1111 idx = 4;
04a0feae
JA
1112 break;
1113 case 10 ... 19:
2b13e716 1114 idx = 3;
04a0feae
JA
1115 break;
1116 case 4 ... 9:
2b13e716 1117 idx = 2;
04a0feae
JA
1118 break;
1119 case 2 ... 3:
2b13e716 1120 idx = 1;
db83b0ab 1121 fallthrough;
04a0feae
JA
1122 case 0 ... 1:
1123 break;
1124 }
1125
2b13e716
JA
1126 assert(idx < FIO_IO_U_LAT_U_NR);
1127 td->ts.io_u_lat_u[idx]++;
04a0feae
JA
1128}
1129
d6bb626e 1130static void io_u_mark_lat_msec(struct thread_data *td, unsigned long long msec)
ec118304 1131{
2b13e716 1132 int idx = 0;
ec118304 1133
d6bb626e
VF
1134 assert(msec >= 1);
1135
ec118304
JA
1136 switch (msec) {
1137 default:
2b13e716 1138 idx = 11;
04a0feae 1139 break;
8abdce66 1140 case 1000 ... 1999:
2b13e716 1141 idx = 10;
04a0feae 1142 break;
8abdce66 1143 case 750 ... 999:
2b13e716 1144 idx = 9;
04a0feae 1145 break;
8abdce66 1146 case 500 ... 749:
2b13e716 1147 idx = 8;
04a0feae 1148 break;
8abdce66 1149 case 250 ... 499:
2b13e716 1150 idx = 7;
04a0feae 1151 break;
8abdce66 1152 case 100 ... 249:
2b13e716 1153 idx = 6;
04a0feae 1154 break;
8abdce66 1155 case 50 ... 99:
2b13e716 1156 idx = 5;
04a0feae 1157 break;
8abdce66 1158 case 20 ... 49:
2b13e716 1159 idx = 4;
04a0feae 1160 break;
8abdce66 1161 case 10 ... 19:
2b13e716 1162 idx = 3;
04a0feae 1163 break;
8abdce66 1164 case 4 ... 9:
2b13e716 1165 idx = 2;
04a0feae 1166 break;
ec118304 1167 case 2 ... 3:
2b13e716 1168 idx = 1;
db83b0ab 1169 fallthrough;
ec118304
JA
1170 case 0 ... 1:
1171 break;
1172 }
1173
2b13e716
JA
1174 assert(idx < FIO_IO_U_LAT_M_NR);
1175 td->ts.io_u_lat_m[idx]++;
04a0feae
JA
1176}
1177
d6bb626e 1178static void io_u_mark_latency(struct thread_data *td, unsigned long long nsec)
04a0feae 1179{
d6bb626e
VF
1180 if (nsec < 1000)
1181 io_u_mark_lat_nsec(td, nsec);
1182 else if (nsec < 1000000)
1183 io_u_mark_lat_usec(td, nsec / 1000);
04a0feae 1184 else
d6bb626e 1185 io_u_mark_lat_msec(td, nsec / 1000000);
ec118304
JA
1186}
1187
8c07860d
JA
1188static unsigned int __get_next_fileno_rand(struct thread_data *td)
1189{
1190 unsigned long fileno;
1191
1192 if (td->o.file_service_type == FIO_FSERVICE_RANDOM) {
1193 uint64_t frand_max = rand_max(&td->next_file_state);
1194 unsigned long r;
1195
1196 r = __rand(&td->next_file_state);
1197 return (unsigned int) ((double) td->o.nr_files
1198 * (r / (frand_max + 1.0)));
1199 }
1200
1201 if (td->o.file_service_type == FIO_FSERVICE_ZIPF)
1202 fileno = zipf_next(&td->next_file_zipf);
1203 else if (td->o.file_service_type == FIO_FSERVICE_PARETO)
1204 fileno = pareto_next(&td->next_file_zipf);
1205 else if (td->o.file_service_type == FIO_FSERVICE_GAUSS)
1206 fileno = gauss_next(&td->next_file_gauss);
1207 else {
1208 log_err("fio: bad file service type: %d\n", td->o.file_service_type);
1209 assert(0);
1210 return 0;
1211 }
1212
1213 return fileno >> FIO_FSERVICE_SHIFT;
1214}
1215
0aabe160
JA
1216/*
1217 * Get next file to service by choosing one at random
1218 */
2cc52930
JA
1219static struct fio_file *get_next_file_rand(struct thread_data *td,
1220 enum fio_file_flags goodf,
d6aed795 1221 enum fio_file_flags badf)
0aabe160 1222{
0aabe160 1223 struct fio_file *f;
1c178180 1224 int fno;
0aabe160
JA
1225
1226 do {
87b10676 1227 int opened = 0;
4c07ad86 1228
8c07860d 1229 fno = __get_next_fileno_rand(td);
7c83c089 1230
126d65c6 1231 f = td->files[fno];
d6aed795 1232 if (fio_file_done(f))
059e63c0 1233 continue;
1c178180 1234
d6aed795 1235 if (!fio_file_open(f)) {
87b10676
JA
1236 int err;
1237
002fe734
JA
1238 if (td->nr_open_files >= td->o.open_files)
1239 return ERR_PTR(-EBUSY);
1240
87b10676
JA
1241 err = td_io_open_file(td, f);
1242 if (err)
1243 continue;
1244 opened = 1;
1245 }
1246
2ba1c290
JA
1247 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) {
1248 dprint(FD_FILE, "get_next_file_rand: %p\n", f);
0aabe160 1249 return f;
2ba1c290 1250 }
87b10676
JA
1251 if (opened)
1252 td_io_close_file(td, f);
0aabe160
JA
1253 } while (1);
1254}
1255
1256/*
1257 * Get next file to service by doing round robin between all available ones
1258 */
1c178180
JA
1259static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
1260 int badf)
3d7c391d
JA
1261{
1262 unsigned int old_next_file = td->next_file;
1263 struct fio_file *f;
1264
1265 do {
87b10676
JA
1266 int opened = 0;
1267
126d65c6 1268 f = td->files[td->next_file];
3d7c391d
JA
1269
1270 td->next_file++;
2dc1bbeb 1271 if (td->next_file >= td->o.nr_files)
3d7c391d
JA
1272 td->next_file = 0;
1273
87b10676 1274 dprint(FD_FILE, "trying file %s %x\n", f->file_name, f->flags);
d6aed795 1275 if (fio_file_done(f)) {
d5ed68ea 1276 f = NULL;
059e63c0 1277 continue;
d5ed68ea 1278 }
059e63c0 1279
d6aed795 1280 if (!fio_file_open(f)) {
87b10676
JA
1281 int err;
1282
002fe734
JA
1283 if (td->nr_open_files >= td->o.open_files)
1284 return ERR_PTR(-EBUSY);
1285
87b10676 1286 err = td_io_open_file(td, f);
b5696bfc
JA
1287 if (err) {
1288 dprint(FD_FILE, "error %d on open of %s\n",
1289 err, f->file_name);
87c27b45 1290 f = NULL;
87b10676 1291 continue;
b5696bfc 1292 }
87b10676
JA
1293 opened = 1;
1294 }
1295
0b9d69ec
JA
1296 dprint(FD_FILE, "goodf=%x, badf=%x, ff=%x\n", goodf, badf,
1297 f->flags);
1c178180 1298 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
3d7c391d
JA
1299 break;
1300
87b10676
JA
1301 if (opened)
1302 td_io_close_file(td, f);
1303
3d7c391d
JA
1304 f = NULL;
1305 } while (td->next_file != old_next_file);
1306
2ba1c290 1307 dprint(FD_FILE, "get_next_file_rr: %p\n", f);
3d7c391d
JA
1308 return f;
1309}
1310
7eb36574 1311static struct fio_file *__get_next_file(struct thread_data *td)
bdb4e2e9 1312{
1907dbc6
JA
1313 struct fio_file *f;
1314
2dc1bbeb 1315 assert(td->o.nr_files <= td->files_index);
1c178180 1316
b5696bfc 1317 if (td->nr_done_files >= td->o.nr_files) {
5ec10eaa
JA
1318 dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d,"
1319 " nr_files=%d\n", td->nr_open_files,
1320 td->nr_done_files,
1321 td->o.nr_files);
bdb4e2e9 1322 return NULL;
2ba1c290 1323 }
bdb4e2e9 1324
1907dbc6 1325 f = td->file_service_file;
d6aed795 1326 if (f && fio_file_open(f) && !fio_file_closing(f)) {
a086c257
JA
1327 if (td->o.file_service_type == FIO_FSERVICE_SEQ)
1328 goto out;
4ef1562a
AK
1329 if (td->file_service_left) {
1330 td->file_service_left--;
1331 goto out;
1332 }
a086c257 1333 }
1907dbc6 1334
a086c257
JA
1335 if (td->o.file_service_type == FIO_FSERVICE_RR ||
1336 td->o.file_service_type == FIO_FSERVICE_SEQ)
d6aed795 1337 f = get_next_file_rr(td, FIO_FILE_open, FIO_FILE_closing);
bdb4e2e9 1338 else
d6aed795 1339 f = get_next_file_rand(td, FIO_FILE_open, FIO_FILE_closing);
1907dbc6 1340
002fe734
JA
1341 if (IS_ERR(f))
1342 return f;
1343
1907dbc6
JA
1344 td->file_service_file = f;
1345 td->file_service_left = td->file_service_nr - 1;
2ba1c290 1346out:
0dac421f
JA
1347 if (f)
1348 dprint(FD_FILE, "get_next_file: %p [%s]\n", f, f->file_name);
1349 else
1350 dprint(FD_FILE, "get_next_file: NULL\n");
1907dbc6 1351 return f;
bdb4e2e9
JA
1352}
1353
7eb36574
JA
1354static struct fio_file *get_next_file(struct thread_data *td)
1355{
7eb36574
JA
1356 return __get_next_file(td);
1357}
1358
002fe734 1359static long set_io_u_file(struct thread_data *td, struct io_u *io_u)
429f6675
JA
1360{
1361 struct fio_file *f;
1362
1363 do {
1364 f = get_next_file(td);
002fe734
JA
1365 if (IS_ERR_OR_NULL(f))
1366 return PTR_ERR(f);
429f6675 1367
429f6675
JA
1368 io_u->file = f;
1369 get_file(f);
1370
1371 if (!fill_io_u(td, io_u))
1372 break;
1373
b2da58c4 1374 zbd_put_io_u(td, io_u);
99952ca7 1375
b5696bfc 1376 put_file_log(td, f);
429f6675 1377 td_io_close_file(td, f);
b5696bfc 1378 io_u->file = NULL;
8c07860d
JA
1379 if (td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM)
1380 fio_file_reset(td, f);
1381 else {
1382 fio_file_set_done(f);
1383 td->nr_done_files++;
1384 dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name,
0b9d69ec 1385 td->nr_done_files, td->o.nr_files);
8c07860d 1386 }
429f6675
JA
1387 } while (1);
1388
1389 return 0;
1390}
1391
f7cf63bf 1392static void lat_fatal(struct thread_data *td, struct io_u *io_u, struct io_completion_data *icd,
c3a32714 1393 unsigned long long tnsec, unsigned long long max_nsec)
3e260a46 1394{
f7cf63bf
VR
1395 if (!td->error) {
1396 log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec): %s %s %llu %llu\n",
1397 tnsec, max_nsec,
1398 io_u->file->file_name,
1399 io_ddir_name(io_u->ddir),
1400 io_u->offset, io_u->buflen);
1401 }
3e260a46
JA
1402 td_verror(td, ETIMEDOUT, "max latency exceeded");
1403 icd->error = ETIMEDOUT;
1404}
1405
1406static void lat_new_cycle(struct thread_data *td)
1407{
1408 fio_gettime(&td->latency_ts, NULL);
1409 td->latency_ios = ddir_rw_sum(td->io_blocks);
1410 td->latency_failed = 0;
1411}
1412
1413/*
1414 * We had an IO outside the latency target. Reduce the queue depth. If we
1415 * are at QD=1, then it's time to give up.
1416 */
e39c0676 1417static bool __lat_target_failed(struct thread_data *td)
3e260a46
JA
1418{
1419 if (td->latency_qd == 1)
e39c0676 1420 return true;
3e260a46
JA
1421
1422 td->latency_qd_high = td->latency_qd;
6bb58215
JA
1423
1424 if (td->latency_qd == td->latency_qd_low)
1425 td->latency_qd_low--;
1426
3e260a46 1427 td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2;
e1bcd541 1428 td->latency_stable_count = 0;
3e260a46
JA
1429
1430 dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
1431
1432 /*
1433 * When we ramp QD down, quiesce existing IO to prevent
1434 * a storm of ramp downs due to pending higher depth.
1435 */
1436 io_u_quiesce(td);
1437 lat_new_cycle(td);
e39c0676 1438 return false;
3e260a46
JA
1439}
1440
e39c0676 1441static bool lat_target_failed(struct thread_data *td)
3e260a46
JA
1442{
1443 if (td->o.latency_percentile.u.f == 100.0)
1444 return __lat_target_failed(td);
1445
1446 td->latency_failed++;
e39c0676 1447 return false;
3e260a46
JA
1448}
1449
1450void lat_target_init(struct thread_data *td)
1451{
6bb58215
JA
1452 td->latency_end_run = 0;
1453
3e260a46
JA
1454 if (td->o.latency_target) {
1455 dprint(FD_RATE, "Latency target=%llu\n", td->o.latency_target);
1456 fio_gettime(&td->latency_ts, NULL);
1457 td->latency_qd = 1;
1458 td->latency_qd_high = td->o.iodepth;
1459 td->latency_qd_low = 1;
1460 td->latency_ios = ddir_rw_sum(td->io_blocks);
1461 } else
1462 td->latency_qd = td->o.iodepth;
1463}
1464
6bb58215
JA
1465void lat_target_reset(struct thread_data *td)
1466{
1467 if (!td->latency_end_run)
1468 lat_target_init(td);
1469}
1470
3e260a46
JA
1471static void lat_target_success(struct thread_data *td)
1472{
1473 const unsigned int qd = td->latency_qd;
6bb58215 1474 struct thread_options *o = &td->o;
3e260a46
JA
1475
1476 td->latency_qd_low = td->latency_qd;
1477
e1bcd541
SL
1478 if (td->latency_qd + 1 == td->latency_qd_high) {
1479 /*
1480 * latency_qd will not incease on lat_target_success(), so
1481 * called stable. If we stick with this queue depth, the
1482 * final latency is likely lower than latency_target. Fix
1483 * this by increasing latency_qd_high slowly. Use a naive
1484 * heuristic here. If we get lat_target_success() 3 times
1485 * in a row, increase latency_qd_high by 1.
1486 */
1487 if (++td->latency_stable_count >= 3) {
1488 td->latency_qd_high++;
1489 td->latency_stable_count = 0;
1490 }
1491 }
1492
3e260a46
JA
1493 /*
1494 * If we haven't failed yet, we double up to a failing value instead
1495 * of bisecting from highest possible queue depth. If we have set
1496 * a limit other than td->o.iodepth, bisect between that.
1497 */
6bb58215 1498 if (td->latency_qd_high != o->iodepth)
3e260a46
JA
1499 td->latency_qd = (td->latency_qd + td->latency_qd_high) / 2;
1500 else
1501 td->latency_qd *= 2;
1502
6bb58215
JA
1503 if (td->latency_qd > o->iodepth)
1504 td->latency_qd = o->iodepth;
3e260a46
JA
1505
1506 dprint(FD_RATE, "Ramped up: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
6bb58215 1507
3e260a46 1508 /*
6bb58215
JA
1509 * Same as last one, we are done. Let it run a latency cycle, so
1510 * we get only the results from the targeted depth.
3e260a46 1511 */
e1bcd541 1512 if (!o->latency_run && td->latency_qd == qd) {
6bb58215
JA
1513 if (td->latency_end_run) {
1514 dprint(FD_RATE, "We are done\n");
1515 td->done = 1;
1516 } else {
1517 dprint(FD_RATE, "Quiesce and final run\n");
1518 io_u_quiesce(td);
1519 td->latency_end_run = 1;
1520 reset_all_stats(td);
1521 reset_io_stats(td);
1522 }
1523 }
3e260a46
JA
1524
1525 lat_new_cycle(td);
1526}
1527
1528/*
1529 * Check if we can bump the queue depth
1530 */
1531void lat_target_check(struct thread_data *td)
1532{
1533 uint64_t usec_window;
1534 uint64_t ios;
1535 double success_ios;
1536
1537 usec_window = utime_since_now(&td->latency_ts);
1538 if (usec_window < td->o.latency_window)
1539 return;
1540
1541 ios = ddir_rw_sum(td->io_blocks) - td->latency_ios;
1542 success_ios = (double) (ios - td->latency_failed) / (double) ios;
1543 success_ios *= 100.0;
1544
1545 dprint(FD_RATE, "Success rate: %.2f%% (target %.2f%%)\n", success_ios, td->o.latency_percentile.u.f);
1546
1547 if (success_ios >= td->o.latency_percentile.u.f)
1548 lat_target_success(td);
1549 else
1550 __lat_target_failed(td);
1551}
1552
1553/*
1554 * If latency target is enabled, we might be ramping up or down and not
1555 * using the full queue depth available.
1556 */
e39c0676 1557bool queue_full(const struct thread_data *td)
3e260a46
JA
1558{
1559 const int qempty = io_u_qempty(&td->io_u_freelist);
1560
1561 if (qempty)
e39c0676 1562 return true;
3e260a46 1563 if (!td->o.latency_target)
e39c0676 1564 return false;
3e260a46
JA
1565
1566 return td->cur_depth >= td->latency_qd;
1567}
429f6675 1568
10ba535a
JA
1569struct io_u *__get_io_u(struct thread_data *td)
1570{
26b3a188 1571 const bool needs_lock = td_async_processing(td);
0cae66f6 1572 struct io_u *io_u = NULL;
93b45bb2 1573 int ret;
10ba535a 1574
ca09be4b
JA
1575 if (td->stop_io)
1576 return NULL;
1577
26b3a188
JA
1578 if (needs_lock)
1579 __td_io_u_lock(td);
e8462bd8
JA
1580
1581again:
bba6b14f 1582 if (!io_u_rempty(&td->io_u_requeues)) {
2ae0b204 1583 io_u = io_u_rpop(&td->io_u_requeues);
bba6b14f
JA
1584 io_u->resid = 0;
1585 } else if (!queue_full(td)) {
2ae0b204 1586 io_u = io_u_qpop(&td->io_u_freelist);
10ba535a 1587
225ba9e3 1588 io_u->file = NULL;
6040dabc 1589 io_u->buflen = 0;
10ba535a 1590 io_u->resid = 0;
d7762cf8 1591 io_u->end_io = NULL;
755200a3
JA
1592 }
1593
1594 if (io_u) {
0c6e7517 1595 assert(io_u->flags & IO_U_F_FREE);
1651e431 1596 io_u_clear(td, io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT |
f8b0bd10 1597 IO_U_F_TRIMMED | IO_U_F_BARRIER |
03ec570f 1598 IO_U_F_VER_LIST | IO_U_F_HIGH_PRIO);
0c6e7517 1599
755200a3 1600 io_u->error = 0;
bcd5abfa 1601 io_u->acct_ddir = -1;
10ba535a 1602 td->cur_depth++;
a9da8ab2 1603 assert(!(td->flags & TD_F_CHILD));
1651e431 1604 io_u_set(td, io_u, IO_U_F_IN_CUR_DEPTH);
f9401285 1605 io_u->ipo = NULL;
a9da8ab2 1606 } else if (td_async_processing(td)) {
1dec3e07
JA
1607 /*
1608 * We ran out, wait for async verify threads to finish and
1609 * return one
1610 */
a9da8ab2 1611 assert(!(td->flags & TD_F_CHILD));
93b45bb2
BVA
1612 ret = pthread_cond_wait(&td->free_cond, &td->io_u_lock);
1613 assert(ret == 0);
ee5e08ad
JA
1614 if (!td->error)
1615 goto again;
10ba535a
JA
1616 }
1617
26b3a188
JA
1618 if (needs_lock)
1619 __td_io_u_unlock(td);
1620
10ba535a
JA
1621 return io_u;
1622}
1623
e39c0676 1624static bool check_get_trim(struct thread_data *td, struct io_u *io_u)
10ba535a 1625{
d72be545 1626 if (!(td->flags & TD_F_TRIM_BACKLOG))
e39c0676 1627 return false;
c9a73054
JA
1628 if (!td->trim_entries)
1629 return false;
d72be545 1630
c9a73054
JA
1631 if (td->trim_batch) {
1632 td->trim_batch--;
1633 if (get_next_trim(td, io_u))
1634 return true;
1635 } else if (!(td->io_hist_len % td->o.trim_backlog) &&
1636 td->last_ddir != DDIR_READ) {
1637 td->trim_batch = td->o.trim_batch;
1638 if (!td->trim_batch)
1639 td->trim_batch = td->o.trim_backlog;
1640 if (get_next_trim(td, io_u))
e39c0676 1641 return true;
2ba1c290 1642 }
10ba535a 1643
e39c0676 1644 return false;
0d29de83
JA
1645}
1646
e39c0676 1647static bool check_get_verify(struct thread_data *td, struct io_u *io_u)
0d29de83 1648{
d72be545 1649 if (!(td->flags & TD_F_VER_BACKLOG))
e39c0676 1650 return false;
d72be545
JA
1651
1652 if (td->io_hist_len) {
9e144189
JA
1653 int get_verify = 0;
1654
d1ece0c7 1655 if (td->verify_batch)
9e144189 1656 get_verify = 1;
d1ece0c7 1657 else if (!(td->io_hist_len % td->o.verify_backlog) &&
9e144189
JA
1658 td->last_ddir != DDIR_READ) {
1659 td->verify_batch = td->o.verify_batch;
f8a75c99
JA
1660 if (!td->verify_batch)
1661 td->verify_batch = td->o.verify_backlog;
9e144189
JA
1662 get_verify = 1;
1663 }
1664
d1ece0c7
JA
1665 if (get_verify && !get_next_verify(td, io_u)) {
1666 td->verify_batch--;
e39c0676 1667 return true;
d1ece0c7 1668 }
9e144189
JA
1669 }
1670
e39c0676 1671 return false;
0d29de83
JA
1672}
1673
de789769
JA
1674/*
1675 * Fill offset and start time into the buffer content, to prevent too
23f394d5
JA
1676 * easy compressible data for simple de-dupe attempts. Do this for every
1677 * 512b block in the range, since that should be the smallest block size
1678 * we can expect from a device.
de789769
JA
1679 */
1680static void small_content_scramble(struct io_u *io_u)
1681{
5fff9543 1682 unsigned long long i, nr_blocks = io_u->buflen >> 9;
23f394d5 1683 unsigned int offset;
319b073f
JA
1684 uint64_t boffset, *iptr;
1685 char *p;
de789769 1686
23f394d5
JA
1687 if (!nr_blocks)
1688 return;
1689
1690 p = io_u->xfer_buf;
fba76ee8 1691 boffset = io_u->offset;
319b073f
JA
1692
1693 if (io_u->buf_filled_len)
1694 io_u->buf_filled_len = 0;
1695
1696 /*
1697 * Generate random index between 0..7. We do chunks of 512b, if
1698 * we assume a cacheline is 64 bytes, then we have 8 of those.
1699 * Scramble content within the blocks in the same cacheline to
1700 * speed things up.
1701 */
1702 offset = (io_u->start_time.tv_nsec ^ boffset) & 7;
fad82f76 1703
23f394d5
JA
1704 for (i = 0; i < nr_blocks; i++) {
1705 /*
319b073f
JA
1706 * Fill offset into start of cacheline, time into end
1707 * of cacheline
23f394d5 1708 */
319b073f
JA
1709 iptr = (void *) p + (offset << 6);
1710 *iptr = boffset;
1711
1712 iptr = (void *) p + 64 - 2 * sizeof(uint64_t);
1713 iptr[0] = io_u->start_time.tv_sec;
1714 iptr[1] = io_u->start_time.tv_nsec;
23f394d5 1715
23f394d5 1716 p += 512;
fad82f76 1717 boffset += 512;
23f394d5 1718 }
de789769
JA
1719}
1720
0d29de83
JA
1721/*
1722 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
5c5c33c1 1723 * etc. The returned io_u is fully ready to be prepped, populated and submitted.
0d29de83
JA
1724 */
1725struct io_u *get_io_u(struct thread_data *td)
1726{
1727 struct fio_file *f;
1728 struct io_u *io_u;
de789769 1729 int do_scramble = 0;
002fe734 1730 long ret = 0;
0d29de83
JA
1731
1732 io_u = __get_io_u(td);
1733 if (!io_u) {
1734 dprint(FD_IO, "__get_io_u failed\n");
1735 return NULL;
1736 }
1737
1738 if (check_get_verify(td, io_u))
1739 goto out;
1740 if (check_get_trim(td, io_u))
1741 goto out;
1742
755200a3
JA
1743 /*
1744 * from a requeue, io_u already setup
1745 */
1746 if (io_u->file)
77f392bf 1747 goto out;
755200a3 1748
429f6675
JA
1749 /*
1750 * If using an iolog, grab next piece if any available.
1751 */
d72be545 1752 if (td->flags & TD_F_READ_IOLOG) {
429f6675
JA
1753 if (read_iolog_get(td, io_u))
1754 goto err_put;
2ba1c290 1755 } else if (set_io_u_file(td, io_u)) {
002fe734 1756 ret = -EBUSY;
2ba1c290 1757 dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
429f6675 1758 goto err_put;
2ba1c290 1759 }
5ec10eaa 1760
429f6675 1761 f = io_u->file;
002fe734
JA
1762 if (!f) {
1763 dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
1764 goto err_put;
1765 }
1766
d6aed795 1767 assert(fio_file_open(f));
97af62ce 1768
ff58fced 1769 if (ddir_rw(io_u->ddir)) {
9b87f09b 1770 if (!io_u->buflen && !td_ioengine_flagged(td, FIO_NOIO)) {
2ba1c290 1771 dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u);
429f6675 1772 goto err_put;
2ba1c290 1773 }
10ba535a 1774
f1dfb668
JA
1775 f->last_start[io_u->ddir] = io_u->offset;
1776 f->last_pos[io_u->ddir] = io_u->offset + io_u->buflen;
10ba535a 1777
fd68418e 1778 if (io_u->ddir == DDIR_WRITE) {
d72be545 1779 if (td->flags & TD_F_REFILL_BUFFERS) {
9c42684e 1780 io_u_fill_buffer(td, io_u,
1066358a 1781 td->o.min_bs[DDIR_WRITE],
9e129577 1782 io_u->buflen);
ff441ae8 1783 } else if ((td->flags & TD_F_SCRAMBLE_BUFFERS) &&
5c5c33c1
BVA
1784 !(td->flags & TD_F_COMPRESS) &&
1785 !(td->flags & TD_F_DO_VERIFY))
fd68418e
JA
1786 do_scramble = 1;
1787 } else if (io_u->ddir == DDIR_READ) {
cbe8d756
RR
1788 /*
1789 * Reset the buf_filled parameters so next time if the
1790 * buffer is used for writes it is refilled.
1791 */
cbe8d756
RR
1792 io_u->buf_filled_len = 0;
1793 }
87dc1ab1 1794 }
10ba535a 1795
165faf16
JA
1796 /*
1797 * Set io data pointers.
1798 */
cec6b55d
JA
1799 io_u->xfer_buf = io_u->buf;
1800 io_u->xfer_buflen = io_u->buflen;
5973cafb 1801
03ec570f
DLM
1802 /*
1803 * Remember the issuing context priority. The IO engine may change this.
1804 */
1805 io_u->ioprio = td->ioprio;
6ac7a331 1806out:
0d29de83 1807 assert(io_u->file);
429f6675 1808 if (!td_io_prep(td, io_u)) {
3ecc8c67 1809 if (!td->o.disable_lat)
993bf48b 1810 fio_gettime(&io_u->start_time, NULL);
03553853 1811
de789769
JA
1812 if (do_scramble)
1813 small_content_scramble(io_u);
03553853 1814
429f6675 1815 return io_u;
36167d82 1816 }
429f6675 1817err_put:
2ba1c290 1818 dprint(FD_IO, "get_io_u failed\n");
429f6675 1819 put_io_u(td, io_u);
002fe734 1820 return ERR_PTR(ret);
10ba535a
JA
1821}
1822
a9da8ab2 1823static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
5451792e 1824{
8b28bd41 1825 enum error_type_bit eb = td_error_type(io_u->ddir, io_u->error);
825f818e 1826
8b28bd41
DM
1827 if (td_non_fatal_error(td, eb, io_u->error) && !td->o.error_dump)
1828 return;
5451792e 1829
5fff9543 1830 log_err("fio: io_u error%s%s: %s: %s offset=%llu, buflen=%llu\n",
709c8313
RE
1831 io_u->file ? " on file " : "",
1832 io_u->file ? io_u->file->file_name : "",
1833 strerror(io_u->error),
1834 io_ddir_name(io_u->ddir),
1835 io_u->offset, io_u->xfer_buflen);
5451792e 1836
5ad7be56
KD
1837 if (td->io_ops->errdetails) {
1838 char *err = td->io_ops->errdetails(io_u);
1839
1840 log_err("fio: %s\n", err);
1841 free(err);
1842 }
1843
5451792e
JA
1844 if (!td->error)
1845 td_verror(td, io_u->error, "io_u error");
1846}
1847
a9da8ab2
JA
1848void io_u_log_error(struct thread_data *td, struct io_u *io_u)
1849{
1850 __io_u_log_error(td, io_u);
1851 if (td->parent)
094e66cb 1852 __io_u_log_error(td->parent, io_u);
a9da8ab2
JA
1853}
1854
e39c0676 1855static inline bool gtod_reduce(struct thread_data *td)
aba6c951 1856{
3ecc8c67
JA
1857 return (td->o.disable_clat && td->o.disable_slat && td->o.disable_bw)
1858 || td->o.gtod_reduce;
aba6c951
JA
1859}
1860
d7e92306
JA
1861static void trim_block_info(struct thread_data *td, struct io_u *io_u)
1862{
1863 uint32_t *info = io_u_block_info(td, io_u);
1864
1865 if (BLOCK_INFO_STATE(*info) >= BLOCK_STATE_TRIM_FAILURE)
1866 return;
1867
1868 *info = BLOCK_INFO(BLOCK_STATE_TRIMMED, BLOCK_INFO_TRIMS(*info) + 1);
1869}
1870
c8eeb9df
JA
1871static void account_io_completion(struct thread_data *td, struct io_u *io_u,
1872 struct io_completion_data *icd,
1873 const enum fio_ddir idx, unsigned int bytes)
1874{
a9da8ab2 1875 const int no_reduce = !gtod_reduce(td);
d6bb626e 1876 unsigned long long llnsec = 0;
c8eeb9df 1877
75dc383e
JA
1878 if (td->parent)
1879 td = td->parent;
1880
132b1ee4 1881 if (!td->o.stats || td_ioengine_flagged(td, FIO_NOSTATS))
8243be59
JA
1882 return;
1883
a9da8ab2 1884 if (no_reduce)
d6bb626e 1885 llnsec = ntime_since(&io_u->issue_time, &icd->time);
c8eeb9df
JA
1886
1887 if (!td->o.disable_lat) {
c3a32714 1888 unsigned long long tnsec;
c8eeb9df 1889
d6bb626e 1890 tnsec = ntime_since(&io_u->start_time, &icd->time);
03ec570f
DLM
1891 add_lat_sample(td, idx, tnsec, bytes, io_u->offset,
1892 io_u->ioprio, io_u_is_high_prio(io_u));
15501535 1893
d4afedfd
JA
1894 if (td->flags & TD_F_PROFILE_OPS) {
1895 struct prof_io_ops *ops = &td->prof_io_ops;
1896
1897 if (ops->io_u_lat)
c3a32714 1898 icd->error = ops->io_u_lat(td, tnsec);
d4afedfd
JA
1899 }
1900
f7cf63bf
VR
1901 if (ddir_rw(idx)) {
1902 if (td->o.max_latency[idx] && tnsec > td->o.max_latency[idx])
1903 lat_fatal(td, io_u, icd, tnsec, td->o.max_latency[idx]);
1904 if (td->o.latency_target && tnsec > td->o.latency_target) {
1905 if (lat_target_failed(td))
1906 lat_fatal(td, io_u, icd, tnsec, td->o.latency_target);
1907 }
15501535 1908 }
c8eeb9df
JA
1909 }
1910
a47591e4
JA
1911 if (ddir_rw(idx)) {
1912 if (!td->o.disable_clat) {
03ec570f
DLM
1913 add_clat_sample(td, idx, llnsec, bytes, io_u->offset,
1914 io_u->ioprio, io_u_is_high_prio(io_u));
d6bb626e 1915 io_u_mark_latency(td, llnsec);
a47591e4 1916 }
c8eeb9df 1917
a47591e4 1918 if (!td->o.disable_bw && per_unit_log(td->bw_log))
d6bb626e 1919 add_bw_sample(td, io_u, bytes, llnsec);
c8eeb9df 1920
a47591e4
JA
1921 if (no_reduce && per_unit_log(td->iops_log))
1922 add_iops_sample(td, io_u, bytes);
b2b3eefe
JA
1923 } else if (ddir_sync(idx) && !td->o.disable_clat)
1924 add_sync_clat_sample(&td->ts, llnsec);
66347cfa 1925
d7e92306
JA
1926 if (td->ts.nr_block_infos && io_u->ddir == DDIR_TRIM)
1927 trim_block_info(td, io_u);
c8eeb9df
JA
1928}
1929
94a6e1bb
JA
1930static void file_log_write_comp(const struct thread_data *td, struct fio_file *f,
1931 uint64_t offset, unsigned int bytes)
1932{
1933 int idx;
1934
639ad1ea
JA
1935 if (!f)
1936 return;
1937
94a6e1bb
JA
1938 if (f->first_write == -1ULL || offset < f->first_write)
1939 f->first_write = offset;
1940 if (f->last_write == -1ULL || ((offset + bytes) > f->last_write))
1941 f->last_write = offset + bytes;
1942
1943 if (!f->last_write_comp)
1944 return;
1945
1946 idx = f->last_write_idx++;
1947 f->last_write_comp[idx] = offset;
1948 if (f->last_write_idx == td->o.iodepth)
1949 f->last_write_idx = 0;
1950}
1951
b2b3eefe
JA
1952static bool should_account(struct thread_data *td)
1953{
1954 return ramp_time_over(td) && (td->runstate == TD_RUNNING ||
1955 td->runstate == TD_VERIFYING);
1956}
1957
f8b0bd10 1958static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
97601024 1959 struct io_completion_data *icd)
10ba535a 1960{
f8b0bd10
JA
1961 struct io_u *io_u = *io_u_ptr;
1962 enum fio_ddir ddir = io_u->ddir;
1963 struct fio_file *f = io_u->file;
10ba535a 1964
e5f9a813 1965 dprint_io_u(io_u, "complete");
2ba1c290 1966
0c6e7517 1967 assert(io_u->flags & IO_U_F_FLIGHT);
1651e431 1968 io_u_clear(td, io_u, IO_U_F_FLIGHT | IO_U_F_BUSY_OK);
f9401285
JA
1969
1970 /*
1971 * Mark IO ok to verify
1972 */
1973 if (io_u->ipo) {
890b6656
JA
1974 /*
1975 * Remove errored entry from the verification list
1976 */
1977 if (io_u->error)
1978 unlog_io_piece(td, io_u);
1979 else {
0337173e
BVA
1980 atomic_store_release(&io_u->ipo->flags,
1981 io_u->ipo->flags & ~IP_F_IN_FLIGHT);
890b6656 1982 }
f9401285
JA
1983 }
1984
f8b0bd10 1985 if (ddir_sync(ddir)) {
2ea93f98 1986 td->last_was_sync = true;
44f29692
JA
1987 if (f) {
1988 f->first_write = -1ULL;
1989 f->last_write = -1ULL;
1990 }
b2b3eefe
JA
1991 if (should_account(td))
1992 account_io_completion(td, io_u, icd, ddir, io_u->buflen);
87dc1ab1
JA
1993 return;
1994 }
1995
2ea93f98 1996 td->last_was_sync = false;
f8b0bd10 1997 td->last_ddir = ddir;
87dc1ab1 1998
f8b0bd10 1999 if (!io_u->error && ddir_rw(ddir)) {
84abeef7 2000 unsigned long long bytes = io_u->xfer_buflen - io_u->resid;
b29ee5b3 2001 int ret;
10ba535a 2002
7d33649c
JA
2003 /*
2004 * Make sure we notice short IO from here, and requeue them
2005 * appropriately!
2006 */
c4cb947e 2007 if (bytes && io_u->resid) {
7d33649c
JA
2008 io_u->xfer_buflen = io_u->resid;
2009 io_u->xfer_buf += bytes;
2010 io_u->offset += bytes;
2011 td->ts.short_io_u[io_u->ddir]++;
2012 if (io_u->offset < io_u->file->real_file_size) {
2013 requeue_io_u(td, io_u_ptr);
2014 return;
2015 }
2016 }
2017
f8b0bd10 2018 td->io_blocks[ddir]++;
f8b0bd10 2019 td->io_bytes[ddir] += bytes;
ae2fafc8 2020
e1c325d2
JA
2021 if (!(io_u->flags & IO_U_F_VER_LIST)) {
2022 td->this_io_blocks[ddir]++;
f8b0bd10 2023 td->this_io_bytes[ddir] += bytes;
e1c325d2 2024 }
f8b0bd10 2025
639ad1ea 2026 if (ddir == DDIR_WRITE)
94a6e1bb 2027 file_log_write_comp(td, f, io_u->offset, bytes);
44f29692 2028
b2b3eefe 2029 if (should_account(td))
f8b0bd10 2030 account_io_completion(td, io_u, icd, ddir, bytes);
40e1a6f0 2031
f8b0bd10 2032 icd->bytes_done[ddir] += bytes;
3af6ef39 2033
d7762cf8 2034 if (io_u->end_io) {
f8b0bd10
JA
2035 ret = io_u->end_io(td, io_u_ptr);
2036 io_u = *io_u_ptr;
3af6ef39
JA
2037 if (ret && !icd->error)
2038 icd->error = ret;
2039 }
ff58fced 2040 } else if (io_u->error) {
10ba535a 2041 icd->error = io_u->error;
5451792e
JA
2042 io_u_log_error(td, io_u);
2043 }
8b28bd41 2044 if (icd->error) {
f8b0bd10
JA
2045 enum error_type_bit eb = td_error_type(ddir, icd->error);
2046
8b28bd41
DM
2047 if (!td_non_fatal_error(td, eb, icd->error))
2048 return;
f8b0bd10 2049
f2bba182
RR
2050 /*
2051 * If there is a non_fatal error, then add to the error count
2052 * and clear all the errors.
2053 */
2054 update_error_count(td, icd->error);
2055 td_clear_error(td);
2056 icd->error = 0;
f8b0bd10
JA
2057 if (io_u)
2058 io_u->error = 0;
f2bba182 2059 }
10ba535a
JA
2060}
2061
9520ebb9
JA
2062static void init_icd(struct thread_data *td, struct io_completion_data *icd,
2063 int nr)
10ba535a 2064{
6eaf09d6 2065 int ddir;
aba6c951
JA
2066
2067 if (!gtod_reduce(td))
9520ebb9 2068 fio_gettime(&icd->time, NULL);
02bcaa8c 2069
3af6ef39
JA
2070 icd->nr = nr;
2071
10ba535a 2072 icd->error = 0;
c1f50f76 2073 for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++)
6eaf09d6 2074 icd->bytes_done[ddir] = 0;
36167d82
JA
2075}
2076
97601024
JA
2077static void ios_completed(struct thread_data *td,
2078 struct io_completion_data *icd)
36167d82
JA
2079{
2080 struct io_u *io_u;
2081 int i;
2082
10ba535a
JA
2083 for (i = 0; i < icd->nr; i++) {
2084 io_u = td->io_ops->event(td, i);
2085
f8b0bd10 2086 io_completed(td, &io_u, icd);
e8462bd8 2087
f8b0bd10 2088 if (io_u)
e8462bd8 2089 put_io_u(td, io_u);
10ba535a
JA
2090 }
2091}
97601024 2092
e7e6cfb4
JA
2093/*
2094 * Complete a single io_u for the sync engines.
2095 */
55312f9f 2096int io_u_sync_complete(struct thread_data *td, struct io_u *io_u)
97601024
JA
2097{
2098 struct io_completion_data icd;
55312f9f 2099 int ddir;
97601024 2100
9520ebb9 2101 init_icd(td, &icd, 1);
f8b0bd10 2102 io_completed(td, &io_u, &icd);
e8462bd8 2103
f8b0bd10 2104 if (io_u)
e8462bd8 2105 put_io_u(td, io_u);
97601024 2106
581e7141
JA
2107 if (icd.error) {
2108 td_verror(td, icd.error, "io_u_sync_complete");
2109 return -1;
2110 }
97601024 2111
c1f50f76 2112 for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++)
55312f9f 2113 td->bytes_done[ddir] += icd.bytes_done[ddir];
581e7141
JA
2114
2115 return 0;
97601024
JA
2116}
2117
e7e6cfb4
JA
2118/*
2119 * Called to complete min_events number of io for the async engines.
2120 */
55312f9f 2121int io_u_queued_complete(struct thread_data *td, int min_evts)
97601024 2122{
97601024 2123 struct io_completion_data icd;
00de55ef 2124 struct timespec *tvp = NULL;
55312f9f 2125 int ret, ddir;
4d06a338 2126 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
97601024 2127
12bb8569 2128 dprint(FD_IO, "io_u_queued_complete: min=%d\n", min_evts);
b271fe62 2129
4950421a 2130 if (!min_evts)
00de55ef 2131 tvp = &ts;
5fb4b366
RE
2132 else if (min_evts > td->cur_depth)
2133 min_evts = td->cur_depth;
97601024 2134
82407585
RP
2135 /* No worries, td_io_getevents fixes min and max if they are
2136 * set incorrectly */
2137 ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete_max, tvp);
97601024 2138 if (ret < 0) {
e1161c32 2139 td_verror(td, -ret, "td_io_getevents");
97601024
JA
2140 return ret;
2141 } else if (!ret)
2142 return ret;
2143
9520ebb9 2144 init_icd(td, &icd, ret);
97601024 2145 ios_completed(td, &icd);
581e7141
JA
2146 if (icd.error) {
2147 td_verror(td, icd.error, "io_u_queued_complete");
2148 return -1;
2149 }
97601024 2150
c1f50f76 2151 for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++)
55312f9f 2152 td->bytes_done[ddir] += icd.bytes_done[ddir];
581e7141 2153
0d593542 2154 return ret;
97601024 2155}
7e77dd02
JA
2156
2157/*
2158 * Call when io_u is really queued, to update the submission latency.
2159 */
2160void io_u_queued(struct thread_data *td, struct io_u *io_u)
2161{
8243be59 2162 if (!td->o.disable_slat && ramp_time_over(td) && td->o.stats) {
9520ebb9 2163 unsigned long slat_time;
7e77dd02 2164
d6bb626e 2165 slat_time = ntime_since(&io_u->start_time, &io_u->issue_time);
75dc383e
JA
2166
2167 if (td->parent)
2168 td = td->parent;
2169
ae588852 2170 add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen,
03ec570f 2171 io_u->offset, io_u->ioprio);
9520ebb9 2172 }
7e77dd02 2173}
433afcb4 2174
5c94b008
JA
2175/*
2176 * See if we should reuse the last seed, if dedupe is enabled
2177 */
9451b93e 2178static struct frand_state *get_buf_state(struct thread_data *td)
5c94b008
JA
2179{
2180 unsigned int v;
0d71aa98 2181 unsigned long long i;
5c94b008
JA
2182
2183 if (!td->o.dedupe_percentage)
2184 return &td->buf_state;
732eedd0 2185 else if (td->o.dedupe_percentage == 100) {
9451b93e
JA
2186 frand_copy(&td->buf_state_prev, &td->buf_state);
2187 return &td->buf_state;
732eedd0 2188 }
5c94b008 2189
1bd5d213 2190 v = rand_between(&td->dedupe_state, 1, 100);
5c94b008 2191
0d71aa98
BD
2192 if (v <= td->o.dedupe_percentage)
2193 switch (td->o.dedupe_mode) {
2194 case DEDUPE_MODE_REPEAT:
2195 /*
2196 * The caller advances the returned frand_state.
2197 * A copy of prev should be returned instead since
2198 * a subsequent intention to generate a deduped buffer
2199 * might result in generating a unique one
2200 */
2201 frand_copy(&td->buf_state_ret, &td->buf_state_prev);
2202 return &td->buf_state_ret;
2203 case DEDUPE_MODE_WORKING_SET:
2204 i = rand_between(&td->dedupe_working_set_index_state, 0, td->num_unique_pages - 1);
2205 frand_copy(&td->buf_state_ret, &td->dedupe_working_set_states[i]);
2206 return &td->buf_state_ret;
2207 default:
2208 log_err("unexpected dedupe mode %u\n", td->o.dedupe_mode);
2209 assert(0);
2210 }
5c94b008
JA
2211
2212 return &td->buf_state;
2213}
2214
9451b93e 2215static void save_buf_state(struct thread_data *td, struct frand_state *rs)
5c94b008 2216{
9451b93e
JA
2217 if (td->o.dedupe_percentage == 100)
2218 frand_copy(rs, &td->buf_state_prev);
2219 else if (rs == &td->buf_state)
5c94b008
JA
2220 frand_copy(&td->buf_state_prev, rs);
2221}
2222
5fff9543
JF
2223void fill_io_buffer(struct thread_data *td, void *buf, unsigned long long min_write,
2224 unsigned long long max_bs)
5973cafb 2225{
d1af2894
JA
2226 struct thread_options *o = &td->o;
2227
15600335
JA
2228 if (o->mem_type == MEM_CUDA_MALLOC)
2229 return;
03553853 2230
4eff3e57 2231 if (o->compress_percentage || o->dedupe_percentage) {
9c42684e 2232 unsigned int perc = td->o.compress_percentage;
5c94b008 2233 struct frand_state *rs;
5fff9543
JF
2234 unsigned long long left = max_bs;
2235 unsigned long long this_write;
5c94b008 2236
1066358a 2237 do {
9451b93e 2238 rs = get_buf_state(td);
9c42684e 2239
1066358a 2240 min_write = min(min_write, left);
f97a43a1 2241
1066358a 2242 if (perc) {
4b157ac6 2243 this_write = min_not_zero(min_write,
5fff9543 2244 (unsigned long long) td->o.compress_chunk);
1e7f82e2
JA
2245
2246 fill_random_buf_percentage(rs, buf, perc,
2247 this_write, this_write,
2248 o->buffer_pattern,
2249 o->buffer_pattern_bytes);
2250 } else {
1066358a 2251 fill_random_buf(rs, buf, min_write);
1e7f82e2
JA
2252 this_write = min_write;
2253 }
1066358a 2254
1e7f82e2
JA
2255 buf += this_write;
2256 left -= this_write;
9451b93e 2257 save_buf_state(td, rs);
1066358a 2258 } while (left);
d1af2894
JA
2259 } else if (o->buffer_pattern_bytes)
2260 fill_buffer_pattern(td, buf, max_bs);
999d245e 2261 else if (o->zero_buffers)
cc86c395 2262 memset(buf, 0, max_bs);
999d245e 2263 else
9451b93e 2264 fill_random_buf(get_buf_state(td), buf, max_bs);
cc86c395
JA
2265}
2266
2267/*
2268 * "randomly" fill the buffer contents
2269 */
2270void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
5fff9543 2271 unsigned long long min_write, unsigned long long max_bs)
cc86c395
JA
2272{
2273 io_u->buf_filled_len = 0;
2274 fill_io_buffer(td, io_u->buf, min_write, max_bs);
5973cafb 2275}
e2c75fc4
TK
2276
2277static int do_sync_file_range(const struct thread_data *td,
2278 struct fio_file *f)
2279{
7e2a317e 2280 uint64_t offset, nbytes;
e2c75fc4
TK
2281
2282 offset = f->first_write;
2283 nbytes = f->last_write - f->first_write;
2284
2285 if (!nbytes)
2286 return 0;
2287
2288 return sync_file_range(f->fd, offset, nbytes, td->o.sync_file_range);
2289}
2290
2291int do_io_u_sync(const struct thread_data *td, struct io_u *io_u)
2292{
2293 int ret;
2294
2295 if (io_u->ddir == DDIR_SYNC) {
2296 ret = fsync(io_u->file->fd);
2297 } else if (io_u->ddir == DDIR_DATASYNC) {
2298#ifdef CONFIG_FDATASYNC
2299 ret = fdatasync(io_u->file->fd);
2300#else
2301 ret = io_u->xfer_buflen;
2302 io_u->error = EINVAL;
2303#endif
2304 } else if (io_u->ddir == DDIR_SYNC_FILE_RANGE)
2305 ret = do_sync_file_range(td, io_u->file);
2306 else {
2307 ret = io_u->xfer_buflen;
2308 io_u->error = EINVAL;
2309 }
2310
2311 if (ret < 0)
2312 io_u->error = errno;
2313
2314 return ret;
2315}
2316
2317int do_io_u_trim(const struct thread_data *td, struct io_u *io_u)
2318{
2319#ifndef FIO_HAVE_TRIM
2320 io_u->error = EINVAL;
2321 return 0;
2322#else
2323 struct fio_file *f = io_u->file;
2324 int ret;
2325
e3be810b
SK
2326 if (td->o.zone_mode == ZONE_MODE_ZBD) {
2327 ret = zbd_do_io_u_trim(td, io_u);
2328 if (ret == io_u_completed)
2329 return io_u->xfer_buflen;
2330 if (ret)
2331 goto err;
2332 }
2333
496b1f9e 2334 ret = os_trim(f, io_u->offset, io_u->xfer_buflen);
e2c75fc4
TK
2335 if (!ret)
2336 return io_u->xfer_buflen;
2337
e3be810b 2338err:
e2c75fc4
TK
2339 io_u->error = ret;
2340 return 0;
2341#endif
2342}