13 struct io_completion_data {
16 int error; /* output */
17 unsigned long bytes_done[2]; /* output */
18 struct timeval time; /* output */
22 * The ->file_map[] contains a map of blocks we have or have not done io
23 * to yet. Used to make sure we cover the entire range in a fair fashion.
25 static int random_map_free(struct fio_file *f, const unsigned long long block)
27 unsigned int idx = RAND_MAP_IDX(f, block);
28 unsigned int bit = RAND_MAP_BIT(f, block);
30 dprint(FD_RANDOM, "free: b=%llu, idx=%u, bit=%u\n", block, idx, bit);
32 return (f->file_map[idx] & (1 << bit)) == 0;
36 * Mark a given offset as used in the map.
38 static void mark_random_map(struct thread_data *td, struct io_u *io_u)
40 unsigned int min_bs = td->o.rw_min_bs;
41 struct fio_file *f = io_u->file;
42 unsigned long long block;
43 unsigned int blocks, nr_blocks;
46 block = (io_u->offset - f->file_offset) / (unsigned long long) min_bs;
47 nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
49 busy_check = !(io_u->flags & IO_U_F_BUSY_OK);
52 unsigned int this_blocks, mask;
53 unsigned int idx, bit;
56 * If we have a mixed random workload, we may
57 * encounter blocks we already did IO to.
63 if ((td->o.ddir_seq_nr == 1) && !random_map_free(f, block))
66 idx = RAND_MAP_IDX(f, block);
67 bit = RAND_MAP_BIT(f, block);
69 fio_assert(td, idx < f->num_maps);
71 this_blocks = nr_blocks;
72 if (this_blocks + bit > BLOCKS_PER_MAP)
73 this_blocks = BLOCKS_PER_MAP - bit;
76 if (this_blocks == BLOCKS_PER_MAP)
79 mask = ((1U << this_blocks) - 1) << bit;
81 if (!(f->file_map[idx] & mask))
85 } while (this_blocks);
90 f->file_map[idx] |= mask;
91 nr_blocks -= this_blocks;
92 blocks += this_blocks;
96 if ((blocks * min_bs) < io_u->buflen)
97 io_u->buflen = blocks * min_bs;
100 static unsigned long long last_block(struct thread_data *td, struct fio_file *f,
103 unsigned long long max_blocks;
104 unsigned long long max_size;
107 * Hmm, should we make sure that ->io_size <= ->real_file_size?
109 max_size = f->io_size;
110 if (max_size > f->real_file_size)
111 max_size = f->real_file_size;
113 max_blocks = max_size / (unsigned long long) td->o.ba[ddir];
121 * Return the next free block in the map.
123 static int get_next_free_block(struct thread_data *td, struct fio_file *f,
124 enum fio_ddir ddir, unsigned long long *b)
126 unsigned long long min_bs = td->o.rw_min_bs;
129 i = f->last_free_lookup;
130 *b = (i * BLOCKS_PER_MAP);
131 while ((*b) * min_bs < f->real_file_size &&
132 (*b) * min_bs < f->io_size) {
133 if (f->file_map[i] != (unsigned int) -1) {
134 *b += ffz(f->file_map[i]);
135 if (*b > last_block(td, f, ddir))
137 f->last_free_lookup = i;
141 *b += BLOCKS_PER_MAP;
145 dprint(FD_IO, "failed finding a free block\n");
149 static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
150 enum fio_ddir ddir, unsigned long long *b)
152 unsigned long long r;
156 r = os_random_long(&td->random_state);
157 dprint(FD_RANDOM, "off rand %llu\n", r);
158 *b = (last_block(td, f, ddir) - 1)
159 * (r / ((unsigned long long) OS_RAND_MAX + 1.0));
162 * if we are not maintaining a random map, we are done.
164 if (!file_randommap(td, f))
168 * calculate map offset and check if it's free
170 if (random_map_free(f, *b))
173 dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
178 * we get here, if we didn't suceed in looking up a block. generate
179 * a random start offset into the filemap, and find the first free
184 f->last_free_lookup = (f->num_maps - 1) *
185 (r / (OS_RAND_MAX + 1.0));
186 if (!get_next_free_block(td, f, ddir, b))
189 r = os_random_long(&td->random_state);
193 * that didn't work either, try exhaustive search from the start
195 f->last_free_lookup = 0;
196 return get_next_free_block(td, f, ddir, b);
199 static int get_next_rand_block(struct thread_data *td, struct fio_file *f,
200 enum fio_ddir ddir, unsigned long long *b)
202 if (get_next_rand_offset(td, f, ddir, b)) {
203 dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n",
204 f->file_name, f->last_pos, f->real_file_size);
211 static int get_next_seq_block(struct thread_data *td, struct fio_file *f,
212 enum fio_ddir ddir, unsigned long long *b)
214 if (f->last_pos < f->real_file_size) {
215 *b = (f->last_pos - f->file_offset) / td->o.min_bs[ddir];
222 static int get_next_block(struct thread_data *td, struct io_u *io_u,
223 enum fio_ddir ddir, int rw_seq, unsigned long long *b)
225 struct fio_file *f = io_u->file;
230 ret = get_next_rand_block(td, f, ddir, b);
232 ret = get_next_seq_block(td, f, ddir, b);
234 io_u->flags |= IO_U_F_BUSY_OK;
236 if (td->o.rw_seq == RW_SEQ_SEQ) {
237 ret = get_next_seq_block(td, f, ddir, b);
239 ret = get_next_rand_block(td, f, ddir, b);
240 } else if (td->o.rw_seq == RW_SEQ_IDENT) {
241 if (f->last_start != -1ULL)
242 *b = (f->last_start - f->file_offset) / td->o.min_bs[ddir];
247 log_err("fio: unknown rw_seq=%d\n", td->o.rw_seq);
256 * For random io, generate a random new block and see if it's used. Repeat
257 * until we find a free one. For sequential io, just return the end of
258 * the last io issued.
260 static int __get_next_offset(struct thread_data *td, struct io_u *io_u)
262 struct fio_file *f = io_u->file;
263 unsigned long long b;
264 enum fio_ddir ddir = io_u->ddir;
267 if (td->o.ddir_seq_nr && !--td->ddir_seq_nr) {
269 td->ddir_seq_nr = td->o.ddir_seq_nr;
272 if (get_next_block(td, io_u, ddir, rw_seq_hit, &b)) {
277 io_u->offset = b * td->o.ba[ddir];
278 if (io_u->offset >= f->io_size) {
279 dprint(FD_IO, "get_next_offset: offset %llu >= io_size %llu\n",
280 io_u->offset, f->io_size);
284 io_u->offset += f->file_offset;
285 if (io_u->offset >= f->real_file_size) {
286 dprint(FD_IO, "get_next_offset: offset %llu >= size %llu\n",
287 io_u->offset, f->real_file_size);
294 static int get_next_offset(struct thread_data *td, struct io_u *io_u)
296 struct prof_io_ops *ops = &td->prof_io_ops;
298 if (ops->fill_io_u_off)
299 return ops->fill_io_u_off(td, io_u);
301 return __get_next_offset(td, io_u);
304 static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u)
306 const int ddir = io_u->ddir;
307 unsigned int uninitialized_var(buflen);
308 unsigned int minbs, maxbs;
311 minbs = td->o.min_bs[ddir];
312 maxbs = td->o.max_bs[ddir];
317 r = os_random_long(&td->bsrange_state);
318 if (!td->o.bssplit_nr[ddir]) {
319 buflen = 1 + (unsigned int) ((double) maxbs *
320 (r / (OS_RAND_MAX + 1.0)));
327 for (i = 0; i < td->o.bssplit_nr[ddir]; i++) {
328 struct bssplit *bsp = &td->o.bssplit[ddir][i];
332 if (r <= ((OS_RAND_MAX / 100L) * perc))
336 if (!td->o.bs_unaligned && is_power_of_2(minbs))
337 buflen = (buflen + minbs - 1) & ~(minbs - 1);
340 if (io_u->offset + buflen > io_u->file->real_file_size) {
341 dprint(FD_IO, "lower buflen %u -> %u (ddir=%d)\n", buflen,
349 static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u)
351 struct prof_io_ops *ops = &td->prof_io_ops;
353 if (ops->fill_io_u_size)
354 return ops->fill_io_u_size(td, io_u);
356 return __get_next_buflen(td, io_u);
359 static void set_rwmix_bytes(struct thread_data *td)
364 * we do time or byte based switch. this is needed because
365 * buffered writes may issue a lot quicker than they complete,
366 * whereas reads do not.
368 diff = td->o.rwmix[td->rwmix_ddir ^ 1];
369 td->rwmix_issues = (td->io_issues[td->rwmix_ddir] * diff) / 100;
372 static inline enum fio_ddir get_rand_ddir(struct thread_data *td)
377 r = os_random_long(&td->rwmix_state);
378 v = 1 + (int) (100.0 * (r / (OS_RAND_MAX + 1.0)));
379 if (v <= td->o.rwmix[DDIR_READ])
385 static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)
387 enum fio_ddir odir = ddir ^ 1;
391 if (td->rate_pending_usleep[ddir] <= 0)
395 * We have too much pending sleep in this direction. See if we
400 * Other direction does not have too much pending, switch
402 if (td->rate_pending_usleep[odir] < 100000)
406 * Both directions have pending sleep. Sleep the minimum time
407 * and deduct from both.
409 if (td->rate_pending_usleep[ddir] <=
410 td->rate_pending_usleep[odir]) {
411 usec = td->rate_pending_usleep[ddir];
413 usec = td->rate_pending_usleep[odir];
417 usec = td->rate_pending_usleep[ddir];
419 fio_gettime(&t, NULL);
420 usec_sleep(td, usec);
421 usec = utime_since_now(&t);
423 td->rate_pending_usleep[ddir] -= usec;
426 if (td_rw(td) && __should_check_rate(td, odir))
427 td->rate_pending_usleep[odir] -= usec;
433 * Return the data direction for the next io_u. If the job is a
434 * mixed read/write workload, check the rwmix cycle and switch if
437 static enum fio_ddir get_rw_ddir(struct thread_data *td)
442 * see if it's time to fsync
444 if (td->o.fsync_blocks &&
445 !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) &&
446 td->io_issues[DDIR_WRITE] && should_fsync(td))
450 * see if it's time to fdatasync
452 if (td->o.fdatasync_blocks &&
453 !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks) &&
454 td->io_issues[DDIR_WRITE] && should_fsync(td))
455 return DDIR_DATASYNC;
458 * see if it's time to sync_file_range
460 if (td->sync_file_range_nr &&
461 !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr) &&
462 td->io_issues[DDIR_WRITE] && should_fsync(td))
463 return DDIR_SYNC_FILE_RANGE;
467 * Check if it's time to seed a new data direction.
469 if (td->io_issues[td->rwmix_ddir] >= td->rwmix_issues) {
471 * Put a top limit on how many bytes we do for
472 * one data direction, to avoid overflowing the
475 ddir = get_rand_ddir(td);
477 if (ddir != td->rwmix_ddir)
480 td->rwmix_ddir = ddir;
482 ddir = td->rwmix_ddir;
483 } else if (td_read(td))
488 td->rwmix_ddir = rate_ddir(td, ddir);
489 return td->rwmix_ddir;
492 void put_file_log(struct thread_data *td, struct fio_file *f)
494 int ret = put_file(td, f);
497 td_verror(td, ret, "file close");
500 void put_io_u(struct thread_data *td, struct io_u *io_u)
504 io_u->flags |= IO_U_F_FREE;
505 io_u->flags &= ~IO_U_F_FREE_DEF;
508 put_file_log(td, io_u->file);
511 if (io_u->flags & IO_U_F_IN_CUR_DEPTH)
513 flist_del_init(&io_u->list);
514 flist_add(&io_u->list, &td->io_u_freelist);
516 td_io_u_free_notify(td);
519 void clear_io_u(struct thread_data *td, struct io_u *io_u)
521 io_u->flags &= ~IO_U_F_FLIGHT;
525 void requeue_io_u(struct thread_data *td, struct io_u **io_u)
527 struct io_u *__io_u = *io_u;
529 dprint(FD_IO, "requeue %p\n", __io_u);
533 __io_u->flags |= IO_U_F_FREE;
534 if ((__io_u->flags & IO_U_F_FLIGHT) && !ddir_sync(__io_u->ddir))
535 td->io_issues[__io_u->ddir]--;
537 __io_u->flags &= ~IO_U_F_FLIGHT;
538 if (__io_u->flags & IO_U_F_IN_CUR_DEPTH)
540 flist_del(&__io_u->list);
541 flist_add_tail(&__io_u->list, &td->io_u_requeues);
546 static int fill_io_u(struct thread_data *td, struct io_u *io_u)
548 if (td->io_ops->flags & FIO_NOIO)
551 io_u->ddir = get_rw_ddir(td);
554 * fsync() or fdatasync(), we are done
556 if (ddir_sync(io_u->ddir))
560 * See if it's time to switch to a new zone
562 if (td->zone_bytes >= td->o.zone_size) {
564 io_u->file->last_pos += td->o.zone_skip;
565 td->io_skip_bytes += td->o.zone_skip;
569 * No log, let the seq/rand engine retrieve the next buflen and
572 if (get_next_offset(td, io_u)) {
573 dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
577 io_u->buflen = get_next_buflen(td, io_u);
579 dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
583 if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
584 dprint(FD_IO, "io_u %p, offset too large\n", io_u);
585 dprint(FD_IO, " off=%llu/%lu > %llu\n", io_u->offset,
586 io_u->buflen, io_u->file->real_file_size);
591 * mark entry before potentially trimming io_u
593 if (td_random(td) && file_randommap(td, io_u->file))
594 mark_random_map(td, io_u);
597 * If using a write iolog, store this entry.
600 dprint_io_u(io_u, "fill_io_u");
601 td->zone_bytes += io_u->buflen;
606 static void __io_u_mark_map(unsigned int *map, unsigned int nr)
635 void io_u_mark_submit(struct thread_data *td, unsigned int nr)
637 __io_u_mark_map(td->ts.io_u_submit, nr);
638 td->ts.total_submit++;
641 void io_u_mark_complete(struct thread_data *td, unsigned int nr)
643 __io_u_mark_map(td->ts.io_u_complete, nr);
644 td->ts.total_complete++;
647 void io_u_mark_depth(struct thread_data *td, unsigned int nr)
651 switch (td->cur_depth) {
673 td->ts.io_u_map[index] += nr;
676 static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec)
713 assert(index < FIO_IO_U_LAT_U_NR);
714 td->ts.io_u_lat_u[index]++;
717 static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec)
758 assert(index < FIO_IO_U_LAT_M_NR);
759 td->ts.io_u_lat_m[index]++;
762 static void io_u_mark_latency(struct thread_data *td, unsigned long usec)
765 io_u_mark_lat_usec(td, usec);
767 io_u_mark_lat_msec(td, usec / 1000);
771 * Get next file to service by choosing one at random
773 static struct fio_file *get_next_file_rand(struct thread_data *td,
774 enum fio_file_flags goodf,
775 enum fio_file_flags badf)
781 long r = os_random_long(&td->next_file_state);
784 fno = (unsigned int) ((double) td->o.nr_files
785 * (r / (OS_RAND_MAX + 1.0)));
787 if (fio_file_done(f))
790 if (!fio_file_open(f)) {
793 err = td_io_open_file(td, f);
799 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) {
800 dprint(FD_FILE, "get_next_file_rand: %p\n", f);
804 td_io_close_file(td, f);
809 * Get next file to service by doing round robin between all available ones
811 static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
814 unsigned int old_next_file = td->next_file;
820 f = td->files[td->next_file];
823 if (td->next_file >= td->o.nr_files)
826 dprint(FD_FILE, "trying file %s %x\n", f->file_name, f->flags);
827 if (fio_file_done(f)) {
832 if (!fio_file_open(f)) {
835 err = td_io_open_file(td, f);
837 dprint(FD_FILE, "error %d on open of %s\n",
845 dprint(FD_FILE, "goodf=%x, badf=%x, ff=%x\n", goodf, badf,
847 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
851 td_io_close_file(td, f);
854 } while (td->next_file != old_next_file);
856 dprint(FD_FILE, "get_next_file_rr: %p\n", f);
860 static struct fio_file *__get_next_file(struct thread_data *td)
864 assert(td->o.nr_files <= td->files_index);
866 if (td->nr_done_files >= td->o.nr_files) {
867 dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d,"
868 " nr_files=%d\n", td->nr_open_files,
874 f = td->file_service_file;
875 if (f && fio_file_open(f) && !fio_file_closing(f)) {
876 if (td->o.file_service_type == FIO_FSERVICE_SEQ)
878 if (td->file_service_left--)
882 if (td->o.file_service_type == FIO_FSERVICE_RR ||
883 td->o.file_service_type == FIO_FSERVICE_SEQ)
884 f = get_next_file_rr(td, FIO_FILE_open, FIO_FILE_closing);
886 f = get_next_file_rand(td, FIO_FILE_open, FIO_FILE_closing);
888 td->file_service_file = f;
889 td->file_service_left = td->file_service_nr - 1;
891 dprint(FD_FILE, "get_next_file: %p [%s]\n", f, f->file_name);
895 static struct fio_file *get_next_file(struct thread_data *td)
897 struct prof_io_ops *ops = &td->prof_io_ops;
899 if (ops->get_next_file)
900 return ops->get_next_file(td);
902 return __get_next_file(td);
905 static int set_io_u_file(struct thread_data *td, struct io_u *io_u)
910 f = get_next_file(td);
917 if (!fill_io_u(td, io_u))
921 td_io_close_file(td, f);
923 fio_file_set_done(f);
925 dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name,
926 td->nr_done_files, td->o.nr_files);
933 struct io_u *__get_io_u(struct thread_data *td)
935 struct io_u *io_u = NULL;
940 if (!flist_empty(&td->io_u_requeues))
941 io_u = flist_entry(td->io_u_requeues.next, struct io_u, list);
942 else if (!queue_full(td)) {
943 io_u = flist_entry(td->io_u_freelist.next, struct io_u, list);
952 assert(io_u->flags & IO_U_F_FREE);
953 io_u->flags &= ~(IO_U_F_FREE | IO_U_F_FREE_DEF);
956 flist_del(&io_u->list);
957 flist_add(&io_u->list, &td->io_u_busylist);
959 io_u->flags |= IO_U_F_IN_CUR_DEPTH;
960 } else if (td->o.verify_async) {
962 * We ran out, wait for async verify threads to finish and
965 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
974 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
975 * etc. The returned io_u is fully ready to be prepped and submitted.
977 struct io_u *get_io_u(struct thread_data *td)
982 io_u = __get_io_u(td);
984 dprint(FD_IO, "__get_io_u failed\n");
988 if (td->o.verify_backlog && td->io_hist_len) {
991 if (td->verify_batch) {
994 } else if (!(td->io_hist_len % td->o.verify_backlog) &&
995 td->last_ddir != DDIR_READ) {
996 td->verify_batch = td->o.verify_batch;
997 if (!td->verify_batch)
998 td->verify_batch = td->o.verify_backlog;
1002 if (get_verify && !get_next_verify(td, io_u))
1007 * from a requeue, io_u already setup
1013 * If using an iolog, grab next piece if any available.
1015 if (td->o.read_iolog_file) {
1016 if (read_iolog_get(td, io_u))
1018 } else if (set_io_u_file(td, io_u)) {
1019 dprint(FD_IO, "io_u %p, setting file failed\n", io_u);
1024 assert(fio_file_open(f));
1026 if (!ddir_sync(io_u->ddir)) {
1027 if (!io_u->buflen && !(td->io_ops->flags & FIO_NOIO)) {
1028 dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u);
1032 f->last_start = io_u->offset;
1033 f->last_pos = io_u->offset + io_u->buflen;
1035 if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_WRITE)
1036 populate_verify_io_u(td, io_u);
1037 else if (td->o.refill_buffers && io_u->ddir == DDIR_WRITE)
1038 io_u_fill_buffer(td, io_u, io_u->xfer_buflen);
1039 else if (io_u->ddir == DDIR_READ) {
1041 * Reset the buf_filled parameters so next time if the
1042 * buffer is used for writes it is refilled.
1044 io_u->buf_filled_len = 0;
1049 * Set io data pointers.
1051 io_u->xfer_buf = io_u->buf;
1052 io_u->xfer_buflen = io_u->buflen;
1055 if (!td_io_prep(td, io_u)) {
1056 if (!td->o.disable_slat)
1057 fio_gettime(&io_u->start_time, NULL);
1061 dprint(FD_IO, "get_io_u failed\n");
1066 void io_u_log_error(struct thread_data *td, struct io_u *io_u)
1068 const char *msg[] = { "read", "write", "sync" };
1070 log_err("fio: io_u error");
1073 log_err(" on file %s", io_u->file->file_name);
1075 log_err(": %s\n", strerror(io_u->error));
1077 log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir],
1078 io_u->offset, io_u->xfer_buflen);
1081 td_verror(td, io_u->error, "io_u error");
1084 static void io_completed(struct thread_data *td, struct io_u *io_u,
1085 struct io_completion_data *icd)
1088 * Older gcc's are too dumb to realize that usec is always used
1089 * initialized, silence that warning.
1091 unsigned long uninitialized_var(usec);
1094 dprint_io_u(io_u, "io complete");
1097 assert(io_u->flags & IO_U_F_FLIGHT);
1098 io_u->flags &= ~(IO_U_F_FLIGHT | IO_U_F_BUSY_OK);
1101 if (ddir_sync(io_u->ddir)) {
1102 td->last_was_sync = 1;
1105 f->first_write = -1ULL;
1106 f->last_write = -1ULL;
1111 td->last_was_sync = 0;
1112 td->last_ddir = io_u->ddir;
1115 unsigned int bytes = io_u->buflen - io_u->resid;
1116 const enum fio_ddir idx = io_u->ddir;
1117 const enum fio_ddir odx = io_u->ddir ^ 1;
1120 td->io_blocks[idx]++;
1121 td->io_bytes[idx] += bytes;
1122 td->this_io_bytes[idx] += bytes;
1124 if (idx == DDIR_WRITE) {
1127 if (f->first_write == -1ULL ||
1128 io_u->offset < f->first_write)
1129 f->first_write = io_u->offset;
1130 if (f->last_write == -1ULL ||
1131 ((io_u->offset + bytes) > f->last_write))
1132 f->last_write = io_u->offset + bytes;
1136 if (ramp_time_over(td)) {
1137 unsigned long uninitialized_var(lusec);
1139 if (!td->o.disable_clat || !td->o.disable_bw)
1140 lusec = utime_since(&io_u->issue_time,
1142 if (!td->o.disable_lat) {
1143 unsigned long tusec;
1145 tusec = utime_since(&io_u->start_time,
1147 add_lat_sample(td, idx, tusec, bytes);
1149 if (!td->o.disable_clat) {
1150 add_clat_sample(td, idx, lusec, bytes);
1151 io_u_mark_latency(td, lusec);
1153 if (!td->o.disable_bw)
1154 add_bw_sample(td, idx, bytes, &icd->time);
1155 if (__should_check_rate(td, idx)) {
1156 td->rate_pending_usleep[idx] =
1157 ((td->this_io_bytes[idx] *
1158 td->rate_nsec_cycle[idx]) / 1000 -
1159 utime_since_now(&td->start));
1161 if (__should_check_rate(td, idx ^ 1))
1162 td->rate_pending_usleep[odx] =
1163 ((td->this_io_bytes[odx] *
1164 td->rate_nsec_cycle[odx]) / 1000 -
1165 utime_since_now(&td->start));
1168 if (td_write(td) && idx == DDIR_WRITE &&
1170 td->o.verify != VERIFY_NONE)
1171 log_io_piece(td, io_u);
1173 icd->bytes_done[idx] += bytes;
1176 ret = io_u->end_io(td, io_u);
1177 if (ret && !icd->error)
1181 icd->error = io_u->error;
1182 io_u_log_error(td, io_u);
1184 if (td->o.continue_on_error && icd->error &&
1185 td_non_fatal_error(icd->error)) {
1187 * If there is a non_fatal error, then add to the error count
1188 * and clear all the errors.
1190 update_error_count(td, icd->error);
1197 static void init_icd(struct thread_data *td, struct io_completion_data *icd,
1200 if (!td->o.disable_clat || !td->o.disable_bw)
1201 fio_gettime(&icd->time, NULL);
1206 icd->bytes_done[0] = icd->bytes_done[1] = 0;
1209 static void ios_completed(struct thread_data *td,
1210 struct io_completion_data *icd)
1215 for (i = 0; i < icd->nr; i++) {
1216 io_u = td->io_ops->event(td, i);
1218 io_completed(td, io_u, icd);
1220 if (!(io_u->flags & IO_U_F_FREE_DEF))
1226 * Complete a single io_u for the sync engines.
1228 int io_u_sync_complete(struct thread_data *td, struct io_u *io_u,
1229 unsigned long *bytes)
1231 struct io_completion_data icd;
1233 init_icd(td, &icd, 1);
1234 io_completed(td, io_u, &icd);
1236 if (!(io_u->flags & IO_U_F_FREE_DEF))
1240 td_verror(td, icd.error, "io_u_sync_complete");
1245 bytes[0] += icd.bytes_done[0];
1246 bytes[1] += icd.bytes_done[1];
1253 * Called to complete min_events number of io for the async engines.
1255 int io_u_queued_complete(struct thread_data *td, int min_evts,
1256 unsigned long *bytes)
1258 struct io_completion_data icd;
1259 struct timespec *tvp = NULL;
1261 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
1263 dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_evts);
1268 ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete, tvp);
1270 td_verror(td, -ret, "td_io_getevents");
1275 init_icd(td, &icd, ret);
1276 ios_completed(td, &icd);
1278 td_verror(td, icd.error, "io_u_queued_complete");
1283 bytes[0] += icd.bytes_done[0];
1284 bytes[1] += icd.bytes_done[1];
1291 * Call when io_u is really queued, to update the submission latency.
1293 void io_u_queued(struct thread_data *td, struct io_u *io_u)
1295 if (!td->o.disable_slat) {
1296 unsigned long slat_time;
1298 slat_time = utime_since(&io_u->start_time, &io_u->issue_time);
1299 add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen);
1304 * "randomly" fill the buffer contents
1306 void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
1307 unsigned int max_bs)
1309 if (!td->o.zero_buffers)
1310 fill_random_buf(io_u->buf, max_bs);
1312 memset(io_u->buf, 0, max_bs);