-struct io_u *__get_io_u(struct thread_data *td)
-{
- struct io_u *io_u;
-
- if (queue_full(td))
- return NULL;
-
- io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
- io_u->error = 0;
- io_u->resid = 0;
- list_del(&io_u->list);
- list_add(&io_u->list, &td->io_u_busylist);
- td->cur_depth++;
- return io_u;
-}
-
-static struct io_u *get_io_u(struct thread_data *td)
-{
- struct io_u *io_u;
-
- io_u = __get_io_u(td);
- if (!io_u)
- return NULL;
-
- if (td->zone_bytes >= td->zone_size) {
- td->zone_bytes = 0;
- td->last_pos += td->zone_skip;
- }
-
- if (fill_io_u(td, io_u)) {
- put_io_u(td, io_u);
- return NULL;
- }
-
- if (io_u->buflen + io_u->offset > td->real_file_size)
- io_u->buflen = td->real_file_size - io_u->offset;
-
- if (!io_u->buflen) {
- put_io_u(td, io_u);
- return NULL;
- }
-
- if (!td->iolog && !td->sequential)
- mark_random_map(td, io_u);
-
- td->last_pos += io_u->buflen;
-
- if (td->verify != VERIFY_NONE)
- populate_io_u(td, io_u);
-
- if (td_io_prep(td, io_u)) {
- put_io_u(td, io_u);
- return NULL;
- }
-
- gettimeofday(&io_u->start_time, NULL);
- return io_u;
-}
-
-static inline void td_set_runstate(struct thread_data *td, int runstate)
-{
- td->old_runstate = td->runstate;
- td->runstate = runstate;
-}
-
-static int get_next_verify(struct thread_data *td, struct io_u *io_u)
-{
- struct io_piece *ipo;
-
- if (list_empty(&td->io_hist_list))
- return 1;
-
- ipo = list_entry(td->io_hist_list.next, struct io_piece, list);
- list_del(&ipo->list);
-
- io_u->offset = ipo->offset;
- io_u->buflen = ipo->len;
- io_u->ddir = DDIR_READ;
- free(ipo);
- return 0;
-}
-
-static void prune_io_piece_log(struct thread_data *td)
-{
- struct io_piece *ipo;
-
- while (!list_empty(&td->io_hist_list)) {
- ipo = list_entry(td->io_hist_list.next, struct io_piece, list);
-
- list_del(&ipo->list);
- free(ipo);
- }
-}
-
-/*
- * log a succesful write, so we can unwind the log for verify
- */
-static void log_io_piece(struct thread_data *td, struct io_u *io_u)
-{
- struct io_piece *ipo = malloc(sizeof(struct io_piece));
- struct list_head *entry;
-
- INIT_LIST_HEAD(&ipo->list);
- ipo->offset = io_u->offset;
- ipo->len = io_u->buflen;
-
- /*
- * for random io where the writes extend the file, it will typically
- * be laid out with the block scattered as written. it's faster to
- * read them in in that order again, so don't sort
- */
- if (td->sequential || !td->overwrite) {
- list_add_tail(&ipo->list, &td->io_hist_list);
- return;
- }
-
- /*
- * for random io, sort the list so verify will run faster
- */
- entry = &td->io_hist_list;
- while ((entry = entry->prev) != &td->io_hist_list) {
- struct io_piece *__ipo = list_entry(entry, struct io_piece, list);
-
- if (__ipo->offset < ipo->offset)
- break;
- }
-
- list_add(&ipo->list, entry);
-}
-
-static int init_iolog(struct thread_data *td)
-{
- unsigned long long offset;
- unsigned int bytes;
- char *str, *p;
- FILE *f;
- int rw, i, reads, writes;
-
- if (!td->iolog)
- return 0;
-
- f = fopen(td->iolog_file, "r");
- if (!f) {
- perror("fopen iolog");
- return 1;
- }
-
- str = malloc(4096);
- reads = writes = i = 0;
- while ((p = fgets(str, 4096, f)) != NULL) {
- struct io_piece *ipo;
-
- if (sscanf(p, "%d,%llu,%u", &rw, &offset, &bytes) != 3) {
- fprintf(stderr, "bad iolog: %s\n", p);
- continue;
- }
- if (rw == DDIR_READ)
- reads++;
- else if (rw == DDIR_WRITE)
- writes++;
- else {
- fprintf(stderr, "bad ddir: %d\n", rw);
- continue;
- }
-
- ipo = malloc(sizeof(*ipo));
- INIT_LIST_HEAD(&ipo->list);
- ipo->offset = offset;
- ipo->len = bytes;
- if (bytes > td->max_bs)
- td->max_bs = bytes;
- ipo->ddir = rw;
- list_add_tail(&ipo->list, &td->io_log_list);
- i++;
- }
-
- free(str);
- fclose(f);
-
- if (!i)
- return 1;
-
- if (reads && !writes)
- td->ddir = DDIR_READ;
- else if (!reads && writes)
- td->ddir = DDIR_READ;
- else
- td->iomix = 1;
-
- return 0;
-}
-
-static int sync_td(struct thread_data *td)
-{
- if (td->io_sync)
- return td->io_sync(td);
-
- return 0;
-}
-
-static int io_u_getevents(struct thread_data *td, int min, int max,
- struct timespec *t)
-{
- return td->io_getevents(td, min, max, t);
-}
-
-static int io_u_queue(struct thread_data *td, struct io_u *io_u)
-{
- gettimeofday(&io_u->issue_time, NULL);
-
- return td->io_queue(td, io_u);
-}
-
-#define iocb_time(iocb) ((unsigned long) (iocb)->data)
-
-static void io_completed(struct thread_data *td, struct io_u *io_u,
- struct io_completion_data *icd)
-{
- struct timeval e;
- unsigned long msec;
-
- gettimeofday(&e, NULL);
-
- if (!io_u->error) {
- unsigned int bytes = io_u->buflen - io_u->resid;
- const int idx = io_u->ddir;
-
- td->io_blocks[idx]++;
- td->io_bytes[idx] += bytes;
- td->zone_bytes += bytes;
- td->this_io_bytes[idx] += bytes;
-
- msec = mtime_since(&io_u->issue_time, &e);
-
- add_clat_sample(td, idx, msec);
- add_bw_sample(td, idx);
-
- if ((td_rw(td) || td_write(td)) && idx == DDIR_WRITE)
- log_io_piece(td, io_u);
-
- icd->bytes_done[idx] += bytes;
- } else
- icd->error = io_u->error;
-}
-
-static void ios_completed(struct thread_data *td,struct io_completion_data *icd)
-{
- struct io_u *io_u;
- int i;
-
- icd->error = 0;
- icd->bytes_done[0] = icd->bytes_done[1] = 0;
-
- for (i = 0; i < icd->nr; i++) {
- io_u = td->io_event(td, i);
-
- io_completed(td, io_u, icd);
- put_io_u(td, io_u);
- }
-}
-
-static void cleanup_pending_aio(struct thread_data *td)
-{
- struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
- struct list_head *entry, *n;
- struct io_completion_data icd;
- struct io_u *io_u;
- int r;
-
- /*
- * get immediately available events, if any
- */
- r = io_u_getevents(td, 0, td->cur_depth, &ts);
- if (r > 0) {
- icd.nr = r;
- ios_completed(td, &icd);
- }
-
- /*
- * now cancel remaining active events
- */
- if (td->io_cancel) {
- list_for_each_safe(entry, n, &td->io_u_busylist) {
- io_u = list_entry(entry, struct io_u, list);
-
- r = td->io_cancel(td, io_u);
- if (!r)
- put_io_u(td, io_u);
- }
- }
-
- if (td->cur_depth) {
- r = io_u_getevents(td, td->cur_depth, td->cur_depth, NULL);
- if (r > 0) {
- icd.nr = r;
- ios_completed(td, &icd);
- }
- }
-}
-
-static int do_io_u_verify(struct thread_data *td, struct io_u **io_u)
-{
- struct io_u *v_io_u = *io_u;
- int ret = 0;
-
- if (v_io_u) {
- ret = verify_io_u(v_io_u);
- put_io_u(td, v_io_u);
- *io_u = NULL;
- }
-
- return ret;
-}
-
-static void do_verify(struct thread_data *td)
-{
- struct timeval t;
- struct io_u *io_u, *v_io_u = NULL;
- struct io_completion_data icd;
- int ret;
-
- td_set_runstate(td, TD_VERIFYING);
-
- do {
- if (td->terminate)
- break;
-
- gettimeofday(&t, NULL);
- if (runtime_exceeded(td, &t))
- break;
-
- io_u = __get_io_u(td);
- if (!io_u)
- break;
-
- if (get_next_verify(td, io_u)) {
- put_io_u(td, io_u);
- break;
- }