Add verify trim support

Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

diff --git a/Makefile b/Makefile
index 288480f1e42cd569aa3b184a948f32d93e4e4363..9fec137d91f41391d8f23fffa17fc25957a45f26 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ SCRIPTS = fio_generate_plots
 OBJS = gettime.o fio.o ioengines.o init.o stat.o log.o time.o filesetup.o \
        eta.o verify.o memory.o io_u.o parse.o mutex.o options.o \
        rbtree.o diskutil.o fifo.o blktrace.o smalloc.o filehash.o helpers.o \
-       cgroup.o profile.o debug.o
+       cgroup.o profile.o debug.o trim.o
 
 OBJS += lib/rand.o
 OBJS += lib/flist_sort.o
@@ -62,7 +62,7 @@ $(PROGS): depend
 all: depend $(PROGS) $(SCRIPTS)
 
 clean:
-       -rm -f .depend cscope.out $(OBJS) $(PROGS) core.* core
+       -rm -f .depend $(OBJS) $(PROGS) core.* core
 
 cscope:
        @cscope -b -R

diff --git a/blktrace.c b/blktrace.c
index 9ce4ae29c8cbe358c4ce0913e84d407721d13c17..297a8a947af254200aed536788b095811a0d34ac 100644 (file)
--- a/blktrace.c
+++ b/blktrace.c
@@ -168,6 +168,7 @@ static void trace_add_open_event(struct thread_data *td, int fileno)
        struct io_piece *ipo;
 
        ipo = calloc(1, sizeof(*ipo));
+       init_ipo(ipo);
 
        ipo->ddir = DDIR_INVAL;
        ipo->fileno = fileno;
@@ -215,8 +216,8 @@ static void store_ipo(struct thread_data *td, unsigned long long offset,
 {
        struct io_piece *ipo = malloc(sizeof(*ipo));
 
-       memset(ipo, 0, sizeof(*ipo));
-       INIT_FLIST_HEAD(&ipo->list);
+       init_ipo(ipo);
+
        /*
         * the 512 is wrong here, it should be the hardware sector size...
         */
@@ -256,6 +257,7 @@ static void handle_trace_discard(struct thread_data *td, struct blk_io_trace *t,
 {
        struct io_piece *ipo = malloc(sizeof(*ipo));
 
+       init_ipo(ipo);
        trace_add_file(td, t->device);
 
        ios[DDIR_WRITE]++;

diff --git a/fio.c b/fio.c
index 1d20cf79413a95ea184a223065635e923674b43b..c8de2ee7a7e3a3f94a0f1af8dc01080c01e22be1 100644 (file)
--- a/fio.c
+++ b/fio.c
@@ -1022,6 +1022,7 @@ static void *thread_main(void *data)
        INIT_FLIST_HEAD(&td->io_log_list);
        INIT_FLIST_HEAD(&td->io_hist_list);
        INIT_FLIST_HEAD(&td->verify_list);
+       INIT_FLIST_HEAD(&td->trim_list);
        pthread_mutex_init(&td->io_u_lock, NULL);
        td->io_hist_tree = RB_ROOT;
 

diff --git a/fio.h b/fio.h
index 729604d321bbf0911f6b40378d6ac3878dcb4f77..e8c025d6880e2d3613abf43578acf3efbbd8dab5 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -120,8 +120,8 @@ struct thread_stat {
        unsigned int io_u_complete[FIO_IO_U_MAP_NR];
        unsigned int io_u_lat_u[FIO_IO_U_LAT_U_NR];
        unsigned int io_u_lat_m[FIO_IO_U_LAT_M_NR];
-       unsigned long total_io_u[2];
-       unsigned long short_io_u[2];
+       unsigned long total_io_u[3];
+       unsigned long short_io_u[3];
        unsigned long total_submit;
        unsigned long total_complete;
 
@@ -255,6 +255,10 @@ struct thread_options {
        unsigned int gtod_offload;
        enum fio_cs clocksource;
        unsigned int no_stall;
+       unsigned int trim_percentage;
+       unsigned int trim_batch;
+       unsigned int trim_zero;
+       unsigned long long trim_backlog;
 
        char *read_iolog_file;
        char *write_iolog_file;
@@ -347,12 +351,14 @@ struct thread_data {
 
        char *sysfs_root;
 
-       unsigned long rand_seeds[6];
+       unsigned long rand_seeds[7];
 
        os_random_state_t bsrange_state;
        os_random_state_t verify_state;
+       os_random_state_t trim_state;
 
        unsigned int verify_batch;
+       unsigned int trim_batch;
 
        int shm_id;
 
@@ -436,6 +442,12 @@ struct thread_data {
         */
        struct flist_head io_log_list;
 
+       /*
+        * For tracking/handling discards
+        */
+       struct flist_head trim_list;
+       unsigned long trim_entries;
+
        /*
         * for fileservice, how often to switch to a new file
         */

diff --git a/init.c b/init.c
index f0ee37a348273ae56935214b4a2fae63783a4580..fe4dbf2551d8926f49190372d1d883716e7e0d14 100644 (file)
--- a/init.c
+++ b/init.c
@@ -447,6 +447,7 @@ void td_fill_rand_seeds(struct thread_data *td)
                os_random_seed(td->rand_seeds[3], &td->next_file_state);
 
        os_random_seed(td->rand_seeds[5], &td->file_size_state);
+       os_random_seed(td->rand_seeds[6], &td->trim_state);
 
        if (!td_random(td))
                return;

diff --git a/io_u.c b/io_u.c
index 21a801f82a0ee77497cbf66fb829cd75ff7fca18..ea0d46c5db3dd0d4e32da6b726b0abda9e249bd9 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -8,6 +8,7 @@
 #include "fio.h"
 #include "hash.h"
 #include "verify.h"
+#include "trim.h"
 #include "lib/rand.h"
 
 struct io_completion_data {
@@ -982,21 +983,31 @@ again:
        return io_u;
 }
 
-/*
- * Return an io_u to be processed. Gets a buflen and offset, sets direction,
- * etc. The returned io_u is fully ready to be prepped and submitted.
- */
-struct io_u *get_io_u(struct thread_data *td)
+static int check_get_trim(struct thread_data *td, struct io_u *io_u)
 {
-       struct fio_file *f;
-       struct io_u *io_u;
+       if (td->o.trim_backlog && td->trim_entries) {
+               int get_trim = 0;
 
-       io_u = __get_io_u(td);
-       if (!io_u) {
-               dprint(FD_IO, "__get_io_u failed\n");
-               return NULL;
+               if (td->trim_batch) {
+                       td->trim_batch--;
+                       get_trim = 1;
+               } else if (!(td->io_hist_len % td->o.trim_backlog) &&
+                        td->last_ddir != DDIR_READ) {
+                       td->trim_batch = td->o.trim_batch;
+                       if (!td->trim_batch)
+                               td->trim_batch = td->o.trim_backlog;
+                       get_trim = 1;
+               }
+
+               if (get_trim && !get_next_trim(td, io_u))
+                       return 1;
        }
 
+       return 0;
+}
+
+static int check_get_verify(struct thread_data *td, struct io_u *io_u)
+{
        if (td->o.verify_backlog && td->io_hist_len) {
                int get_verify = 0;
 
@@ -1012,9 +1023,32 @@ struct io_u *get_io_u(struct thread_data *td)
                }
 
                if (get_verify && !get_next_verify(td, io_u))
-                       goto out;
+                       return 1;
        }
 
+       return 0;
+}
+
+/*
+ * Return an io_u to be processed. Gets a buflen and offset, sets direction,
+ * etc. The returned io_u is fully ready to be prepped and submitted.
+ */
+struct io_u *get_io_u(struct thread_data *td)
+{
+       struct fio_file *f;
+       struct io_u *io_u;
+
+       io_u = __get_io_u(td);
+       if (!io_u) {
+               dprint(FD_IO, "__get_io_u failed\n");
+               return NULL;
+       }
+
+       if (check_get_verify(td, io_u))
+               goto out;
+       if (check_get_trim(td, io_u))
+               goto out;
+
        /*
         * from a requeue, io_u already setup
         */
@@ -1064,6 +1098,7 @@ struct io_u *get_io_u(struct thread_data *td)
        io_u->xfer_buflen = io_u->buflen;
 
 out:
+       assert(io_u->file);
        if (!td_io_prep(td, io_u)) {
                if (!td->o.disable_slat)
                        fio_gettime(&io_u->start_time, NULL);

diff --git a/ioengine.h b/ioengine.h
index 389e95a560c0006671a2c2ee21f680d18dbe2e6e..f6238f83a6a64139fcd941c086ab94dbf6a6e1b5 100644 (file)
--- a/ioengine.h
+++ b/ioengine.h
@@ -9,6 +9,7 @@ enum {
        IO_U_F_FREE_DEF         = 1 << 2,
        IO_U_F_IN_CUR_DEPTH     = 1 << 3,
        IO_U_F_BUSY_OK          = 1 << 4,
+       IO_U_F_TRIMMED          = 1 << 5,
 };
 
 /*

diff --git a/ioengines.c b/ioengines.c
index f976efbf693ec1b502041933442e70ab0b6d2852..7df0abac0bf9a427fbb8273ddf6f151797c8fead 100644 (file)
--- a/ioengines.c
+++ b/ioengines.c
@@ -269,7 +269,8 @@ int td_io_queue(struct thread_data *td, struct io_u *io_u)
                if (ddir_rw(io_u->ddir)) {
                        io_u_mark_depth(td, 1);
                        td->ts.total_io_u[io_u->ddir]++;
-               }
+               } else if (io_u->ddir == DDIR_TRIM)
+                       td->ts.total_io_u[2]++;
        } else if (ret == FIO_Q_QUEUED) {
                int r;
 

diff --git a/iolog.h b/iolog.h
index 2a97e2855c2e11455a508947f594d20316f184c0..c59e6aa0eea4018d1874bb78b53574e354799e2a 100644 (file)
--- a/iolog.h
+++ b/iolog.h
@@ -32,6 +32,12 @@ struct io_log {
        struct io_sample *log;
 };
 
+enum {
+       IP_F_ONRB       = 1,
+       IP_F_ONLIST     = 2,
+       IP_F_TRIMMED    = 4,
+};
+
 /*
  * When logging io actions, this matches a single sent io_u
  */
@@ -40,12 +46,14 @@ struct io_piece {
                struct rb_node rb_node;
                struct flist_head list;
        };
+       struct flist_head trim_list;
        union {
                int fileno;
                struct fio_file *file;
        };
        unsigned long long offset;
        unsigned long len;
+       unsigned long flags;
        enum fio_ddir ddir;
        union {
                unsigned long delay;
@@ -95,4 +103,10 @@ extern struct io_log *agg_io_log[2];
 extern int write_bw_log;
 extern void add_agg_sample(unsigned long, enum fio_ddir, unsigned int);
 
+static inline void init_ipo(struct io_piece *ipo)
+{
+       memset(ipo, 0, sizeof(*ipo));
+       INIT_FLIST_HEAD(&ipo->trim_list);
+}
+
 #endif

diff --git a/log.c b/log.c
index ce4ac9f4126be3de8b93cd6fd843c633368ad3aa..266dc06b3b2df74d4d765835583bef44ee20d753 100644 (file)
--- a/log.c
+++ b/log.c
@@ -9,6 +9,7 @@
 #include "flist.h"
 #include "fio.h"
 #include "verify.h"
+#include "trim.h"
 
 static const char iolog_ver2[] = "fio version 2 iolog";
 
@@ -115,6 +116,7 @@ int read_iolog_get(struct thread_data *td, struct io_u *io_u)
 
                ipo = flist_entry(td->io_log_list.next, struct io_piece, list);
                flist_del(&ipo->list);
+               remove_trim_entry(td, ipo);
 
                ret = ipo_special(td, ipo);
                if (ret < 0) {
@@ -160,6 +162,7 @@ void prune_io_piece_log(struct thread_data *td)
        while ((n = rb_first(&td->io_hist_tree)) != NULL) {
                ipo = rb_entry(n, struct io_piece, rb_node);
                rb_erase(n, &td->io_hist_tree);
+               remove_trim_entry(td, ipo);
                td->io_hist_len--;
                free(ipo);
        }
@@ -167,6 +170,7 @@ void prune_io_piece_log(struct thread_data *td)
        while (!flist_empty(&td->io_hist_list)) {
                ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
                flist_del(&ipo->list);
+               remove_trim_entry(td, ipo);
                td->io_hist_len--;
                free(ipo);
        }
@@ -181,10 +185,16 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u)
        struct io_piece *ipo, *__ipo;
 
        ipo = malloc(sizeof(struct io_piece));
+       init_ipo(ipo);
        ipo->file = io_u->file;
        ipo->offset = io_u->offset;
        ipo->len = io_u->buflen;
 
+       if (io_u_should_trim(td, io_u)) {
+               flist_add_tail(&ipo->trim_list, &td->trim_list);
+               td->trim_entries++;
+       }
+
        /*
         * We don't need to sort the entries, if:
         *
@@ -203,6 +213,7 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u)
              (file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
                INIT_FLIST_HEAD(&ipo->list);
                flist_add_tail(&ipo->list, &td->io_hist_list);
+               ipo->flags |= IP_F_ONLIST;
                td->io_hist_len++;
                return;
        }
@@ -231,6 +242,7 @@ restart:
                        assert(ipo->len == __ipo->len);
                        td->io_hist_len--;
                        rb_erase(parent, &td->io_hist_tree);
+                       remove_trim_entry(td, __ipo);
                        free(__ipo);
                        goto restart;
                }
@@ -238,6 +250,7 @@ restart:
 
        rb_link_node(&ipo->rb_node, parent, p);
        rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
+       ipo->flags |= IP_F_ONRB;
        td->io_hist_len++;
 }
 
@@ -345,8 +358,7 @@ static int read_iolog2(struct thread_data *td, FILE *f)
                 * Make note of file
                 */
                ipo = malloc(sizeof(*ipo));
-               memset(ipo, 0, sizeof(*ipo));
-               INIT_FLIST_HEAD(&ipo->list);
+               init_ipo(ipo);
                ipo->ddir = rw;
                if (rw == DDIR_WAIT) {
                        ipo->delay = offset;

diff --git a/options.c b/options.c
index 3d32c8e05fbd68504a80721b685283f13a47bd08..e255e94f78cd552f35f53e0de9080bd04f94cf27 100644 (file)
--- a/options.c
+++ b/options.c
@@ -440,6 +440,16 @@ static int str_verify_cpus_allowed_cb(void *data, const char *input)
 }
 #endif
 
+#ifdef FIO_HAVE_TRIM
+static int str_verify_trim_cb(void *data, unsigned long long *val)
+{
+       struct thread_data *td = data;
+
+       td->o.trim_percentage = *val;
+       return 0;
+}
+#endif
+
 static int str_fst_cb(void *data, const char *str)
 {
        struct thread_data *td = data;
@@ -1458,7 +1468,7 @@ static struct fio_option options[FIO_MAX_OPTS] = {
                .type   = FIO_OPT_INT,
                .off1   = td_var_offset(verify_batch),
                .help   = "Verify this number of IO blocks",
-               .parent = "verify_backlog",
+               .parent = "verify",
        },
 #ifdef FIO_HAVE_CPU_AFFINITY
        {
@@ -1468,6 +1478,39 @@ static struct fio_option options[FIO_MAX_OPTS] = {
                .help   = "Set CPUs allowed for async verify threads",
                .parent = "verify_async",
        },
+#endif
+#ifdef FIO_HAVE_TRIM
+       {
+               .name   = "trim_percentage",
+               .type   = FIO_OPT_INT,
+               .cb     = str_verify_trim_cb,
+               .maxval = 100,
+               .help   = "Number of verify blocks to discard/trim",
+               .parent = "verify",
+               .def    = "0",
+       },
+       {
+               .name   = "trim_verify_zero",
+               .type   = FIO_OPT_INT,
+               .help   = "Verify that trim/discarded blocks are returned as zeroes",
+               .off1   = td_var_offset(trim_zero),
+               .parent = "trim_percentage",
+               .def    = "1",
+       },
+       {
+               .name   = "trim_backlog",
+               .type   = FIO_OPT_STR_VAL,
+               .off1   = td_var_offset(trim_backlog),
+               .help   = "Trim after this number of blocks are written",
+               .parent = "trim_percentage",
+       },
+       {
+               .name   = "trim_backlog_batch",
+               .type   = FIO_OPT_INT,
+               .off1   = td_var_offset(trim_batch),
+               .help   = "Trim this number of IO blocks",
+               .parent = "trim_percentage",
+       },
 #endif
        {
                .name   = "write_iolog",

diff --git a/stat.c b/stat.c
index 8e9fba0e7e32839b5ad1785dd4834fb4a60a5dbb..326b1f763460827f05334fda03453ea39d398f27 100644 (file)
--- a/stat.c
+++ b/stat.c
@@ -351,9 +351,11 @@ static void show_thread_status(struct thread_stat *ts,
                                        io_u_dist[1], io_u_dist[2],
                                        io_u_dist[3], io_u_dist[4],
                                        io_u_dist[5], io_u_dist[6]);
-       log_info("     issued r/w: total=%lu/%lu, short=%lu/%lu\n",
+       log_info("     issued r/w/d: total=%lu/%lu/%lu, short=%lu/%lu/%lu\n",
                                        ts->total_io_u[0], ts->total_io_u[1],
-                                       ts->short_io_u[0], ts->short_io_u[1]);
+                                       ts->total_io_u[2],
+                                       ts->short_io_u[0], ts->short_io_u[1],
+                                       ts->short_io_u[2]);
        stat_calc_lat_u(ts, io_u_lat_u);
        stat_calc_lat_m(ts, io_u_lat_m);
        show_latencies(io_u_lat_u, io_u_lat_m);
@@ -615,7 +617,7 @@ void show_run_stats(void)
                        ts->io_u_lat_m[k] += td->ts.io_u_lat_m[k];
 
 
-               for (k = 0; k <= DDIR_WRITE; k++) {
+               for (k = 0; k <= 2; k++) {
                        ts->total_io_u[k] += td->ts.total_io_u[k];
                        ts->short_io_u[k] += td->ts.short_io_u[k];
                }

diff --git a/trim.c b/trim.c
new file mode 100644 (file)
index 0000000..cf42625
--- /dev/null
+++ b/trim.c
@@ -0,0 +1,84 @@
+/*
+ * TRIM/DISCARD support
+ */
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <assert.h>
+#include <pthread.h>
+
+#include "fio.h"
+#include "trim.h"
+
+#ifdef FIO_HAVE_TRIM
+int get_next_trim(struct thread_data *td, struct io_u *io_u)
+{
+       struct io_piece *ipo;
+
+       /*
+        * this io_u is from a requeue, we already filled the offsets
+        */
+       if (io_u->file)
+               return 0;
+       if (flist_empty(&td->trim_list))
+               return 0;
+
+       assert(td->trim_entries);
+       ipo = flist_entry(td->trim_list.next, struct io_piece, trim_list);
+       remove_trim_entry(td, ipo);
+       ipo->flags |= IP_F_TRIMMED;
+
+       /*
+        * If not verifying that trimmed ranges return zeroed data,
+        * remove this from the to-read verify lists
+        */
+       if (!td->o.trim_zero) {
+               if (ipo->flags & IP_F_ONLIST)
+                       flist_del(&ipo->list);
+               else {
+                       assert(ipo->flags & IP_F_ONRB);
+                       rb_erase(&ipo->rb_node, &td->io_hist_tree);
+               }
+               td->io_hist_len--;
+       }
+
+       io_u->offset = ipo->offset;
+       io_u->buflen = ipo->len;
+       io_u->file = ipo->file;
+
+       if (!fio_file_open(io_u->file)) {
+               int r = td_io_open_file(td, io_u->file);
+
+               if (r) {
+                       dprint(FD_VERIFY, "failed file %s open\n",
+                                       io_u->file->file_name);
+                       return 1;
+               }
+       }
+
+       get_file(ipo->file);
+       assert(fio_file_open(io_u->file));
+       io_u->ddir = DDIR_TRIM;
+       io_u->xfer_buf = NULL;
+       io_u->xfer_buflen = io_u->buflen;
+
+       free(ipo);
+       dprint(FD_VERIFY, "get_next_trim: ret io_u %p\n", io_u);
+       return 0;
+}
+
+int io_u_should_trim(struct thread_data *td, struct io_u *io_u)
+{
+       unsigned long long val;
+       long r;
+
+       if (!td->o.trim_percentage)
+               return 0;
+
+       r = os_random_long(&td->trim_state);
+       val = (OS_RAND_MAX / 100ULL);
+       val *= (unsigned long long) td->o.trim_percentage;
+
+       return r <= val;
+}
+#endif

diff --git a/trim.h b/trim.h
new file mode 100644 (file)
index 0000000..d0d7a8d
--- /dev/null
+++ b/trim.h
@@ -0,0 +1,37 @@
+#ifndef FIO_TRIM_H
+#define FIO_TRIM_H
+
+#include "fio.h"
+
+#ifdef FIO_HAVE_TRIM
+extern int __must_check get_next_trim(struct thread_data *td, struct io_u *io_u);
+extern int io_u_should_trim(struct thread_data *td, struct io_u *io_u);
+
+/*
+ * Determine whether a given io_u should be logged for verify or
+ * for discard
+ */
+static inline void remove_trim_entry(struct thread_data *td, struct io_piece *ipo)
+{
+       if (!flist_empty(&ipo->trim_list)) {
+               flist_del_init(&ipo->trim_list);
+               td->trim_entries--;
+       }
+}
+
+#else
+static inline int get_next_trim(struct thread_data *td, struct io_u *io_u)
+{
+       return 1;
+}
+static inline int io_u_should_trim(struct thread_data *td, struct io_u *io_u)
+{
+       return 0;
+}
+static inline void remove_trim_entry(struct thread_data *td, struct io_piece *ipo)
+{
+}
+#error foo
+#endif
+
+#endif

diff --git a/verify.c b/verify.c
index 7957bd4110d88bea8aecc242c6460f739871ef99..073eec534107c560cfc4963669f2caeb61c1d345 100644 (file)
--- a/verify.c
+++ b/verify.c
@@ -10,6 +10,7 @@
 #include "fio.h"
 #include "verify.h"
 #include "smalloc.h"
+#include "trim.h"
 #include "lib/rand.h"
 
 #include "crc/md5.h"
@@ -470,6 +471,38 @@ int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
        return 0;
 }
 
+static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
+{
+       static char zero_buf[1024];
+       unsigned int this_len, len;
+       int ret = 0;
+       void *p;
+
+       if (!td->o.trim_zero)
+               return 0;
+
+       len = io_u->buflen;
+       p = io_u->buf;
+       do {
+               this_len = sizeof(zero_buf);
+               if (this_len > len)
+                       this_len = len;
+               if (memcmp(p, zero_buf, this_len)) {
+                       ret = EILSEQ;
+                       break;
+               }
+               len -= this_len;
+               p += this_len;
+       } while (len);
+
+       if (!ret)
+               return 0;
+
+       log_err("trims: verify failed at file %s offset %llu, length %lu\n",
+                       io_u->file->file_name, io_u->offset, io_u->buflen);
+       return ret;
+}
+
 int verify_io_u(struct thread_data *td, struct io_u *io_u)
 {
        struct verify_header *hdr;
@@ -479,6 +512,10 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
 
        if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
                return 0;
+       if (io_u->flags & IO_U_F_TRIMMED) {
+               ret = verify_trimmed_io_u(td, io_u);
+               goto done;
+       }
 
        hdr_inc = io_u->buflen;
        if (td->o.verify_interval)
@@ -570,6 +607,7 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
                }
        }
 
+done:
        if (ret && td->o.verify_fatal)
                td->terminate = 1;
 
@@ -778,18 +816,21 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
 
                ipo = rb_entry(n, struct io_piece, rb_node);
                rb_erase(n, &td->io_hist_tree);
-               td->io_hist_len--;
        } else if (!flist_empty(&td->io_hist_list)) {
                ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
-               td->io_hist_len--;
                flist_del(&ipo->list);
        }
 
        if (ipo) {
+               td->io_hist_len--;
+
                io_u->offset = ipo->offset;
                io_u->buflen = ipo->len;
                io_u->file = ipo->file;
 
+               if (ipo->flags & IP_F_TRIMMED)
+                       io_u->flags |= IO_U_F_TRIMMED;
+
                if (!fio_file_open(io_u->file)) {
                        int r = td_io_open_file(td, io_u->file);
 
@@ -805,6 +846,8 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
                io_u->ddir = DDIR_READ;
                io_u->xfer_buf = io_u->buf;
                io_u->xfer_buflen = io_u->buflen;
+
+               remove_trim_entry(td, ipo);
                free(ipo);
                dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
                return 0;