From 0d29de831183dfd049c97a03008d425ce21e2fa4 Mon Sep 17 00:00:00 2001
From: Jens Axboe <jaxboe@fusionio.com>
Date: Wed, 1 Sep 2010 13:54:15 +0200
Subject: [PATCH] Add verify trim support

Signed-off-by: Jens Axboe <jaxboe@fusionio.com>
---
 Makefile    |  4 +--
 blktrace.c  |  6 ++--
 fio.c       |  1 +
 fio.h       | 18 ++++++++++--
 init.c      |  1 +
 io_u.c      | 59 +++++++++++++++++++++++++++++--------
 ioengine.h  |  1 +
 ioengines.c |  3 +-
 iolog.h     | 14 +++++++++
 log.c       | 16 ++++++++--
 options.c   | 45 +++++++++++++++++++++++++++-
 stat.c      |  8 +++--
 trim.c      | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 trim.h      | 37 +++++++++++++++++++++++
 verify.c    | 47 ++++++++++++++++++++++++++++--
 15 files changed, 316 insertions(+), 28 deletions(-)
 create mode 100644 trim.c
 create mode 100644 trim.h

diff --git a/Makefile b/Makefile
index 288480f1..9fec137d 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ SCRIPTS = fio_generate_plots
 OBJS = gettime.o fio.o ioengines.o init.o stat.o log.o time.o filesetup.o \
 	eta.o verify.o memory.o io_u.o parse.o mutex.o options.o \
 	rbtree.o diskutil.o fifo.o blktrace.o smalloc.o filehash.o helpers.o \
-	cgroup.o profile.o debug.o
+	cgroup.o profile.o debug.o trim.o
 
 OBJS += lib/rand.o
 OBJS += lib/flist_sort.o
@@ -62,7 +62,7 @@ $(PROGS): depend
 all: depend $(PROGS) $(SCRIPTS)
 
 clean:
-	-rm -f .depend cscope.out $(OBJS) $(PROGS) core.* core
+	-rm -f .depend $(OBJS) $(PROGS) core.* core
 
 cscope:
 	@cscope -b -R
diff --git a/blktrace.c b/blktrace.c
index 9ce4ae29..297a8a94 100644
--- a/blktrace.c
+++ b/blktrace.c
@@ -168,6 +168,7 @@ static void trace_add_open_event(struct thread_data *td, int fileno)
 	struct io_piece *ipo;
 
 	ipo = calloc(1, sizeof(*ipo));
+	init_ipo(ipo);
 
 	ipo->ddir = DDIR_INVAL;
 	ipo->fileno = fileno;
@@ -215,8 +216,8 @@ static void store_ipo(struct thread_data *td, unsigned long long offset,
 {
 	struct io_piece *ipo = malloc(sizeof(*ipo));
 
-	memset(ipo, 0, sizeof(*ipo));
-	INIT_FLIST_HEAD(&ipo->list);
+	init_ipo(ipo);
+
 	/*
 	 * the 512 is wrong here, it should be the hardware sector size...
 	 */
@@ -256,6 +257,7 @@ static void handle_trace_discard(struct thread_data *td, struct blk_io_trace *t,
 {
 	struct io_piece *ipo = malloc(sizeof(*ipo));
 
+	init_ipo(ipo);
 	trace_add_file(td, t->device);
 
 	ios[DDIR_WRITE]++;
diff --git a/fio.c b/fio.c
index 1d20cf79..c8de2ee7 100644
--- a/fio.c
+++ b/fio.c
@@ -1022,6 +1022,7 @@ static void *thread_main(void *data)
 	INIT_FLIST_HEAD(&td->io_log_list);
 	INIT_FLIST_HEAD(&td->io_hist_list);
 	INIT_FLIST_HEAD(&td->verify_list);
+	INIT_FLIST_HEAD(&td->trim_list);
 	pthread_mutex_init(&td->io_u_lock, NULL);
 	td->io_hist_tree = RB_ROOT;
 
diff --git a/fio.h b/fio.h
index 729604d3..e8c025d6 100644
--- a/fio.h
+++ b/fio.h
@@ -120,8 +120,8 @@ struct thread_stat {
 	unsigned int io_u_complete[FIO_IO_U_MAP_NR];
 	unsigned int io_u_lat_u[FIO_IO_U_LAT_U_NR];
 	unsigned int io_u_lat_m[FIO_IO_U_LAT_M_NR];
-	unsigned long total_io_u[2];
-	unsigned long short_io_u[2];
+	unsigned long total_io_u[3];
+	unsigned long short_io_u[3];
 	unsigned long total_submit;
 	unsigned long total_complete;
 
@@ -255,6 +255,10 @@ struct thread_options {
 	unsigned int gtod_offload;
 	enum fio_cs clocksource;
 	unsigned int no_stall;
+	unsigned int trim_percentage;
+	unsigned int trim_batch;
+	unsigned int trim_zero;
+	unsigned long long trim_backlog;
 
 	char *read_iolog_file;
 	char *write_iolog_file;
@@ -347,12 +351,14 @@ struct thread_data {
 
 	char *sysfs_root;
 
-	unsigned long rand_seeds[6];
+	unsigned long rand_seeds[7];
 
 	os_random_state_t bsrange_state;
 	os_random_state_t verify_state;
+	os_random_state_t trim_state;
 
 	unsigned int verify_batch;
+	unsigned int trim_batch;
 
 	int shm_id;
 
@@ -436,6 +442,12 @@ struct thread_data {
 	 */
 	struct flist_head io_log_list;
 
+	/*
+	 * For tracking/handling discards
+	 */
+	struct flist_head trim_list;
+	unsigned long trim_entries;
+
 	/*
 	 * for fileservice, how often to switch to a new file
 	 */
diff --git a/init.c b/init.c
index f0ee37a3..fe4dbf25 100644
--- a/init.c
+++ b/init.c
@@ -447,6 +447,7 @@ void td_fill_rand_seeds(struct thread_data *td)
 		os_random_seed(td->rand_seeds[3], &td->next_file_state);
 
 	os_random_seed(td->rand_seeds[5], &td->file_size_state);
+	os_random_seed(td->rand_seeds[6], &td->trim_state);
 
 	if (!td_random(td))
 		return;
diff --git a/io_u.c b/io_u.c
index 21a801f8..ea0d46c5 100644
--- a/io_u.c
+++ b/io_u.c
@@ -8,6 +8,7 @@
 #include "fio.h"
 #include "hash.h"
 #include "verify.h"
+#include "trim.h"
 #include "lib/rand.h"
 
 struct io_completion_data {
@@ -982,21 +983,31 @@ again:
 	return io_u;
 }
 
-/*
- * Return an io_u to be processed. Gets a buflen and offset, sets direction,
- * etc. The returned io_u is fully ready to be prepped and submitted.
- */
-struct io_u *get_io_u(struct thread_data *td)
+static int check_get_trim(struct thread_data *td, struct io_u *io_u)
 {
-	struct fio_file *f;
-	struct io_u *io_u;
+	if (td->o.trim_backlog && td->trim_entries) {
+		int get_trim = 0;
 
-	io_u = __get_io_u(td);
-	if (!io_u) {
-		dprint(FD_IO, "__get_io_u failed\n");
-		return NULL;
+		if (td->trim_batch) {
+			td->trim_batch--;
+			get_trim = 1;
+		} else if (!(td->io_hist_len % td->o.trim_backlog) &&
+			 td->last_ddir != DDIR_READ) {
+			td->trim_batch = td->o.trim_batch;
+			if (!td->trim_batch)
+				td->trim_batch = td->o.trim_backlog;
+			get_trim = 1;
+		}
+
+		if (get_trim && !get_next_trim(td, io_u))
+			return 1;
 	}
 
+	return 0;
+}
+
+static int check_get_verify(struct thread_data *td, struct io_u *io_u)
+{
 	if (td->o.verify_backlog && td->io_hist_len) {
 		int get_verify = 0;
 
@@ -1012,9 +1023,32 @@ struct io_u *get_io_u(struct thread_data *td)
 		}
 
 		if (get_verify && !get_next_verify(td, io_u))
-			goto out;
+			return 1;
 	}
 
+	return 0;
+}
+
+/*
+ * Return an io_u to be processed. Gets a buflen and offset, sets direction,
+ * etc. The returned io_u is fully ready to be prepped and submitted.
+ */
+struct io_u *get_io_u(struct thread_data *td)
+{
+	struct fio_file *f;
+	struct io_u *io_u;
+
+	io_u = __get_io_u(td);
+	if (!io_u) {
+		dprint(FD_IO, "__get_io_u failed\n");
+		return NULL;
+	}
+
+	if (check_get_verify(td, io_u))
+		goto out;
+	if (check_get_trim(td, io_u))
+		goto out;
+
 	/*
 	 * from a requeue, io_u already setup
 	 */
@@ -1064,6 +1098,7 @@ struct io_u *get_io_u(struct thread_data *td)
 	io_u->xfer_buflen = io_u->buflen;
 
 out:
+	assert(io_u->file);
 	if (!td_io_prep(td, io_u)) {
 		if (!td->o.disable_slat)
 			fio_gettime(&io_u->start_time, NULL);
diff --git a/ioengine.h b/ioengine.h
index 389e95a5..f6238f83 100644
--- a/ioengine.h
+++ b/ioengine.h
@@ -9,6 +9,7 @@ enum {
 	IO_U_F_FREE_DEF		= 1 << 2,
 	IO_U_F_IN_CUR_DEPTH	= 1 << 3,
 	IO_U_F_BUSY_OK		= 1 << 4,
+	IO_U_F_TRIMMED		= 1 << 5,
 };
 
 /*
diff --git a/ioengines.c b/ioengines.c
index f976efbf..7df0abac 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -269,7 +269,8 @@ int td_io_queue(struct thread_data *td, struct io_u *io_u)
 		if (ddir_rw(io_u->ddir)) {
 			io_u_mark_depth(td, 1);
 			td->ts.total_io_u[io_u->ddir]++;
-		}
+		} else if (io_u->ddir == DDIR_TRIM)
+			td->ts.total_io_u[2]++;
 	} else if (ret == FIO_Q_QUEUED) {
 		int r;
 
diff --git a/iolog.h b/iolog.h
index 2a97e285..c59e6aa0 100644
--- a/iolog.h
+++ b/iolog.h
@@ -32,6 +32,12 @@ struct io_log {
 	struct io_sample *log;
 };
 
+enum {
+	IP_F_ONRB	= 1,
+	IP_F_ONLIST	= 2,
+	IP_F_TRIMMED	= 4,
+};
+
 /*
  * When logging io actions, this matches a single sent io_u
  */
@@ -40,12 +46,14 @@ struct io_piece {
 		struct rb_node rb_node;
 		struct flist_head list;
 	};
+	struct flist_head trim_list;
 	union {
 		int fileno;
 		struct fio_file *file;
 	};
 	unsigned long long offset;
 	unsigned long len;
+	unsigned long flags;
 	enum fio_ddir ddir;
 	union {
 		unsigned long delay;
@@ -95,4 +103,10 @@ extern struct io_log *agg_io_log[2];
 extern int write_bw_log;
 extern void add_agg_sample(unsigned long, enum fio_ddir, unsigned int);
 
+static inline void init_ipo(struct io_piece *ipo)
+{
+	memset(ipo, 0, sizeof(*ipo));
+	INIT_FLIST_HEAD(&ipo->trim_list);
+}
+
 #endif
diff --git a/log.c b/log.c
index ce4ac9f4..266dc06b 100644
--- a/log.c
+++ b/log.c
@@ -9,6 +9,7 @@
 #include "flist.h"
 #include "fio.h"
 #include "verify.h"
+#include "trim.h"
 
 static const char iolog_ver2[] = "fio version 2 iolog";
 
@@ -115,6 +116,7 @@ int read_iolog_get(struct thread_data *td, struct io_u *io_u)
 
 		ipo = flist_entry(td->io_log_list.next, struct io_piece, list);
 		flist_del(&ipo->list);
+		remove_trim_entry(td, ipo);
 
 		ret = ipo_special(td, ipo);
 		if (ret < 0) {
@@ -160,6 +162,7 @@ void prune_io_piece_log(struct thread_data *td)
 	while ((n = rb_first(&td->io_hist_tree)) != NULL) {
 		ipo = rb_entry(n, struct io_piece, rb_node);
 		rb_erase(n, &td->io_hist_tree);
+		remove_trim_entry(td, ipo);
 		td->io_hist_len--;
 		free(ipo);
 	}
@@ -167,6 +170,7 @@ void prune_io_piece_log(struct thread_data *td)
 	while (!flist_empty(&td->io_hist_list)) {
 		ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
 		flist_del(&ipo->list);
+		remove_trim_entry(td, ipo);
 		td->io_hist_len--;
 		free(ipo);
 	}
@@ -181,10 +185,16 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u)
 	struct io_piece *ipo, *__ipo;
 
 	ipo = malloc(sizeof(struct io_piece));
+	init_ipo(ipo);
 	ipo->file = io_u->file;
 	ipo->offset = io_u->offset;
 	ipo->len = io_u->buflen;
 
+	if (io_u_should_trim(td, io_u)) {
+		flist_add_tail(&ipo->trim_list, &td->trim_list);
+		td->trim_entries++;
+	}
+
 	/*
 	 * We don't need to sort the entries, if:
 	 *
@@ -203,6 +213,7 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u)
 	      (file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
 		INIT_FLIST_HEAD(&ipo->list);
 		flist_add_tail(&ipo->list, &td->io_hist_list);
+		ipo->flags |= IP_F_ONLIST;
 		td->io_hist_len++;
 		return;
 	}
@@ -231,6 +242,7 @@ restart:
 			assert(ipo->len == __ipo->len);
 			td->io_hist_len--;
 			rb_erase(parent, &td->io_hist_tree);
+			remove_trim_entry(td, __ipo);
 			free(__ipo);
 			goto restart;
 		}
@@ -238,6 +250,7 @@ restart:
 
 	rb_link_node(&ipo->rb_node, parent, p);
 	rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
+	ipo->flags |= IP_F_ONRB;
 	td->io_hist_len++;
 }
 
@@ -345,8 +358,7 @@ static int read_iolog2(struct thread_data *td, FILE *f)
 		 * Make note of file
 		 */
 		ipo = malloc(sizeof(*ipo));
-		memset(ipo, 0, sizeof(*ipo));
-		INIT_FLIST_HEAD(&ipo->list);
+		init_ipo(ipo);
 		ipo->ddir = rw;
 		if (rw == DDIR_WAIT) {
 			ipo->delay = offset;
diff --git a/options.c b/options.c
index 3d32c8e0..e255e94f 100644
--- a/options.c
+++ b/options.c
@@ -440,6 +440,16 @@ static int str_verify_cpus_allowed_cb(void *data, const char *input)
 }
 #endif
 
+#ifdef FIO_HAVE_TRIM
+static int str_verify_trim_cb(void *data, unsigned long long *val)
+{
+	struct thread_data *td = data;
+
+	td->o.trim_percentage = *val;
+	return 0;
+}
+#endif
+
 static int str_fst_cb(void *data, const char *str)
 {
 	struct thread_data *td = data;
@@ -1458,7 +1468,7 @@ static struct fio_option options[FIO_MAX_OPTS] = {
 		.type	= FIO_OPT_INT,
 		.off1	= td_var_offset(verify_batch),
 		.help	= "Verify this number of IO blocks",
-		.parent	= "verify_backlog",
+		.parent	= "verify",
 	},
 #ifdef FIO_HAVE_CPU_AFFINITY
 	{
@@ -1468,6 +1478,39 @@ static struct fio_option options[FIO_MAX_OPTS] = {
 		.help	= "Set CPUs allowed for async verify threads",
 		.parent	= "verify_async",
 	},
+#endif
+#ifdef FIO_HAVE_TRIM
+	{
+		.name	= "trim_percentage",
+		.type	= FIO_OPT_INT,
+		.cb	= str_verify_trim_cb,
+		.maxval = 100,
+		.help	= "Number of verify blocks to discard/trim",
+		.parent	= "verify",
+		.def	= "0",
+	},
+	{
+		.name	= "trim_verify_zero",
+		.type	= FIO_OPT_INT,
+		.help	= "Verify that trim/discarded blocks are returned as zeroes",
+		.off1	= td_var_offset(trim_zero),
+		.parent	= "trim_percentage",
+		.def	= "1",
+	},
+	{
+		.name	= "trim_backlog",
+		.type	= FIO_OPT_STR_VAL,
+		.off1	= td_var_offset(trim_backlog),
+		.help	= "Trim after this number of blocks are written",
+		.parent	= "trim_percentage",
+	},
+	{
+		.name	= "trim_backlog_batch",
+		.type	= FIO_OPT_INT,
+		.off1	= td_var_offset(trim_batch),
+		.help	= "Trim this number of IO blocks",
+		.parent	= "trim_percentage",
+	},
 #endif
 	{
 		.name	= "write_iolog",
diff --git a/stat.c b/stat.c
index 8e9fba0e..326b1f76 100644
--- a/stat.c
+++ b/stat.c
@@ -351,9 +351,11 @@ static void show_thread_status(struct thread_stat *ts,
 					io_u_dist[1], io_u_dist[2],
 					io_u_dist[3], io_u_dist[4],
 					io_u_dist[5], io_u_dist[6]);
-	log_info("     issued r/w: total=%lu/%lu, short=%lu/%lu\n",
+	log_info("     issued r/w/d: total=%lu/%lu/%lu, short=%lu/%lu/%lu\n",
 					ts->total_io_u[0], ts->total_io_u[1],
-					ts->short_io_u[0], ts->short_io_u[1]);
+					ts->total_io_u[2],
+					ts->short_io_u[0], ts->short_io_u[1],
+					ts->short_io_u[2]);
 	stat_calc_lat_u(ts, io_u_lat_u);
 	stat_calc_lat_m(ts, io_u_lat_m);
 	show_latencies(io_u_lat_u, io_u_lat_m);
@@ -615,7 +617,7 @@ void show_run_stats(void)
 			ts->io_u_lat_m[k] += td->ts.io_u_lat_m[k];
 
 
-		for (k = 0; k <= DDIR_WRITE; k++) {
+		for (k = 0; k <= 2; k++) {
 			ts->total_io_u[k] += td->ts.total_io_u[k];
 			ts->short_io_u[k] += td->ts.short_io_u[k];
 		}
diff --git a/trim.c b/trim.c
new file mode 100644
index 00000000..cf42625b
--- /dev/null
+++ b/trim.c
@@ -0,0 +1,84 @@
+/*
+ * TRIM/DISCARD support
+ */
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <assert.h>
+#include <pthread.h>
+
+#include "fio.h"
+#include "trim.h"
+
+#ifdef FIO_HAVE_TRIM
+int get_next_trim(struct thread_data *td, struct io_u *io_u)
+{
+	struct io_piece *ipo;
+
+	/*
+	 * this io_u is from a requeue, we already filled the offsets
+	 */
+	if (io_u->file)
+		return 0;
+	if (flist_empty(&td->trim_list))
+		return 0;
+
+	assert(td->trim_entries);
+	ipo = flist_entry(td->trim_list.next, struct io_piece, trim_list);
+	remove_trim_entry(td, ipo);
+	ipo->flags |= IP_F_TRIMMED;
+
+	/*
+	 * If not verifying that trimmed ranges return zeroed data,
+	 * remove this from the to-read verify lists
+	 */
+	if (!td->o.trim_zero) {
+		if (ipo->flags & IP_F_ONLIST)
+			flist_del(&ipo->list);
+		else {
+			assert(ipo->flags & IP_F_ONRB);
+			rb_erase(&ipo->rb_node, &td->io_hist_tree);
+		}
+		td->io_hist_len--;
+	}
+
+	io_u->offset = ipo->offset;
+	io_u->buflen = ipo->len;
+	io_u->file = ipo->file;
+
+	if (!fio_file_open(io_u->file)) {
+		int r = td_io_open_file(td, io_u->file);
+
+		if (r) {
+			dprint(FD_VERIFY, "failed file %s open\n",
+					io_u->file->file_name);
+			return 1;
+		}
+	}
+
+	get_file(ipo->file);
+	assert(fio_file_open(io_u->file));
+	io_u->ddir = DDIR_TRIM;
+	io_u->xfer_buf = NULL;
+	io_u->xfer_buflen = io_u->buflen;
+
+	free(ipo);
+	dprint(FD_VERIFY, "get_next_trim: ret io_u %p\n", io_u);
+	return 0;
+}
+
+int io_u_should_trim(struct thread_data *td, struct io_u *io_u)
+{
+	unsigned long long val;
+	long r;
+
+	if (!td->o.trim_percentage)
+		return 0;
+
+	r = os_random_long(&td->trim_state);
+	val = (OS_RAND_MAX / 100ULL);
+	val *= (unsigned long long) td->o.trim_percentage;
+
+	return r <= val;
+}
+#endif
diff --git a/trim.h b/trim.h
new file mode 100644
index 00000000..d0d7a8dc
--- /dev/null
+++ b/trim.h
@@ -0,0 +1,37 @@
+#ifndef FIO_TRIM_H
+#define FIO_TRIM_H
+
+#include "fio.h"
+
+#ifdef FIO_HAVE_TRIM
+extern int __must_check get_next_trim(struct thread_data *td, struct io_u *io_u);
+extern int io_u_should_trim(struct thread_data *td, struct io_u *io_u);
+
+/*
+ * Determine whether a given io_u should be logged for verify or
+ * for discard
+ */
+static inline void remove_trim_entry(struct thread_data *td, struct io_piece *ipo)
+{
+	if (!flist_empty(&ipo->trim_list)) {
+		flist_del_init(&ipo->trim_list);
+		td->trim_entries--;
+	}
+}
+
+#else
+static inline int get_next_trim(struct thread_data *td, struct io_u *io_u)
+{
+	return 1;
+}
+static inline int io_u_should_trim(struct thread_data *td, struct io_u *io_u)
+{
+	return 0;
+}
+static inline void remove_trim_entry(struct thread_data *td, struct io_piece *ipo)
+{
+}
+#error foo
+#endif
+
+#endif
diff --git a/verify.c b/verify.c
index 7957bd41..073eec53 100644
--- a/verify.c
+++ b/verify.c
@@ -10,6 +10,7 @@
 #include "fio.h"
 #include "verify.h"
 #include "smalloc.h"
+#include "trim.h"
 #include "lib/rand.h"
 
 #include "crc/md5.h"
@@ -470,6 +471,38 @@ int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
 	return 0;
 }
 
+static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
+{
+	static char zero_buf[1024];
+	unsigned int this_len, len;
+	int ret = 0;
+	void *p;
+
+	if (!td->o.trim_zero)
+		return 0;
+
+	len = io_u->buflen;
+	p = io_u->buf;
+	do {
+		this_len = sizeof(zero_buf);
+		if (this_len > len)
+			this_len = len;
+		if (memcmp(p, zero_buf, this_len)) {
+			ret = EILSEQ;
+			break;
+		}
+		len -= this_len;
+		p += this_len;
+	} while (len);
+
+	if (!ret)
+		return 0;
+
+	log_err("trims: verify failed at file %s offset %llu, length %lu\n",
+			io_u->file->file_name, io_u->offset, io_u->buflen);
+	return ret;
+}
+
 int verify_io_u(struct thread_data *td, struct io_u *io_u)
 {
 	struct verify_header *hdr;
@@ -479,6 +512,10 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
 
 	if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
 		return 0;
+	if (io_u->flags & IO_U_F_TRIMMED) {
+		ret = verify_trimmed_io_u(td, io_u);
+		goto done;
+	}
 
 	hdr_inc = io_u->buflen;
 	if (td->o.verify_interval)
@@ -570,6 +607,7 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
 		}
 	}
 
+done:
 	if (ret && td->o.verify_fatal)
 		td->terminate = 1;
 
@@ -778,18 +816,21 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
 
 		ipo = rb_entry(n, struct io_piece, rb_node);
 		rb_erase(n, &td->io_hist_tree);
-		td->io_hist_len--;
 	} else if (!flist_empty(&td->io_hist_list)) {
 		ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
-		td->io_hist_len--;
 		flist_del(&ipo->list);
 	}
 
 	if (ipo) {
+		td->io_hist_len--;
+
 		io_u->offset = ipo->offset;
 		io_u->buflen = ipo->len;
 		io_u->file = ipo->file;
 
+		if (ipo->flags & IP_F_TRIMMED)
+			io_u->flags |= IO_U_F_TRIMMED;
+
 		if (!fio_file_open(io_u->file)) {
 			int r = td_io_open_file(td, io_u->file);
 
@@ -805,6 +846,8 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
 		io_u->ddir = DDIR_READ;
 		io_u->xfer_buf = io_u->buf;
 		io_u->xfer_buflen = io_u->buflen;
+
+		remove_trim_entry(td, ipo);
 		free(ipo);
 		dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
 		return 0;
-- 
2.25.1