From: Jens Axboe <jaxboe@fusionio.com>
Date: Thu, 27 Jan 2011 21:25:29 +0000 (+0100)
Subject: Fix end-of-job slowdown for random IO with a random map
X-Git-Tag: fio-1.50.2~1
X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=0ce8b119b65849e537cab628a176a0ec4238aab0

Fix end-of-job slowdown for random IO with a random map

Signed-off-by: Jens Axboe <jaxboe@fusionio.com>
---

diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h
index 6dbeee06..f2dcf497 100644
--- a/arch/arch-x86_64.h
+++ b/arch/arch-x86_64.h
@@ -33,9 +33,9 @@
 #define read_barrier()	__asm__ __volatile__("lfence":::"memory")
 #define write_barrier()	__asm__ __volatile__("sfence":::"memory")
 
-static inline unsigned int arch_ffz(unsigned int bitmask)
+static inline unsigned long arch_ffz(unsigned long bitmask)
 {
-	__asm__("bsfl %1,%0" :"=r" (bitmask) :"r" (~bitmask));
+	__asm__("bsf %1,%0" :"=r" (bitmask) :"r" (~bitmask));
 	return bitmask;
 }
 
diff --git a/file.h b/file.h
index 7948a412..04c0d453 100644
--- a/file.h
+++ b/file.h
@@ -96,9 +96,10 @@ struct fio_file {
 	/*
 	 * block map for random io
 	 */
-	unsigned int *file_map;
+	unsigned long *file_map;
 	unsigned int num_maps;
 	unsigned int last_free_lookup;
+	unsigned int failed_rands;
 
 	int references;
 	enum fio_file_flags flags;
@@ -158,11 +159,12 @@ extern void free_release_files(struct thread_data *);
 static inline void fio_file_reset(struct fio_file *f)
 {
 	f->last_free_lookup = 0;
+	f->failed_rands = 0;
 	f->last_pos = f->file_offset;
 	f->last_start = -1ULL;
 	f->file_pos = -1ULL;
 	if (f->file_map)
-		memset(f->file_map, 0, f->num_maps * sizeof(int));
+		memset(f->file_map, 0, f->num_maps * sizeof(unsigned long));
 }
 
 #endif
diff --git a/filesetup.c b/filesetup.c
index d0505064..0454cc40 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -812,7 +812,7 @@ int init_random_map(struct thread_data *td)
 				(unsigned long long) td->o.rw_min_bs;
 		num_maps = (blocks + BLOCKS_PER_MAP - 1) /
 				(unsigned long long) BLOCKS_PER_MAP;
-		f->file_map = smalloc(num_maps * sizeof(int));
+		f->file_map = smalloc(num_maps * sizeof(unsigned long));
 		if (f->file_map) {
 			f->num_maps = num_maps;
 			continue;
diff --git a/fio.h b/fio.h
index d2cca118..08f17331 100644
--- a/fio.h
+++ b/fio.h
@@ -531,7 +531,7 @@ static inline void fio_ro_check(struct thread_data *td, struct io_u *io_u)
 	assert(!(io_u->ddir == DDIR_WRITE && !td_write(td)));
 }
 
-#define BLOCKS_PER_MAP		(8 * sizeof(int))
+#define BLOCKS_PER_MAP		(8 * sizeof(unsigned long))
 #define TO_MAP_BLOCK(f, b)	(b)
 #define RAND_MAP_IDX(f, b)	(TO_MAP_BLOCK(f, b) / BLOCKS_PER_MAP)
 #define RAND_MAP_BIT(f, b)	(TO_MAP_BLOCK(f, b) & (BLOCKS_PER_MAP - 1))
diff --git a/io_u.c b/io_u.c
index 1a45706f..5a3ca744 100644
--- a/io_u.c
+++ b/io_u.c
@@ -30,7 +30,7 @@ static int random_map_free(struct fio_file *f, const unsigned long long block)
 
 	dprint(FD_RANDOM, "free: b=%llu, idx=%u, bit=%u\n", block, idx, bit);
 
-	return (f->file_map[idx] & (1 << bit)) == 0;
+	return (f->file_map[idx] & (1UL << bit)) == 0;
 }
 
 /*
@@ -50,8 +50,8 @@ static void mark_random_map(struct thread_data *td, struct io_u *io_u)
 	busy_check = !(io_u->flags & IO_U_F_BUSY_OK);
 
 	while (nr_blocks) {
-		unsigned int this_blocks, mask;
 		unsigned int idx, bit;
+		unsigned long mask, this_blocks;
 
 		/*
 		 * If we have a mixed random workload, we may
@@ -75,9 +75,9 @@ static void mark_random_map(struct thread_data *td, struct io_u *io_u)
 
 		do {
 			if (this_blocks == BLOCKS_PER_MAP)
-				mask = -1U;
+				mask = -1UL;
 			else
-				mask = ((1U << this_blocks) - 1) << bit;
+				mask = ((1UL << this_blocks) - 1) << bit;
 	
 			if (!(f->file_map[idx] & mask))
 				break;
@@ -126,7 +126,7 @@ static unsigned long long last_block(struct thread_data *td, struct fio_file *f,
 static int get_next_free_block(struct thread_data *td, struct fio_file *f,
 			       enum fio_ddir ddir, unsigned long long *b)
 {
-	unsigned long long min_bs = td->o.rw_min_bs, lastb;
+	unsigned long long block, min_bs = td->o.rw_min_bs, lastb;
 	int i;
 
 	lastb = last_block(td, f, ddir);
@@ -134,18 +134,19 @@ static int get_next_free_block(struct thread_data *td, struct fio_file *f,
 		return 1;
 
 	i = f->last_free_lookup;
-	*b = (i * BLOCKS_PER_MAP);
-	while ((*b) * min_bs < f->real_file_size &&
-		(*b) * min_bs < f->io_size) {
-		if (f->file_map[i] != (unsigned int) -1) {
-			*b += ffz(f->file_map[i]);
-			if (*b > lastb)
+	block = i * BLOCKS_PER_MAP;
+	while (block * min_bs < f->real_file_size &&
+		block * min_bs < f->io_size) {
+		if (f->file_map[i] != -1UL) {
+			block += ffz(f->file_map[i]);
+			if (block > lastb)
 				break;
 			f->last_free_lookup = i;
+			*b = block;
 			return 0;
 		}
 
-		*b += BLOCKS_PER_MAP;
+		block += BLOCKS_PER_MAP;
 		i++;
 	}
 
@@ -163,6 +164,9 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
 	if (!lastb)
 		return 1;
 
+	if (f->failed_rands >= 200)
+		goto ffz;
+
 	do {
 		r = os_random_long(&td->random_state);
 		dprint(FD_RANDOM, "off rand %llu\n", r);
@@ -172,18 +176,21 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
 		 * if we are not maintaining a random map, we are done.
 		 */
 		if (!file_randommap(td, f))
-			return 0;
+			goto ret_good;
 
 		/*
 		 * calculate map offset and check if it's free
 		 */
 		if (random_map_free(f, *b))
-			return 0;
+			goto ret_good;
 
 		dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
 									*b);
 	} while (--loops);
 
+	if (!f->failed_rands++)
+		f->last_free_lookup = 0;
+
 	/*
 	 * we get here, if we didn't suceed in looking up a block. generate
 	 * a random start offset into the filemap, and find the first free
@@ -194,7 +201,7 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
 		f->last_free_lookup = (f->num_maps - 1) *
 					(r / (OS_RAND_MAX + 1.0));
 		if (!get_next_free_block(td, f, ddir, b))
-			return 0;
+			goto ret;
 
 		r = os_random_long(&td->random_state);
 	} while (--loops);
@@ -203,7 +210,15 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
 	 * that didn't work either, try exhaustive search from the start
 	 */
 	f->last_free_lookup = 0;
+ffz:
+	if (!get_next_free_block(td, f, ddir, b))
+		return 0;
+	f->last_free_lookup = 0;
 	return get_next_free_block(td, f, ddir, b);
+ret_good:
+	f->failed_rands = 0;
+ret:
+	return 0;
 }
 
 static int get_next_rand_block(struct thread_data *td, struct fio_file *f,
diff --git a/lib/ffz.h b/lib/ffz.h
index e17a2e9c..eef612d1 100644
--- a/lib/ffz.h
+++ b/lib/ffz.h
@@ -1,10 +1,16 @@
 #ifndef FIO_FFZ_H
 #define FIO_FFZ_H
 
-static inline int __ffs(int word)
+static inline int __ffs(unsigned long word)
 {
 	int r = 0;
 
+#if BITS_PER_LONG == 64
+	if ((word & 0xffffffff) == 0) {
+		r += 32;
+		word >>= 32;
+	}
+#endif
 	if (!(word & 0xffff)) {
 		word >>= 16;
 		r += 16;
@@ -29,7 +35,7 @@ static inline int __ffs(int word)
 	return r;
 }
 
-static inline int ffz(unsigned int bitmask)
+static inline int ffz(unsigned long bitmask)
 {
 	return __ffs(~bitmask);
 }