From 225ba9e3433cf27d8ff7b213d9f78b7ef2776c70 Mon Sep 17 00:00:00 2001
From: Jens Axboe <axboe@fb.com>
Date: Wed, 26 Feb 2014 14:31:15 -0800
Subject: [PATCH] Branch and cache miss speedups

Just some low hanging fruit.

Signed-off-by: Jens Axboe <axboe@fb.com>
---
 compiler/compiler.h |  2 ++
 filesetup.c         |  2 +-
 gettime.c           |  2 +-
 io_u.c              |  2 +-
 ioengine.h          | 38 +++++++++++++++++++-------------------
 lib/lfsr.c          |  9 ++++-----
 6 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/compiler/compiler.h b/compiler/compiler.h
index 036ba208..0a0213b6 100644
--- a/compiler/compiler.h
+++ b/compiler/compiler.h
@@ -20,4 +20,6 @@
 #define fio_init	__attribute__((constructor))
 #define fio_exit	__attribute__((destructor))
 
+#define fio_unlikely(x)	__builtin_expect(!!(x), 0)
+
 #endif
diff --git a/filesetup.c b/filesetup.c
index 1acf6adc..2744d4fc 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -1010,7 +1010,7 @@ int init_random_map(struct thread_data *td)
 
 			seed = td->rand_seeds[FIO_RAND_BLOCK_OFF];
 			
-			if (!lfsr_init(&f->lfsr, blocks, seed, seed & 0xF))
+			if (!lfsr_init(&f->lfsr, blocks, seed, 0))
 				continue;
 		} else if (!td->o.norandommap) {
 			f->io_axmap = axmap_new(blocks);
diff --git a/gettime.c b/gettime.c
index 277f2cf8..8991703f 100644
--- a/gettime.c
+++ b/gettime.c
@@ -205,7 +205,7 @@ void fio_gettime(struct timeval *tp, void fio_unused *caller)
 
 	gtod_log_caller(caller);
 #endif
-	if (fio_tv) {
+	if (fio_unlikely(fio_tv)) {
 		memcpy(tp, fio_tv, sizeof(*tp));
 		return;
 	}
diff --git a/io_u.c b/io_u.c
index a69efb76..77557dfd 100644
--- a/io_u.c
+++ b/io_u.c
@@ -1307,9 +1307,9 @@ again:
 	else if (!queue_full(td)) {
 		io_u = io_u_qpop(&td->io_u_freelist);
 
+		io_u->file = NULL;
 		io_u->buflen = 0;
 		io_u->resid = 0;
-		io_u->file = NULL;
 		io_u->end_io = NULL;
 	}
 
diff --git a/ioengine.h b/ioengine.h
index abf2b464..7e0707bc 100644
--- a/ioengine.h
+++ b/ioengine.h
@@ -73,6 +73,25 @@ struct io_u {
 
 	struct io_piece *ipo;
 
+	unsigned int resid;
+	unsigned int error;
+
+	/*
+	 * io engine private data
+	 */
+	union {
+		unsigned int index;
+		unsigned int seen;
+		void *engine_data;
+	};
+
+	struct flist_head verify_list;
+
+	/*
+	 * Callback for io completion
+	 */
+	int (*end_io)(struct thread_data *, struct io_u *);
+
 	union {
 #ifdef CONFIG_LIBAIO
 		struct iocb iocb;
@@ -97,25 +116,6 @@ struct io_u {
 #endif
 		void *mmap_data;
 	};
-
-	unsigned int resid;
-	unsigned int error;
-
-	/*
-	 * io engine private data
-	 */
-	union {
-		unsigned int index;
-		unsigned int seen;
-		void *engine_data;
-	};
-
-	struct flist_head verify_list;
-
-	/*
-	 * Callback for io completion
-	 */
-	int (*end_io)(struct thread_data *, struct io_u *);
 };
 
 /*
diff --git a/lib/lfsr.c b/lib/lfsr.c
index 927b2a10..9771318a 100644
--- a/lib/lfsr.c
+++ b/lib/lfsr.c
@@ -2,6 +2,7 @@
 #include <math.h>
 
 #include "lfsr.h"
+#include "../compiler/compiler.h"
 
 /*
  * LFSR taps retrieved from:
@@ -132,11 +133,9 @@ int lfsr_next(struct fio_lfsr *fl, uint64_t *off, uint64_t last)
 		if (fl->cycle_length && !--fl->cycle_length) {
 			__lfsr_next(fl, fl->spin + 1);
 			fl->cycle_length = fl->cached_cycle_length;
-			goto check;
-		}
-		__lfsr_next(fl, fl->spin);
-check: ;
-	} while (fl->last_val > fl->max_val);
+		} else
+			__lfsr_next(fl, fl->spin);
+	} while (fio_unlikely(fl->last_val > fl->max_val));
 
 	*off = fl->last_val;
 	return 0;
-- 
2.25.1