rand: add AES random buffer generator

This will be used by default for just buffer data generation, as I
suspect the overhead of doing it for smaller bits of data would mean
it's not really a win.

Pretty sure there are massive amounts of improvements available here,
this is just meant to serve as a proof-of-concept on the simplest
possible implementation. The grunt of this work is really doing the
detection properly and ensuring it's abstracted out nicely.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/Makefile b/Makefile
index 634d2c931345658a2cfcc6c5872f1ec0ce24fe2d..1f3cbb537cc665eb1baad888eceaa5620458d4a0 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ ifndef CONFIG_FIO_NO_OPT
   FIO_CFLAGS += -O3
 endif
 ifdef CONFIG_BUILD_NATIVE
-  FIO_CFLAGS += -march=native
+  FIO_CFLAGS += -march=native -maes
 endif
 
 ifdef CONFIG_PDB

diff --git a/arch/arch-x86-common.h b/arch/arch-x86-common.h
index f32835cce33bc4a74adeb5b9ad59f49515bceeeb..6a3567307c334a7c83f657b837b999f4a29b3bf1 100644 (file)
--- a/arch/arch-x86-common.h
+++ b/arch/arch-x86-common.h
@@ -15,7 +15,7 @@ static inline void cpuid(unsigned int op,
 #define ARCH_HAVE_INIT
 
 extern bool tsc_reliable;
-extern int arch_random;
+extern int arch_random, arch_aes;
 
 static inline void arch_init_intel(void)
 {
@@ -37,11 +37,12 @@ static inline void arch_init_intel(void)
        tsc_reliable = (edx & (1U << 8)) != 0;
 
        /*
-        * Check for FDRAND
+        * Check for FDRAND / AES
         */
        eax = 0x1;
        do_cpuid(&eax, &ebx, &ecx, &edx);
        arch_random = (ecx & (1U << 30)) != 0;
+       arch_aes = (ecx & (1U << 25)) != 0;
 }
 
 static inline void arch_init_amd(void)
@@ -54,6 +55,13 @@ static inline void arch_init_amd(void)
 
        cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
        tsc_reliable = (edx & (1U << 8)) != 0;
+
+       /*
+        * Check for AES
+        */
+       eax = 0x1;
+       do_cpuid(&eax, &ebx, &ecx, &edx);
+       arch_aes = (ecx & (1U << 25)) != 0;
 }
 
 static inline void arch_init(char *envp[])

diff --git a/configure b/configure
index a2b9bd4cc8f4bba04de60a207f4b8913614f0b2c..7c1f38fcf466c854e50e94897b27e6a2cbc7afb6 100755 (executable)
--- a/configure
+++ b/configure
@@ -2908,6 +2908,28 @@ EOF
 fi
 print_config "timerfd_create" "$timerfd_create"
 
+##########################################
+# check for AES
+arch_aes="no"
+if test "$disable_native" != "yes" ; then
+cat > $TMPC << EOF
+#include <stdint.h>
+#include <wmmintrin.h>
+
+int main(int argc, char **argv)
+{
+  __m128i tmp, foo;
+  _mm_setr_epi32(0, 2, 4, 8);
+  tmp = _mm_aesenc_si128(tmp, foo);
+  return 0;
+}
+EOF
+  if compile_prog "-march=native -maes" "" "arch_aes"; then
+    arch_aes="yes"
+  fi
+fi
+print_config "arch_aes" "$arch_aes"
+
 #############################################################################
 
 if test "$wordsize" = "64" ; then
@@ -3262,6 +3284,11 @@ if test "$asan" = "yes"; then
   CFLAGS="$CFLAGS -fsanitize=address"
   LDFLAGS="$LDFLAGS -fsanitize=address"
 fi
+if test "$arch_aes" = "yes"; then
+  CFLAGS="$CFLAGS -maes"
+  output_sym "CONFIG_ARCH_AES"
+fi
+
 print_config "Lib-based ioengines dynamic" "$dynamic_engines"
 cat > $TMPC << EOF
 int main(int argc, char **argv)

diff --git a/init.c b/init.c
index f6d724a349d59823aaed4b5002b63a74255ff481..0cebe37fa75696ea3e300550aa7749f76cd9520a 100644 (file)
--- a/init.c
+++ b/init.c
@@ -1111,7 +1111,7 @@ void td_fill_rand_seeds(struct thread_data *td)
        td_fill_rand_seeds_internal(td, rand_type);
 
        init_rand_seed(&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF],
-                       rand_type);
+                       arch_aes ? FIO_RAND_AES : rand_type);
        frand_copy(&td->buf_state_prev, &td->buf_state);
 }
 

diff --git a/io_u.c b/io_u.c
index eec378ddc0258169e3edbeb84046f7f27cef6282..45f7f955f740825568050f3d14614c69d181132a 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -2260,12 +2260,18 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned long long min_wr
                        left -= this_write;
                        save_buf_state(td, rs);
                } while (left);
-       } else if (o->buffer_pattern_bytes)
+       } else if (o->buffer_pattern_bytes) {
                fill_buffer_pattern(td, buf, max_bs);
-       else if (o->zero_buffers)
+       } else if (o->zero_buffers) {
                memset(buf, 0, max_bs);
-       else
-               fill_random_buf(get_buf_state(td), buf, max_bs);
+       } else {
+#ifdef CONFIG_ARCH_AES
+               if (arch_aes)
+                       fill_random_buf_aes(&td->buf_state, buf, max_bs);
+               else
+#endif
+                       fill_random_buf(get_buf_state(td), buf, max_bs);
+       }
 }
 
 /*

diff --git a/lib/rand.c b/lib/rand.c
index 2c593569aad5a2f472ca68642278598156806c1d..62aa978d0d812316351814700944171545345309 100644 (file)
--- a/lib/rand.c
+++ b/lib/rand.c
@@ -34,11 +34,14 @@
 */
 
 #include <string.h>
+#ifdef CONFIG_ARCH_AES
+#include <wmmintrin.h>
+#endif
 #include "rand.h"
 #include "pattern.h"
 #include "../hash.h"
 
-int arch_random;
+int arch_random, arch_aes;
 
 static inline uint64_t __seed(uint64_t x, uint64_t m)
 {
@@ -75,6 +78,14 @@ void __init_rand64(struct taus258_state *state, uint64_t seed)
                __rand64(state);
 }
 
+static void __init_rand_aes(struct frand_state *state, unsigned int seed)
+{
+#ifdef CONFIG_ARCH_AES
+       if (arch_aes)
+               aes_seed(state, seed);
+#endif
+}
+
 void init_rand(struct frand_state *state, enum fio_rand_type rand_type)
 {
        state->rand_type = rand_type;
@@ -86,6 +97,9 @@ void init_rand(struct frand_state *state, enum fio_rand_type rand_type)
        case FIO_RAND_64:
                __init_rand64(&state->state64, 1);
                break;
+       case FIO_RAND_AES:
+               __init_rand_aes(state, 1);
+               break;
        }
 }
 
@@ -98,6 +112,14 @@ void init_rand_seed(struct frand_state *state, uint64_t seed,
        case FIO_RAND_32:
                __init_rand32(&state->state32, (unsigned int) seed);
                break;
+       case FIO_RAND_AES:
+#ifdef CONFIG_ARCH_AES
+               if (arch_aes) {
+                       aes_seed(state, seed);
+                       break;
+               }
+#endif
+               fio_fallthrough;
        case FIO_RAND_64:
                __init_rand64(&state->state64, seed);
                break;
@@ -219,3 +241,42 @@ uint64_t fill_random_buf_percentage(struct frand_state *fs, void *buf,
                                        pattern, pbytes);
        return r;
 }
+
+#ifdef CONFIG_ARCH_AES
+void aes_seed(struct frand_state *fs, unsigned int seed)
+{
+       fs->aes_accum = _mm_setr_epi32(seed, seed * 2, seed * 3, seed * 4);
+       fs->aes_key = _mm_setr_epi32(seed * 5, seed * 6, seed * 7, seed * 8);
+}
+
+void fill_random_buf_aes(struct frand_state *fs, void *buf, unsigned int len)
+{
+       __m128i accum = fs->aes_accum;
+       __m128i key = fs->aes_key;
+       unsigned int rem = len & 15;
+       int i, loops;
+
+       loops = len / 16;
+       if (fio_unlikely(!len))
+               goto old_fill;
+
+       for (i = 0; i < loops; i++) {
+               __m128i tmp;
+
+               tmp = _mm_aesenc_si128(accum, key);
+               accum = tmp;
+               _mm_store_si128(buf, tmp);
+               buf += 16;
+       }
+
+       fs->aes_accum = accum;
+       fs->aes_key = key;
+
+       if (fio_unlikely(rem)) {
+               unsigned int r;
+old_fill:
+               r = *(unsigned long *) buf;
+               __fill_random_buf(buf, rem, r);
+       }
+}
+#endif

diff --git a/lib/rand.h b/lib/rand.h
index 77548f90ef28c95d94ce607b5bc287b2ba2ac495..4b9c8d84d98da4a60f9a97d8a93580d66dc6cb06 100644 (file)
--- a/lib/rand.h
+++ b/lib/rand.h
@@ -4,8 +4,13 @@
 #include <inttypes.h>
 #include <assert.h>
 #include <stdio.h>
+#ifdef CONFIG_ARCH_AES
+#include <xmmintrin.h>
+#endif
 #include "types.h"
 
+extern int arch_random, arch_aes;
+
 #define FRAND32_MAX    (-1U)
 #define FRAND32_MAX_PLUS_ONE   (1.0 * (1ULL << 32))
 #define FRAND64_MAX    (-1ULL)
@@ -14,6 +19,7 @@
 enum fio_rand_type {
        FIO_RAND_32,
        FIO_RAND_64,
+       FIO_RAND_AES,
 };
 
 struct taus88_state {
@@ -30,6 +36,10 @@ struct frand_state {
                struct taus88_state state32;
                struct taus258_state state64;
        };
+#ifdef CONFIG_ARCH_AES
+       __m128i aes_key;
+       __m128i aes_accum;
+#endif
 };
 
 static inline uint64_t rand_max(struct frand_state *state)
@@ -62,6 +72,15 @@ static inline void __frand64_copy(struct taus258_state *dst,
        dst->s5 = src->s5;
 }
 
+static inline void __frand_aes_copy(struct frand_state *dst,
+                                   struct frand_state *src)
+{
+#ifdef CONFIG_ARCH_AES
+       dst->aes_key = src->aes_key;
+       dst->aes_accum = src->aes_accum;
+#endif
+}
+
 static inline void frand_copy(struct frand_state *dst, struct frand_state *src)
 {
        switch (src->rand_type) {
@@ -71,6 +90,9 @@ static inline void frand_copy(struct frand_state *dst, struct frand_state *src)
        case FIO_RAND_32:
                __frand32_copy(&dst->state32, &src->state32);
                break;
+       case FIO_RAND_AES:
+               __frand_aes_copy(dst, src);
+               break;
        }
        dst->rand_type = src->rand_type;
 }
@@ -196,4 +218,9 @@ extern uint64_t fill_random_buf(struct frand_state *, void *buf, unsigned int le
 extern void __fill_random_buf_percentage(uint64_t, void *, unsigned int, unsigned int, unsigned int, char *, unsigned int);
 extern uint64_t fill_random_buf_percentage(struct frand_state *, void *, unsigned int, unsigned int, unsigned int, char *, unsigned int);
 
+#ifdef CONFIG_ARCH_AES
+void fill_random_buf_aes(struct frand_state *, void *, unsigned int);
+void aes_seed(struct frand_state *, unsigned int);
+#endif
+
 #endif

diff --git a/t/arch.c b/t/arch.c
index a72cef3a295f56d6bff6c5331a39ae4b3a2d25f1..2955b23abce2fb084c61004a4892854a5722bcaa 100644 (file)
--- a/t/arch.c
+++ b/t/arch.c
@@ -1,4 +1,4 @@
 #include "../arch/arch.h"
 
 unsigned long arch_flags = 0;
-int arch_random;
+int arch_random, arch_aes;