arch/x86/crypto/aesni-intel_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Support for Intel AES-NI instructions. This file contains glue
   4  * code, the real AES implementation is in intel-aes_asm.S.
   5  *
   6  * Copyright (C) 2008, Intel Corp.
   7  *    Author: Huang Ying <ying.huang@intel.com>
   8  *
   9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10  * interface for 64-bit kernels.
  11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13  *             Tadeusz Struk (tadeusz.struk@intel.com)
  14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15  *    Copyright (c) 2010, Intel Corporation.
  16  */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/types.h>
  20 #include <linux/module.h>
  21 #include <linux/err.h>
  22 #include <crypto/algapi.h>
  23 #include <crypto/aes.h>
  24 #include <crypto/ctr.h>
  25 #include <crypto/b128ops.h>
  26 #include <crypto/gcm.h>
  27 #include <crypto/xts.h>
  28 #include <asm/cpu_device_id.h>
  29 #include <asm/simd.h>
  30 #include <crypto/scatterwalk.h>
  31 #include <crypto/internal/aead.h>
  32 #include <crypto/internal/simd.h>
  33 #include <crypto/internal/skcipher.h>
  34 #include <linux/workqueue.h>
  35 #include <linux/spinlock.h>
  36 #ifdef CONFIG_X86_64
  37 #include <asm/crypto/glue_helper.h>
  38 #endif
  39
  40
  41 #define AESNI_ALIGN     16
  42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  43 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  44 #define RFC4106_HASH_SUBKEY_SIZE 16
  45 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  46 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  47 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  48
  49 /* This data is stored at the end of the crypto_tfm struct.
  50  * It's a type of per "session" data storage location.
  51  * This needs to be 16 byte aligned.
  52  */
  53 struct aesni_rfc4106_gcm_ctx {
  54         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  55         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  56         u8 nonce[4];
  57 };
  58
  59 struct generic_gcmaes_ctx {
  60         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  61         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  62 };
  63
  64 struct aesni_xts_ctx {
  65         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  66         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  67 };
  68
  69 #define GCM_BLOCK_LEN 16
  70
  71 struct gcm_context_data {
  72         /* init, update and finalize context data */
  73         u8 aad_hash[GCM_BLOCK_LEN];
  74         u64 aad_length;
  75         u64 in_length;
  76         u8 partial_block_enc_key[GCM_BLOCK_LEN];
  77         u8 orig_IV[GCM_BLOCK_LEN];
  78         u8 current_counter[GCM_BLOCK_LEN];
  79         u64 partial_block_len;
  80         u64 unused;
  81         u8 hash_keys[GCM_BLOCK_LEN * 16];
  82 };
  83
  84 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  85                              unsigned int key_len);
  86 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  87                           const u8 *in);
  88 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  89                           const u8 *in);
  90 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  91                               const u8 *in, unsigned int len);
  92 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  93                               const u8 *in, unsigned int len);
  94 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  95                               const u8 *in, unsigned int len, u8 *iv);
  96 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  97                               const u8 *in, unsigned int len, u8 *iv);
  98
  99 #define AVX_GEN2_OPTSIZE 640
 100 #define AVX_GEN4_OPTSIZE 4096
 101
 102 #ifdef CONFIG_X86_64
 103
 104 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
 105                               const u8 *in, unsigned int len, u8 *iv);
 106 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 107                               const u8 *in, unsigned int len, u8 *iv);
 108
 109 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
 110                                  const u8 *in, bool enc, u8 *iv);
 111
 112 /* asmlinkage void aesni_gcm_enc()
 113  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 114  * struct gcm_context_data.  May be uninitialized.
 115  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 116  * const u8 *in, Plaintext input
 117  * unsigned long plaintext_len, Length of data in bytes for encryption.
 118  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 119  *         16-byte aligned pointer.
 120  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 121  * const u8 *aad, Additional Authentication Data (AAD)
 122  * unsigned long aad_len, Length of AAD in bytes.
 123  * u8 *auth_tag, Authenticated Tag output.
 124  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 125  *          Valid values are 16 (most likely), 12 or 8.
 126  */
 127 asmlinkage void aesni_gcm_enc(void *ctx,
 128                         struct gcm_context_data *gdata, u8 *out,
 129                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 130                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 131                         u8 *auth_tag, unsigned long auth_tag_len);
 132
 133 /* asmlinkage void aesni_gcm_dec()
 134  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 135  * struct gcm_context_data.  May be uninitialized.
 136  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 137  * const u8 *in, Ciphertext input
 138  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 139  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 140  *         16-byte aligned pointer.
 141  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 142  * const u8 *aad, Additional Authentication Data (AAD)
 143  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 144  * to be 8 or 12 bytes
 145  * u8 *auth_tag, Authenticated Tag output.
 146  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 147  * Valid values are 16 (most likely), 12 or 8.
 148  */
 149 asmlinkage void aesni_gcm_dec(void *ctx,
 150                         struct gcm_context_data *gdata, u8 *out,
 151                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 152                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 153                         u8 *auth_tag, unsigned long auth_tag_len);
 154
 155 /* Scatter / Gather routines, with args similar to above */
 156 asmlinkage void aesni_gcm_init(void *ctx,
 157                                struct gcm_context_data *gdata,
 158                                u8 *iv,
 159                                u8 *hash_subkey, const u8 *aad,
 160                                unsigned long aad_len);
 161 asmlinkage void aesni_gcm_enc_update(void *ctx,
 162                                      struct gcm_context_data *gdata, u8 *out,
 163                                      const u8 *in, unsigned long plaintext_len);
 164 asmlinkage void aesni_gcm_dec_update(void *ctx,
 165                                      struct gcm_context_data *gdata, u8 *out,
 166                                      const u8 *in,
 167                                      unsigned long ciphertext_len);
 168 asmlinkage void aesni_gcm_finalize(void *ctx,
 169                                    struct gcm_context_data *gdata,
 170                                    u8 *auth_tag, unsigned long auth_tag_len);
 171
 172 static const struct aesni_gcm_tfm_s {
 173         void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
 174                      u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
 175         void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
 176                            const u8 *in, unsigned long plaintext_len);
 177         void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
 178                            const u8 *in, unsigned long ciphertext_len);
 179         void (*finalize)(void *ctx, struct gcm_context_data *gdata,
 180                          u8 *auth_tag, unsigned long auth_tag_len);
 181 } *aesni_gcm_tfm;
 182
 183 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
 184         .init = &aesni_gcm_init,
 185         .enc_update = &aesni_gcm_enc_update,
 186         .dec_update = &aesni_gcm_dec_update,
 187         .finalize = &aesni_gcm_finalize,
 188 };
 189
 190 #ifdef CONFIG_AS_AVX
 191 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 192                 void *keys, u8 *out, unsigned int num_bytes);
 193 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 194                 void *keys, u8 *out, unsigned int num_bytes);
 195 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 196                 void *keys, u8 *out, unsigned int num_bytes);
 197 /*
 198  * asmlinkage void aesni_gcm_init_avx_gen2()
 199  * gcm_data *my_ctx_data, context data
 200  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 201  */
 202 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
 203                                         struct gcm_context_data *gdata,
 204                                         u8 *iv,
 205                                         u8 *hash_subkey,
 206                                         const u8 *aad,
 207                                         unsigned long aad_len);
 208
 209 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
 210                                      struct gcm_context_data *gdata, u8 *out,
 211                                      const u8 *in, unsigned long plaintext_len);
 212 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
 213                                      struct gcm_context_data *gdata, u8 *out,
 214                                      const u8 *in,
 215                                      unsigned long ciphertext_len);
 216 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
 217                                    struct gcm_context_data *gdata,
 218                                    u8 *auth_tag, unsigned long auth_tag_len);
 219
 220 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
 221                                 struct gcm_context_data *gdata, u8 *out,
 222                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 223                         const u8 *aad, unsigned long aad_len,
 224                         u8 *auth_tag, unsigned long auth_tag_len);
 225
 226 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
 227                                 struct gcm_context_data *gdata, u8 *out,
 228                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 229                         const u8 *aad, unsigned long aad_len,
 230                         u8 *auth_tag, unsigned long auth_tag_len);
 231
 232 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
 233         .init = &aesni_gcm_init_avx_gen2,
 234         .enc_update = &aesni_gcm_enc_update_avx_gen2,
 235         .dec_update = &aesni_gcm_dec_update_avx_gen2,
 236         .finalize = &aesni_gcm_finalize_avx_gen2,
 237 };
 238
 239 #endif
 240
 241 #ifdef CONFIG_AS_AVX2
 242 /*
 243  * asmlinkage void aesni_gcm_init_avx_gen4()
 244  * gcm_data *my_ctx_data, context data
 245  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 246  */
 247 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
 248                                         struct gcm_context_data *gdata,
 249                                         u8 *iv,
 250                                         u8 *hash_subkey,
 251                                         const u8 *aad,
 252                                         unsigned long aad_len);
 253
 254 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
 255                                      struct gcm_context_data *gdata, u8 *out,
 256                                      const u8 *in, unsigned long plaintext_len);
 257 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
 258                                      struct gcm_context_data *gdata, u8 *out,
 259                                      const u8 *in,
 260                                      unsigned long ciphertext_len);
 261 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
 262                                    struct gcm_context_data *gdata,
 263                                    u8 *auth_tag, unsigned long auth_tag_len);
 264
 265 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
 266                                 struct gcm_context_data *gdata, u8 *out,
 267                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 268                         const u8 *aad, unsigned long aad_len,
 269                         u8 *auth_tag, unsigned long auth_tag_len);
 270
 271 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
 272                                 struct gcm_context_data *gdata, u8 *out,
 273                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 274                         const u8 *aad, unsigned long aad_len,
 275                         u8 *auth_tag, unsigned long auth_tag_len);
 276
 277 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
 278         .init = &aesni_gcm_init_avx_gen4,
 279         .enc_update = &aesni_gcm_enc_update_avx_gen4,
 280         .dec_update = &aesni_gcm_dec_update_avx_gen4,
 281         .finalize = &aesni_gcm_finalize_avx_gen4,
 282 };
 283
 284 #endif
 285
 286 static inline struct
 287 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 288 {
 289         unsigned long align = AESNI_ALIGN;
 290
 291         if (align <= crypto_tfm_ctx_alignment())
 292                 align = 1;
 293         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 294 }
 295
 296 static inline struct
 297 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 298 {
 299         unsigned long align = AESNI_ALIGN;
 300
 301         if (align <= crypto_tfm_ctx_alignment())
 302                 align = 1;
 303         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 304 }
 305 #endif
 306
 307 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 308 {
 309         unsigned long addr = (unsigned long)raw_ctx;
 310         unsigned long align = AESNI_ALIGN;
 311
 312         if (align <= crypto_tfm_ctx_alignment())
 313                 align = 1;
 314         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 315 }
 316
 317 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 318                               const u8 *in_key, unsigned int key_len)
 319 {
 320         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 321         u32 *flags = &tfm->crt_flags;
 322         int err;
 323
 324         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 325             key_len != AES_KEYSIZE_256) {
 326                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 327                 return -EINVAL;
 328         }
 329
 330         if (!crypto_simd_usable())
 331                 err = aes_expandkey(ctx, in_key, key_len);
 332         else {
 333                 kernel_fpu_begin();
 334                 err = aesni_set_key(ctx, in_key, key_len);
 335                 kernel_fpu_end();
 336         }
 337
 338         return err;
 339 }
 340
 341 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 342                        unsigned int key_len)
 343 {
 344         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 345 }
 346
 347 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 348 {
 349         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 350
 351         if (!crypto_simd_usable()) {
 352                 aes_encrypt(ctx, dst, src);
 353         } else {
 354                 kernel_fpu_begin();
 355                 aesni_enc(ctx, dst, src);
 356                 kernel_fpu_end();
 357         }
 358 }
 359
 360 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 361 {
 362         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 363
 364         if (!crypto_simd_usable()) {
 365                 aes_decrypt(ctx, dst, src);
 366         } else {
 367                 kernel_fpu_begin();
 368                 aesni_dec(ctx, dst, src);
 369                 kernel_fpu_end();
 370         }
 371 }
 372
 373 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 374                                  unsigned int len)
 375 {
 376         return aes_set_key_common(crypto_skcipher_tfm(tfm),
 377                                   crypto_skcipher_ctx(tfm), key, len);
 378 }
 379
 380 static int ecb_encrypt(struct skcipher_request *req)
 381 {
 382         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 383         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 384         struct skcipher_walk walk;
 385         unsigned int nbytes;
 386         int err;
 387
 388         err = skcipher_walk_virt(&walk, req, true);
 389
 390         kernel_fpu_begin();
 391         while ((nbytes = walk.nbytes)) {
 392                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 393                               nbytes & AES_BLOCK_MASK);
 394                 nbytes &= AES_BLOCK_SIZE - 1;
 395                 err = skcipher_walk_done(&walk, nbytes);
 396         }
 397         kernel_fpu_end();
 398
 399         return err;
 400 }
 401
 402 static int ecb_decrypt(struct skcipher_request *req)
 403 {
 404         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 405         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 406         struct skcipher_walk walk;
 407         unsigned int nbytes;
 408         int err;
 409
 410         err = skcipher_walk_virt(&walk, req, true);
 411
 412         kernel_fpu_begin();
 413         while ((nbytes = walk.nbytes)) {
 414                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 415                               nbytes & AES_BLOCK_MASK);
 416                 nbytes &= AES_BLOCK_SIZE - 1;
 417                 err = skcipher_walk_done(&walk, nbytes);
 418         }
 419         kernel_fpu_end();
 420
 421         return err;
 422 }
 423
 424 static int cbc_encrypt(struct skcipher_request *req)
 425 {
 426         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 427         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 428         struct skcipher_walk walk;
 429         unsigned int nbytes;
 430         int err;
 431
 432         err = skcipher_walk_virt(&walk, req, true);
 433
 434         kernel_fpu_begin();
 435         while ((nbytes = walk.nbytes)) {
 436                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 437                               nbytes & AES_BLOCK_MASK, walk.iv);
 438                 nbytes &= AES_BLOCK_SIZE - 1;
 439                 err = skcipher_walk_done(&walk, nbytes);
 440         }
 441         kernel_fpu_end();
 442
 443         return err;
 444 }
 445
 446 static int cbc_decrypt(struct skcipher_request *req)
 447 {
 448         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 449         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 450         struct skcipher_walk walk;
 451         unsigned int nbytes;
 452         int err;
 453
 454         err = skcipher_walk_virt(&walk, req, true);
 455
 456         kernel_fpu_begin();
 457         while ((nbytes = walk.nbytes)) {
 458                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 459                               nbytes & AES_BLOCK_MASK, walk.iv);
 460                 nbytes &= AES_BLOCK_SIZE - 1;
 461                 err = skcipher_walk_done(&walk, nbytes);
 462         }
 463         kernel_fpu_end();
 464
 465         return err;
 466 }
 467
 468 #ifdef CONFIG_X86_64
 469 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 470                             struct skcipher_walk *walk)
 471 {
 472         u8 *ctrblk = walk->iv;
 473         u8 keystream[AES_BLOCK_SIZE];
 474         u8 *src = walk->src.virt.addr;
 475         u8 *dst = walk->dst.virt.addr;
 476         unsigned int nbytes = walk->nbytes;
 477
 478         aesni_enc(ctx, keystream, ctrblk);
 479         crypto_xor_cpy(dst, keystream, src, nbytes);
 480
 481         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 482 }
 483
 484 #ifdef CONFIG_AS_AVX
 485 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 486                               const u8 *in, unsigned int len, u8 *iv)
 487 {
 488         /*
 489          * based on key length, override with the by8 version
 490          * of ctr mode encryption/decryption for improved performance
 491          * aes_set_key_common() ensures that key length is one of
 492          * {128,192,256}
 493          */
 494         if (ctx->key_length == AES_KEYSIZE_128)
 495                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 496         else if (ctx->key_length == AES_KEYSIZE_192)
 497                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 498         else
 499                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 500 }
 501 #endif
 502
 503 static int ctr_crypt(struct skcipher_request *req)
 504 {
 505         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 506         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 507         struct skcipher_walk walk;
 508         unsigned int nbytes;
 509         int err;
 510
 511         err = skcipher_walk_virt(&walk, req, true);
 512
 513         kernel_fpu_begin();
 514         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 515                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 516                                       nbytes & AES_BLOCK_MASK, walk.iv);
 517                 nbytes &= AES_BLOCK_SIZE - 1;
 518                 err = skcipher_walk_done(&walk, nbytes);
 519         }
 520         if (walk.nbytes) {
 521                 ctr_crypt_final(ctx, &walk);
 522                 err = skcipher_walk_done(&walk, 0);
 523         }
 524         kernel_fpu_end();
 525
 526         return err;
 527 }
 528
 529 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 530                             unsigned int keylen)
 531 {
 532         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 533         int err;
 534
 535         err = xts_verify_key(tfm, key, keylen);
 536         if (err)
 537                 return err;
 538
 539         keylen /= 2;
 540
 541         /* first half of xts-key is for crypt */
 542         err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
 543                                  key, keylen);
 544         if (err)
 545                 return err;
 546
 547         /* second half of xts-key is for tweak */
 548         return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
 549                                   key + keylen, keylen);
 550 }
 551
 552
 553 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
 554 {
 555         aesni_enc(ctx, out, in);
 556 }
 557
 558 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 559 {
 560         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
 561 }
 562
 563 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 564 {
 565         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
 566 }
 567
 568 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 569 {
 570         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
 571 }
 572
 573 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 574 {
 575         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
 576 }
 577
 578 static const struct common_glue_ctx aesni_enc_xts = {
 579         .num_funcs = 2,
 580         .fpu_blocks_limit = 1,
 581
 582         .funcs = { {
 583                 .num_blocks = 8,
 584                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
 585         }, {
 586                 .num_blocks = 1,
 587                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
 588         } }
 589 };
 590
 591 static const struct common_glue_ctx aesni_dec_xts = {
 592         .num_funcs = 2,
 593         .fpu_blocks_limit = 1,
 594
 595         .funcs = { {
 596                 .num_blocks = 8,
 597                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
 598         }, {
 599                 .num_blocks = 1,
 600                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
 601         } }
 602 };
 603
 604 static int xts_encrypt(struct skcipher_request *req)
 605 {
 606         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 607         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 608
 609         return glue_xts_req_128bit(&aesni_enc_xts, req,
 610                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 611                                    aes_ctx(ctx->raw_tweak_ctx),
 612                                    aes_ctx(ctx->raw_crypt_ctx),
 613                                    false);
 614 }
 615
 616 static int xts_decrypt(struct skcipher_request *req)
 617 {
 618         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 619         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 620
 621         return glue_xts_req_128bit(&aesni_dec_xts, req,
 622                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 623                                    aes_ctx(ctx->raw_tweak_ctx),
 624                                    aes_ctx(ctx->raw_crypt_ctx),
 625                                    true);
 626 }
 627
 628 static int
 629 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 630 {
 631         struct crypto_aes_ctx ctx;
 632         int ret;
 633
 634         ret = aes_expandkey(&ctx, key, key_len);
 635         if (ret)
 636                 return ret;
 637
 638         /* Clear the data in the hash sub key container to zero.*/
 639         /* We want to cipher all zeros to create the hash sub key. */
 640         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 641
 642         aes_encrypt(&ctx, hash_subkey, hash_subkey);
 643
 644         memzero_explicit(&ctx, sizeof(ctx));
 645         return 0;
 646 }
 647
 648 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 649                                   unsigned int key_len)
 650 {
 651         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 652
 653         if (key_len < 4) {
 654                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 655                 return -EINVAL;
 656         }
 657         /*Account for 4 byte nonce at the end.*/
 658         key_len -= 4;
 659
 660         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 661
 662         return aes_set_key_common(crypto_aead_tfm(aead),
 663                                   &ctx->aes_key_expanded, key, key_len) ?:
 664                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 665 }
 666
 667 /* This is the Integrity Check Value (aka the authentication tag) length and can
 668  * be 8, 12 or 16 bytes long. */
 669 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 670                                        unsigned int authsize)
 671 {
 672         switch (authsize) {
 673         case 8:
 674         case 12:
 675         case 16:
 676                 break;
 677         default:
 678                 return -EINVAL;
 679         }
 680
 681         return 0;
 682 }
 683
 684 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 685                                        unsigned int authsize)
 686 {
 687         switch (authsize) {
 688         case 4:
 689         case 8:
 690         case 12:
 691         case 13:
 692         case 14:
 693         case 15:
 694         case 16:
 695                 break;
 696         default:
 697                 return -EINVAL;
 698         }
 699
 700         return 0;
 701 }
 702
 703 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
 704                               unsigned int assoclen, u8 *hash_subkey,
 705                               u8 *iv, void *aes_ctx)
 706 {
 707         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 708         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 709         const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
 710         struct gcm_context_data data AESNI_ALIGN_ATTR;
 711         struct scatter_walk dst_sg_walk = {};
 712         unsigned long left = req->cryptlen;
 713         unsigned long len, srclen, dstlen;
 714         struct scatter_walk assoc_sg_walk;
 715         struct scatter_walk src_sg_walk;
 716         struct scatterlist src_start[2];
 717         struct scatterlist dst_start[2];
 718         struct scatterlist *src_sg;
 719         struct scatterlist *dst_sg;
 720         u8 *src, *dst, *assoc;
 721         u8 *assocmem = NULL;
 722         u8 authTag[16];
 723
 724         if (!enc)
 725                 left -= auth_tag_len;
 726
 727 #ifdef CONFIG_AS_AVX2
 728         if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
 729                 gcm_tfm = &aesni_gcm_tfm_avx_gen2;
 730 #endif
 731 #ifdef CONFIG_AS_AVX
 732         if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
 733                 gcm_tfm = &aesni_gcm_tfm_sse;
 734 #endif
 735
 736         /* Linearize assoc, if not already linear */
 737         if (req->src->length >= assoclen && req->src->length &&
 738                 (!PageHighMem(sg_page(req->src)) ||
 739                         req->src->offset + req->src->length <= PAGE_SIZE)) {
 740                 scatterwalk_start(&assoc_sg_walk, req->src);
 741                 assoc = scatterwalk_map(&assoc_sg_walk);
 742         } else {
 743                 /* assoc can be any length, so must be on heap */
 744                 assocmem = kmalloc(assoclen, GFP_ATOMIC);
 745                 if (unlikely(!assocmem))
 746                         return -ENOMEM;
 747                 assoc = assocmem;
 748
 749                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 750         }
 751
 752         if (left) {
 753                 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
 754                 scatterwalk_start(&src_sg_walk, src_sg);
 755                 if (req->src != req->dst) {
 756                         dst_sg = scatterwalk_ffwd(dst_start, req->dst,
 757                                                   req->assoclen);
 758                         scatterwalk_start(&dst_sg_walk, dst_sg);
 759                 }
 760         }
 761
 762         kernel_fpu_begin();
 763         gcm_tfm->init(aes_ctx, &data, iv,
 764                 hash_subkey, assoc, assoclen);
 765         if (req->src != req->dst) {
 766                 while (left) {
 767                         src = scatterwalk_map(&src_sg_walk);
 768                         dst = scatterwalk_map(&dst_sg_walk);
 769                         srclen = scatterwalk_clamp(&src_sg_walk, left);
 770                         dstlen = scatterwalk_clamp(&dst_sg_walk, left);
 771                         len = min(srclen, dstlen);
 772                         if (len) {
 773                                 if (enc)
 774                                         gcm_tfm->enc_update(aes_ctx, &data,
 775                                                              dst, src, len);
 776                                 else
 777                                         gcm_tfm->dec_update(aes_ctx, &data,
 778                                                              dst, src, len);
 779                         }
 780                         left -= len;
 781
 782                         scatterwalk_unmap(src);
 783                         scatterwalk_unmap(dst);
 784                         scatterwalk_advance(&src_sg_walk, len);
 785                         scatterwalk_advance(&dst_sg_walk, len);
 786                         scatterwalk_done(&src_sg_walk, 0, left);
 787                         scatterwalk_done(&dst_sg_walk, 1, left);
 788                 }
 789         } else {
 790                 while (left) {
 791                         dst = src = scatterwalk_map(&src_sg_walk);
 792                         len = scatterwalk_clamp(&src_sg_walk, left);
 793                         if (len) {
 794                                 if (enc)
 795                                         gcm_tfm->enc_update(aes_ctx, &data,
 796                                                              src, src, len);
 797                                 else
 798                                         gcm_tfm->dec_update(aes_ctx, &data,
 799                                                              src, src, len);
 800                         }
 801                         left -= len;
 802                         scatterwalk_unmap(src);
 803                         scatterwalk_advance(&src_sg_walk, len);
 804                         scatterwalk_done(&src_sg_walk, 1, left);
 805                 }
 806         }
 807         gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len);
 808         kernel_fpu_end();
 809
 810         if (!assocmem)
 811                 scatterwalk_unmap(assoc);
 812         else
 813                 kfree(assocmem);
 814
 815         if (!enc) {
 816                 u8 authTagMsg[16];
 817
 818                 /* Copy out original authTag */
 819                 scatterwalk_map_and_copy(authTagMsg, req->src,
 820                                          req->assoclen + req->cryptlen -
 821                                          auth_tag_len,
 822                                          auth_tag_len, 0);
 823
 824                 /* Compare generated tag with passed in tag. */
 825                 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
 826                         -EBADMSG : 0;
 827         }
 828
 829         /* Copy in the authTag */
 830         scatterwalk_map_and_copy(authTag, req->dst,
 831                                  req->assoclen + req->cryptlen,
 832                                  auth_tag_len, 1);
 833
 834         return 0;
 835 }
 836
 837 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 838                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 839 {
 840         return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
 841                                 aes_ctx);
 842 }
 843
 844 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 845                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 846 {
 847         return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
 848                                 aes_ctx);
 849 }
 850
 851 static int helper_rfc4106_encrypt(struct aead_request *req)
 852 {
 853         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 854         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 855         void *aes_ctx = &(ctx->aes_key_expanded);
 856         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 857         unsigned int i;
 858         __be32 counter = cpu_to_be32(1);
 859
 860         /* Assuming we are supporting rfc4106 64-bit extended */
 861         /* sequence numbers We need to have the AAD length equal */
 862         /* to 16 or 20 bytes */
 863         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 864                 return -EINVAL;
 865
 866         /* IV below built */
 867         for (i = 0; i < 4; i++)
 868                 *(iv+i) = ctx->nonce[i];
 869         for (i = 0; i < 8; i++)
 870                 *(iv+4+i) = req->iv[i];
 871         *((__be32 *)(iv+12)) = counter;
 872
 873         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 874                               aes_ctx);
 875 }
 876
 877 static int helper_rfc4106_decrypt(struct aead_request *req)
 878 {
 879         __be32 counter = cpu_to_be32(1);
 880         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 881         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 882         void *aes_ctx = &(ctx->aes_key_expanded);
 883         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 884         unsigned int i;
 885
 886         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 887                 return -EINVAL;
 888
 889         /* Assuming we are supporting rfc4106 64-bit extended */
 890         /* sequence numbers We need to have the AAD length */
 891         /* equal to 16 or 20 bytes */
 892
 893         /* IV below built */
 894         for (i = 0; i < 4; i++)
 895                 *(iv+i) = ctx->nonce[i];
 896         for (i = 0; i < 8; i++)
 897                 *(iv+4+i) = req->iv[i];
 898         *((__be32 *)(iv+12)) = counter;
 899
 900         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 901                               aes_ctx);
 902 }
 903 #endif
 904
 905 static struct crypto_alg aesni_cipher_alg = {
 906         .cra_name               = "aes",
 907         .cra_driver_name        = "aes-aesni",
 908         .cra_priority           = 300,
 909         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 910         .cra_blocksize          = AES_BLOCK_SIZE,
 911         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 912         .cra_module             = THIS_MODULE,
 913         .cra_u  = {
 914                 .cipher = {
 915                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 916                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 917                         .cia_setkey             = aes_set_key,
 918                         .cia_encrypt            = aesni_encrypt,
 919                         .cia_decrypt            = aesni_decrypt
 920                 }
 921         }
 922 };
 923
 924 static struct skcipher_alg aesni_skciphers[] = {
 925         {
 926                 .base = {
 927                         .cra_name               = "__ecb(aes)",
 928                         .cra_driver_name        = "__ecb-aes-aesni",
 929                         .cra_priority           = 400,
 930                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 931                         .cra_blocksize          = AES_BLOCK_SIZE,
 932                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 933                         .cra_module             = THIS_MODULE,
 934                 },
 935                 .min_keysize    = AES_MIN_KEY_SIZE,
 936                 .max_keysize    = AES_MAX_KEY_SIZE,
 937                 .setkey         = aesni_skcipher_setkey,
 938                 .encrypt        = ecb_encrypt,
 939                 .decrypt        = ecb_decrypt,
 940         }, {
 941                 .base = {
 942                         .cra_name               = "__cbc(aes)",
 943                         .cra_driver_name        = "__cbc-aes-aesni",
 944                         .cra_priority           = 400,
 945                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 946                         .cra_blocksize          = AES_BLOCK_SIZE,
 947                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 948                         .cra_module             = THIS_MODULE,
 949                 },
 950                 .min_keysize    = AES_MIN_KEY_SIZE,
 951                 .max_keysize    = AES_MAX_KEY_SIZE,
 952                 .ivsize         = AES_BLOCK_SIZE,
 953                 .setkey         = aesni_skcipher_setkey,
 954                 .encrypt        = cbc_encrypt,
 955                 .decrypt        = cbc_decrypt,
 956 #ifdef CONFIG_X86_64
 957         }, {
 958                 .base = {
 959                         .cra_name               = "__ctr(aes)",
 960                         .cra_driver_name        = "__ctr-aes-aesni",
 961                         .cra_priority           = 400,
 962                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 963                         .cra_blocksize          = 1,
 964                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 965                         .cra_module             = THIS_MODULE,
 966                 },
 967                 .min_keysize    = AES_MIN_KEY_SIZE,
 968                 .max_keysize    = AES_MAX_KEY_SIZE,
 969                 .ivsize         = AES_BLOCK_SIZE,
 970                 .chunksize      = AES_BLOCK_SIZE,
 971                 .setkey         = aesni_skcipher_setkey,
 972                 .encrypt        = ctr_crypt,
 973                 .decrypt        = ctr_crypt,
 974         }, {
 975                 .base = {
 976                         .cra_name               = "__xts(aes)",
 977                         .cra_driver_name        = "__xts-aes-aesni",
 978                         .cra_priority           = 401,
 979                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 980                         .cra_blocksize          = AES_BLOCK_SIZE,
 981                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
 982                         .cra_module             = THIS_MODULE,
 983                 },
 984                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 985                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 986                 .ivsize         = AES_BLOCK_SIZE,
 987                 .setkey         = xts_aesni_setkey,
 988                 .encrypt        = xts_encrypt,
 989                 .decrypt        = xts_decrypt,
 990 #endif
 991         }
 992 };
 993
 994 static
 995 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
 996
 997 #ifdef CONFIG_X86_64
 998 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
 999                                   unsigned int key_len)
1000 {
1001         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1002
1003         return aes_set_key_common(crypto_aead_tfm(aead),
1004                                   &ctx->aes_key_expanded, key, key_len) ?:
1005                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1006 }
1007
1008 static int generic_gcmaes_encrypt(struct aead_request *req)
1009 {
1010         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1011         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1012         void *aes_ctx = &(ctx->aes_key_expanded);
1013         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1014         __be32 counter = cpu_to_be32(1);
1015
1016         memcpy(iv, req->iv, 12);
1017         *((__be32 *)(iv+12)) = counter;
1018
1019         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1020                               aes_ctx);
1021 }
1022
1023 static int generic_gcmaes_decrypt(struct aead_request *req)
1024 {
1025         __be32 counter = cpu_to_be32(1);
1026         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1027         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1028         void *aes_ctx = &(ctx->aes_key_expanded);
1029         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1030
1031         memcpy(iv, req->iv, 12);
1032         *((__be32 *)(iv+12)) = counter;
1033
1034         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1035                               aes_ctx);
1036 }
1037
1038 static struct aead_alg aesni_aeads[] = { {
1039         .setkey                 = common_rfc4106_set_key,
1040         .setauthsize            = common_rfc4106_set_authsize,
1041         .encrypt                = helper_rfc4106_encrypt,
1042         .decrypt                = helper_rfc4106_decrypt,
1043         .ivsize                 = GCM_RFC4106_IV_SIZE,
1044         .maxauthsize            = 16,
1045         .base = {
1046                 .cra_name               = "__rfc4106(gcm(aes))",
1047                 .cra_driver_name        = "__rfc4106-gcm-aesni",
1048                 .cra_priority           = 400,
1049                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1050                 .cra_blocksize          = 1,
1051                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1052                 .cra_alignmask          = AESNI_ALIGN - 1,
1053                 .cra_module             = THIS_MODULE,
1054         },
1055 }, {
1056         .setkey                 = generic_gcmaes_set_key,
1057         .setauthsize            = generic_gcmaes_set_authsize,
1058         .encrypt                = generic_gcmaes_encrypt,
1059         .decrypt                = generic_gcmaes_decrypt,
1060         .ivsize                 = GCM_AES_IV_SIZE,
1061         .maxauthsize            = 16,
1062         .base = {
1063                 .cra_name               = "__gcm(aes)",
1064                 .cra_driver_name        = "__generic-gcm-aesni",
1065                 .cra_priority           = 400,
1066                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1067                 .cra_blocksize          = 1,
1068                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1069                 .cra_alignmask          = AESNI_ALIGN - 1,
1070                 .cra_module             = THIS_MODULE,
1071         },
1072 } };
1073 #else
1074 static struct aead_alg aesni_aeads[0];
1075 #endif
1076
1077 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1078
1079 static const struct x86_cpu_id aesni_cpu_id[] = {
1080         X86_FEATURE_MATCH(X86_FEATURE_AES),
1081         {}
1082 };
1083 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1084
1085 static int __init aesni_init(void)
1086 {
1087         int err;
1088
1089         if (!x86_match_cpu(aesni_cpu_id))
1090                 return -ENODEV;
1091 #ifdef CONFIG_X86_64
1092 #ifdef CONFIG_AS_AVX2
1093         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1094                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1095                 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
1096         } else
1097 #endif
1098 #ifdef CONFIG_AS_AVX
1099         if (boot_cpu_has(X86_FEATURE_AVX)) {
1100                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1101                 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
1102         } else
1103 #endif
1104         {
1105                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1106                 aesni_gcm_tfm = &aesni_gcm_tfm_sse;
1107         }
1108         aesni_ctr_enc_tfm = aesni_ctr_enc;
1109 #ifdef CONFIG_AS_AVX
1110         if (boot_cpu_has(X86_FEATURE_AVX)) {
1111                 /* optimize performance of ctr mode encryption transform */
1112                 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1113                 pr_info("AES CTR mode by8 optimization enabled\n");
1114         }
1115 #endif
1116 #endif
1117
1118         err = crypto_register_alg(&aesni_cipher_alg);
1119         if (err)
1120                 return err;
1121
1122         err = simd_register_skciphers_compat(aesni_skciphers,
1123                                              ARRAY_SIZE(aesni_skciphers),
1124                                              aesni_simd_skciphers);
1125         if (err)
1126                 goto unregister_cipher;
1127
1128         err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1129                                          aesni_simd_aeads);
1130         if (err)
1131                 goto unregister_skciphers;
1132
1133         return 0;
1134
1135 unregister_skciphers:
1136         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1137                                   aesni_simd_skciphers);
1138 unregister_cipher:
1139         crypto_unregister_alg(&aesni_cipher_alg);
1140         return err;
1141 }
1142
1143 static void __exit aesni_exit(void)
1144 {
1145         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1146                               aesni_simd_aeads);
1147         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1148                                   aesni_simd_skciphers);
1149         crypto_unregister_alg(&aesni_cipher_alg);
1150 }
1151
1152 late_initcall(aesni_init);
1153 module_exit(aesni_exit);
1154
1155 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1156 MODULE_LICENSE("GPL");
1157 MODULE_ALIAS_CRYPTO("aes");