crypto: arm/ghash - Use API partial block handling

Use the Crypto API partial block handling.

Also switch to the generic export format.

Finally remove a couple of stray may_use_simd() calls in gcm.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index aabfcf522a2cc18ace1a714ac6b826daede5a42a..a52dcc8c1e33483f301afcdc9b14f3d5d078d40a 100644 (file)
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -8,22 +8,22 @@
 
 #include <asm/hwcap.h>
 #include <asm/neon.h>
-#include <asm/simd.h>
-#include <linux/unaligned.h>
 #include <crypto/aes.h>
-#include <crypto/gcm.h>
 #include <crypto/b128ops.h>
-#include <crypto/cryptd.h>
+#include <crypto/gcm.h>
+#include <crypto/gf128mul.h>
+#include <crypto/ghash.h>
 #include <crypto/internal/aead.h>
 #include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
-#include <crypto/gf128mul.h>
 #include <crypto/scatterwalk.h>
 #include <linux/cpufeature.h>
-#include <linux/crypto.h>
+#include <linux/errno.h>
 #include <linux/jump_label.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
 
 MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions");
 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>");
@@ -32,9 +32,6 @@ MODULE_ALIAS_CRYPTO("ghash");
 MODULE_ALIAS_CRYPTO("gcm(aes)");
 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
 
-#define GHASH_BLOCK_SIZE       16
-#define GHASH_DIGEST_SIZE      16
-
 #define RFC4106_NONCE_SIZE     4
 
 struct ghash_key {
@@ -49,10 +46,8 @@ struct gcm_key {
        u8      nonce[];        // for RFC4106 nonce
 };
 
-struct ghash_desc_ctx {
+struct arm_ghash_desc_ctx {
        u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
-       u8 buf[GHASH_BLOCK_SIZE];
-       u32 count;
 };
 
 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
@@ -65,9 +60,9 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);
 
 static int ghash_init(struct shash_desc *desc)
 {
-       struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+       struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 
-       *ctx = (struct ghash_desc_ctx){};
+       *ctx = (struct arm_ghash_desc_ctx){};
        return 0;
 }
 
@@ -85,52 +80,49 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src,
 static int ghash_update(struct shash_desc *desc, const u8 *src,
                        unsigned int len)
 {
-       struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
-       unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
-
-       ctx->count += len;
+       struct ghash_key *key = crypto_shash_ctx(desc->tfm);
+       struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+       int blocks;
 
-       if ((partial + len) >= GHASH_BLOCK_SIZE) {
-               struct ghash_key *key = crypto_shash_ctx(desc->tfm);
-               int blocks;
+       blocks = len / GHASH_BLOCK_SIZE;
+       ghash_do_update(blocks, ctx->digest, src, key, NULL);
+       return len - blocks * GHASH_BLOCK_SIZE;
+}
 
-               if (partial) {
-                       int p = GHASH_BLOCK_SIZE - partial;
+static int ghash_export(struct shash_desc *desc, void *out)
+{
+       struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+       u8 *dst = out;
 
-                       memcpy(ctx->buf + partial, src, p);
-                       src += p;
-                       len -= p;
-               }
+       put_unaligned_be64(ctx->digest[1], dst);
+       put_unaligned_be64(ctx->digest[0], dst + 8);
+       return 0;
+}
 
-               blocks = len / GHASH_BLOCK_SIZE;
-               len %= GHASH_BLOCK_SIZE;
+static int ghash_import(struct shash_desc *desc, const void *in)
+{
+       struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+       const u8 *src = in;
 
-               ghash_do_update(blocks, ctx->digest, src, key,
-                               partial ? ctx->buf : NULL);
-               src += blocks * GHASH_BLOCK_SIZE;
-               partial = 0;
-       }
-       if (len)
-               memcpy(ctx->buf + partial, src, len);
+       ctx->digest[1] = get_unaligned_be64(src);
+       ctx->digest[0] = get_unaligned_be64(src + 8);
        return 0;
 }
 
-static int ghash_final(struct shash_desc *desc, u8 *dst)
+static int ghash_finup(struct shash_desc *desc, const u8 *src,
+                      unsigned int len, u8 *dst)
 {
-       struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
-       unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
+       struct ghash_key *key = crypto_shash_ctx(desc->tfm);
+       struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 
-       if (partial) {
-               struct ghash_key *key = crypto_shash_ctx(desc->tfm);
+       if (len) {
+               u8 buf[GHASH_BLOCK_SIZE] = {};
 
-               memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
-               ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
+               memcpy(buf, src, len);
+               ghash_do_update(1, ctx->digest, buf, key, NULL);
+               memzero_explicit(buf, sizeof(buf));
        }
-       put_unaligned_be64(ctx->digest[1], dst);
-       put_unaligned_be64(ctx->digest[0], dst + 8);
-
-       *ctx = (struct ghash_desc_ctx){};
-       return 0;
+       return ghash_export(desc, dst);
 }
 
 static void ghash_reflect(u64 h[], const be128 *k)
@@ -175,13 +167,17 @@ static struct shash_alg ghash_alg = {
        .digestsize             = GHASH_DIGEST_SIZE,
        .init                   = ghash_init,
        .update                 = ghash_update,
-       .final                  = ghash_final,
+       .finup                  = ghash_finup,
        .setkey                 = ghash_setkey,
-       .descsize               = sizeof(struct ghash_desc_ctx),
+       .export                 = ghash_export,
+       .import                 = ghash_import,
+       .descsize               = sizeof(struct arm_ghash_desc_ctx),
+       .statesize              = sizeof(struct ghash_desc_ctx),
 
        .base.cra_name          = "ghash",
        .base.cra_driver_name   = "ghash-ce",
        .base.cra_priority      = 300,
+       .base.cra_flags         = CRYPTO_AHASH_ALG_BLOCK_ONLY,
        .base.cra_blocksize     = GHASH_BLOCK_SIZE,
        .base.cra_ctxsize       = sizeof(struct ghash_key) + sizeof(u64[2]),
        .base.cra_module        = THIS_MODULE,
@@ -317,9 +313,6 @@ static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen)
        u8 *tag, *dst;
        int tail, err;
 
-       if (WARN_ON_ONCE(!may_use_simd()))
-               return -EBUSY;
-
        err = skcipher_walk_aead_encrypt(&walk, req, false);
 
        kernel_neon_begin();
@@ -409,9 +402,6 @@ static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen)
        u8 *tag, *dst;
        int tail, err, ret;
 
-       if (WARN_ON_ONCE(!may_use_simd()))
-               return -EBUSY;
-
        scatterwalk_map_and_copy(otag, req->src,
                                 req->assoclen + req->cryptlen - authsize,
                                 authsize, 0);