crypto: powerpc/poly1305 - Add block-only interface

Add block-only interface.

Also remove the unnecessary SIMD fallback path.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c
index 00617f4c58e6979575db0bc1d947b5d4745922ac..708435beaba69d508a6a72f4d79411bb9f51495b 100644 (file)
--- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c
+++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c
@@ -4,19 +4,20 @@
  *
  * Copyright 2023- IBM Corp. All rights reserved.
  */
+#include <asm/switch_to.h>
+#include <crypto/internal/poly1305.h>
+#include <linux/cpufeature.h>
+#include <linux/jump_label.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/jump_label.h>
-#include <crypto/internal/simd.h>
-#include <crypto/poly1305.h>
-#include <linux/cpufeature.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
-#include <asm/simd.h>
-#include <asm/switch_to.h>
 
-asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen);
-asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);
-asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);
+asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
+asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit);
+asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
+                                  u8 digest[POLY1305_DIGEST_SIZE],
+                                  const u32 nonce[4]);
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
 
@@ -32,22 +33,49 @@ static void vsx_end(void)
        preempt_enable();
 }
 
-void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
+void poly1305_block_init_arch(struct poly1305_block_state *dctx,
+                             const u8 raw_key[POLY1305_BLOCK_SIZE])
 {
        if (!static_key_enabled(&have_p10))
-               return poly1305_init_generic(dctx, key);
+               return poly1305_block_init_generic(dctx, raw_key);
 
        dctx->h = (struct poly1305_state){};
-       dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0);
-       dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8);
+       dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0);
+       dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8);
+}
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
+
+void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
+{
        dctx->s[0] = get_unaligned_le32(key + 16);
        dctx->s[1] = get_unaligned_le32(key + 20);
        dctx->s[2] = get_unaligned_le32(key + 24);
        dctx->s[3] = get_unaligned_le32(key + 28);
        dctx->buflen = 0;
+       poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);
 
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
+                         unsigned int len, u32 padbit)
+{
+       if (!static_key_enabled(&have_p10))
+               return poly1305_blocks_generic(state, src, len, padbit);
+       vsx_begin();
+       if (len >= POLY1305_BLOCK_SIZE * 4) {
+               poly1305_p10le_4blocks(state, src, len);
+               src += len - (len % (POLY1305_BLOCK_SIZE * 4));
+               len %= POLY1305_BLOCK_SIZE * 4;
+       }
+       while (len >= POLY1305_BLOCK_SIZE) {
+               poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit);
+               len -= POLY1305_BLOCK_SIZE;
+               src += POLY1305_BLOCK_SIZE;
+       }
+       vsx_end();
+}
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
+
 void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
                          const u8 *src, unsigned int srclen)
 {
@@ -64,28 +92,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
                dctx->buflen += bytes;
                if (dctx->buflen < POLY1305_BLOCK_SIZE)
                        return;
-               vsx_begin();
-               poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
-               vsx_end();
+               poly1305_blocks_arch(&dctx->state, dctx->buf,
+                                    POLY1305_BLOCK_SIZE, 1);
                dctx->buflen = 0;
        }
 
        if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
-               bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
-               if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {
-                       vsx_begin();
-                       poly1305_p10le_4blocks(&dctx->h, src, srclen);
-                       vsx_end();
-                       src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4));
-                       srclen %= POLY1305_BLOCK_SIZE * 4;
-               }
-               while (srclen >= POLY1305_BLOCK_SIZE) {
-                       vsx_begin();
-                       poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1);
-                       vsx_end();
-                       srclen -= POLY1305_BLOCK_SIZE;
-                       src += POLY1305_BLOCK_SIZE;
-               }
+               poly1305_blocks_arch(&dctx->state, src, srclen, 1);
+               src += srclen - (srclen % POLY1305_BLOCK_SIZE);
+               srclen %= POLY1305_BLOCK_SIZE;
        }
 
        if (unlikely(srclen)) {
@@ -104,12 +119,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
                dctx->buf[dctx->buflen++] = 1;
                memset(dctx->buf + dctx->buflen, 0,
                       POLY1305_BLOCK_SIZE - dctx->buflen);
-               vsx_begin();
-               poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
-               vsx_end();
+               poly1305_blocks_arch(&dctx->state, dctx->buf,
+                                    POLY1305_BLOCK_SIZE, 0);
        }
 
-       poly1305_emit_64(&dctx->h, &dctx->s, dst);
+       poly1305_emit_arch(&dctx->h, dst, dctx->s);
 }
 EXPORT_SYMBOL(poly1305_final_arch);