Commit | Line | Data |
---|---|---|
0f961f9f EB |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum | |
4 | * (AVX2 accelerated version) | |
5 | * | |
6 | * Copyright 2018 Google LLC | |
7 | */ | |
8 | ||
9 | #include <crypto/internal/hash.h> | |
f2abe0d7 | 10 | #include <crypto/internal/simd.h> |
0f961f9f EB |
11 | #include <crypto/nhpoly1305.h> |
12 | #include <linux/module.h> | |
f2abe0d7 | 13 | #include <asm/simd.h> |
0f961f9f EB |
14 | |
15 | asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, | |
16 | u8 hash[NH_HASH_BYTES]); | |
17 | ||
18 | /* wrapper to avoid indirect call to assembly, which doesn't work with CFI */ | |
19 | static void _nh_avx2(const u32 *key, const u8 *message, size_t message_len, | |
20 | __le64 hash[NH_NUM_PASSES]) | |
21 | { | |
22 | nh_avx2(key, message, message_len, (u8 *)hash); | |
23 | } | |
24 | ||
25 | static int nhpoly1305_avx2_update(struct shash_desc *desc, | |
26 | const u8 *src, unsigned int srclen) | |
27 | { | |
f2abe0d7 | 28 | if (srclen < 64 || !crypto_simd_usable()) |
0f961f9f EB |
29 | return crypto_nhpoly1305_update(desc, src, srclen); |
30 | ||
31 | do { | |
32 | unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE); | |
33 | ||
34 | kernel_fpu_begin(); | |
35 | crypto_nhpoly1305_update_helper(desc, src, n, _nh_avx2); | |
36 | kernel_fpu_end(); | |
37 | src += n; | |
38 | srclen -= n; | |
39 | } while (srclen); | |
40 | return 0; | |
41 | } | |
42 | ||
43 | static struct shash_alg nhpoly1305_alg = { | |
44 | .base.cra_name = "nhpoly1305", | |
45 | .base.cra_driver_name = "nhpoly1305-avx2", | |
46 | .base.cra_priority = 300, | |
47 | .base.cra_ctxsize = sizeof(struct nhpoly1305_key), | |
48 | .base.cra_module = THIS_MODULE, | |
49 | .digestsize = POLY1305_DIGEST_SIZE, | |
50 | .init = crypto_nhpoly1305_init, | |
51 | .update = nhpoly1305_avx2_update, | |
52 | .final = crypto_nhpoly1305_final, | |
53 | .setkey = crypto_nhpoly1305_setkey, | |
54 | .descsize = sizeof(struct nhpoly1305_state), | |
55 | }; | |
56 | ||
57 | static int __init nhpoly1305_mod_init(void) | |
58 | { | |
59 | if (!boot_cpu_has(X86_FEATURE_AVX2) || | |
60 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) | |
61 | return -ENODEV; | |
62 | ||
63 | return crypto_register_shash(&nhpoly1305_alg); | |
64 | } | |
65 | ||
66 | static void __exit nhpoly1305_mod_exit(void) | |
67 | { | |
68 | crypto_unregister_shash(&nhpoly1305_alg); | |
69 | } | |
70 | ||
71 | module_init(nhpoly1305_mod_init); | |
72 | module_exit(nhpoly1305_mod_exit); | |
73 | ||
74 | MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)"); | |
75 | MODULE_LICENSE("GPL v2"); | |
76 | MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); | |
77 | MODULE_ALIAS_CRYPTO("nhpoly1305"); | |
78 | MODULE_ALIAS_CRYPTO("nhpoly1305-avx2"); |