Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
66be8951 MK |
2 | /* |
3 | * Cryptographic API. | |
4 | * | |
5 | * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using | |
6 | * Supplemental SSE3 instructions. | |
7 | * | |
8 | * This file is based on sha1_generic.c | |
9 | * | |
10 | * Copyright (c) Alan Smithee. | |
11 | * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> | |
12 | * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> | |
13 | * Copyright (c) Mathias Krause <minipli@googlemail.com> | |
7c1da8d0 | 14 | * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> |
66be8951 MK |
15 | */ |
16 | ||
17 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
18 | ||
19 | #include <crypto/internal/hash.h> | |
f2abe0d7 | 20 | #include <crypto/internal/simd.h> |
66be8951 MK |
21 | #include <linux/init.h> |
22 | #include <linux/module.h> | |
23 | #include <linux/mm.h> | |
24 | #include <linux/cryptohash.h> | |
25 | #include <linux/types.h> | |
26 | #include <crypto/sha.h> | |
824b4376 | 27 | #include <crypto/sha1_base.h> |
f2abe0d7 | 28 | #include <asm/simd.h> |
66be8951 | 29 | |
85c66ecd | 30 | typedef void (sha1_transform_fn)(u32 *digest, const char *data, |
31 | unsigned int rounds); | |
66be8951 | 32 | |
85c66ecd | 33 | static int sha1_update(struct shash_desc *desc, const u8 *data, |
34 | unsigned int len, sha1_transform_fn *sha1_xform) | |
66be8951 MK |
35 | { |
36 | struct sha1_state *sctx = shash_desc_ctx(desc); | |
66be8951 | 37 | |
f2abe0d7 | 38 | if (!crypto_simd_usable() || |
824b4376 AB |
39 | (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) |
40 | return crypto_sha1_update(desc, data, len); | |
66be8951 | 41 | |
824b4376 AB |
42 | /* make sure casting to sha1_block_fn() is safe */ |
43 | BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); | |
66be8951 | 44 | |
824b4376 AB |
45 | kernel_fpu_begin(); |
46 | sha1_base_do_update(desc, data, len, | |
85c66ecd | 47 | (sha1_block_fn *)sha1_xform); |
824b4376 | 48 | kernel_fpu_end(); |
66be8951 MK |
49 | |
50 | return 0; | |
51 | } | |
52 | ||
85c66ecd | 53 | static int sha1_finup(struct shash_desc *desc, const u8 *data, |
54 | unsigned int len, u8 *out, sha1_transform_fn *sha1_xform) | |
66be8951 | 55 | { |
f2abe0d7 | 56 | if (!crypto_simd_usable()) |
824b4376 | 57 | return crypto_sha1_finup(desc, data, len, out); |
66be8951 | 58 | |
824b4376 AB |
59 | kernel_fpu_begin(); |
60 | if (len) | |
61 | sha1_base_do_update(desc, data, len, | |
85c66ecd | 62 | (sha1_block_fn *)sha1_xform); |
63 | sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform); | |
824b4376 | 64 | kernel_fpu_end(); |
66be8951 | 65 | |
824b4376 | 66 | return sha1_base_finish(desc, out); |
66be8951 MK |
67 | } |
68 | ||
85c66ecd | 69 | asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data, |
70 | unsigned int rounds); | |
71 | ||
72 | static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, | |
73 | unsigned int len) | |
66be8951 | 74 | { |
85c66ecd | 75 | return sha1_update(desc, data, len, |
76 | (sha1_transform_fn *) sha1_transform_ssse3); | |
66be8951 MK |
77 | } |
78 | ||
85c66ecd | 79 | static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, |
80 | unsigned int len, u8 *out) | |
7c1da8d0 | 81 | { |
85c66ecd | 82 | return sha1_finup(desc, data, len, out, |
83 | (sha1_transform_fn *) sha1_transform_ssse3); | |
84 | } | |
85 | ||
86 | /* Add padding and return the message digest. */ | |
87 | static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) | |
88 | { | |
89 | return sha1_ssse3_finup(desc, NULL, 0, out); | |
7c1da8d0 | 90 | } |
7c1da8d0 | 91 | |
85c66ecd | 92 | static struct shash_alg sha1_ssse3_alg = { |
66be8951 | 93 | .digestsize = SHA1_DIGEST_SIZE, |
824b4376 | 94 | .init = sha1_base_init, |
66be8951 MK |
95 | .update = sha1_ssse3_update, |
96 | .final = sha1_ssse3_final, | |
824b4376 | 97 | .finup = sha1_ssse3_finup, |
66be8951 | 98 | .descsize = sizeof(struct sha1_state), |
66be8951 MK |
99 | .base = { |
100 | .cra_name = "sha1", | |
85c66ecd | 101 | .cra_driver_name = "sha1-ssse3", |
66be8951 | 102 | .cra_priority = 150, |
66be8951 MK |
103 | .cra_blocksize = SHA1_BLOCK_SIZE, |
104 | .cra_module = THIS_MODULE, | |
105 | } | |
106 | }; | |
107 | ||
85c66ecd | 108 | static int register_sha1_ssse3(void) |
109 | { | |
110 | if (boot_cpu_has(X86_FEATURE_SSSE3)) | |
111 | return crypto_register_shash(&sha1_ssse3_alg); | |
112 | return 0; | |
113 | } | |
114 | ||
115 | static void unregister_sha1_ssse3(void) | |
116 | { | |
117 | if (boot_cpu_has(X86_FEATURE_SSSE3)) | |
118 | crypto_unregister_shash(&sha1_ssse3_alg); | |
119 | } | |
120 | ||
65df5774 | 121 | #ifdef CONFIG_AS_AVX |
85c66ecd | 122 | asmlinkage void sha1_transform_avx(u32 *digest, const char *data, |
123 | unsigned int rounds); | |
124 | ||
125 | static int sha1_avx_update(struct shash_desc *desc, const u8 *data, | |
126 | unsigned int len) | |
127 | { | |
128 | return sha1_update(desc, data, len, | |
129 | (sha1_transform_fn *) sha1_transform_avx); | |
130 | } | |
131 | ||
132 | static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, | |
133 | unsigned int len, u8 *out) | |
134 | { | |
135 | return sha1_finup(desc, data, len, out, | |
136 | (sha1_transform_fn *) sha1_transform_avx); | |
137 | } | |
138 | ||
139 | static int sha1_avx_final(struct shash_desc *desc, u8 *out) | |
140 | { | |
141 | return sha1_avx_finup(desc, NULL, 0, out); | |
142 | } | |
143 | ||
144 | static struct shash_alg sha1_avx_alg = { | |
145 | .digestsize = SHA1_DIGEST_SIZE, | |
146 | .init = sha1_base_init, | |
147 | .update = sha1_avx_update, | |
148 | .final = sha1_avx_final, | |
149 | .finup = sha1_avx_finup, | |
150 | .descsize = sizeof(struct sha1_state), | |
151 | .base = { | |
152 | .cra_name = "sha1", | |
153 | .cra_driver_name = "sha1-avx", | |
154 | .cra_priority = 160, | |
85c66ecd | 155 | .cra_blocksize = SHA1_BLOCK_SIZE, |
156 | .cra_module = THIS_MODULE, | |
157 | } | |
158 | }; | |
159 | ||
160 | static bool avx_usable(void) | |
66be8951 | 161 | { |
d91cab78 | 162 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
da154e82 | 163 | if (boot_cpu_has(X86_FEATURE_AVX)) |
d1e50966 | 164 | pr_info("AVX detected but unusable.\n"); |
66be8951 MK |
165 | return false; |
166 | } | |
167 | ||
168 | return true; | |
169 | } | |
6ca5afb8 | 170 | |
85c66ecd | 171 | static int register_sha1_avx(void) |
172 | { | |
173 | if (avx_usable()) | |
174 | return crypto_register_shash(&sha1_avx_alg); | |
175 | return 0; | |
176 | } | |
177 | ||
178 | static void unregister_sha1_avx(void) | |
6ca5afb8 | 179 | { |
85c66ecd | 180 | if (avx_usable()) |
181 | crypto_unregister_shash(&sha1_avx_alg); | |
182 | } | |
183 | ||
184 | #else /* CONFIG_AS_AVX */ | |
185 | static inline int register_sha1_avx(void) { return 0; } | |
186 | static inline void unregister_sha1_avx(void) { } | |
187 | #endif /* CONFIG_AS_AVX */ | |
188 | ||
189 | ||
190 | #if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX) | |
191 | #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ | |
192 | ||
193 | asmlinkage void sha1_transform_avx2(u32 *digest, const char *data, | |
194 | unsigned int rounds); | |
195 | ||
196 | static bool avx2_usable(void) | |
6ca5afb8 | 197 | { |
8861249c | 198 | if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) |
85c66ecd | 199 | && boot_cpu_has(X86_FEATURE_BMI1) |
200 | && boot_cpu_has(X86_FEATURE_BMI2)) | |
6ca5afb8 MK |
201 | return true; |
202 | ||
203 | return false; | |
204 | } | |
66be8951 | 205 | |
85c66ecd | 206 | static void sha1_apply_transform_avx2(u32 *digest, const char *data, |
207 | unsigned int rounds) | |
66be8951 | 208 | { |
85c66ecd | 209 | /* Select the optimal transform based on data block size */ |
210 | if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE) | |
211 | sha1_transform_avx2(digest, data, rounds); | |
212 | else | |
213 | sha1_transform_avx(digest, data, rounds); | |
214 | } | |
6ca5afb8 | 215 | |
85c66ecd | 216 | static int sha1_avx2_update(struct shash_desc *desc, const u8 *data, |
217 | unsigned int len) | |
218 | { | |
219 | return sha1_update(desc, data, len, | |
220 | (sha1_transform_fn *) sha1_apply_transform_avx2); | |
221 | } | |
66be8951 | 222 | |
85c66ecd | 223 | static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, |
224 | unsigned int len, u8 *out) | |
225 | { | |
226 | return sha1_finup(desc, data, len, out, | |
227 | (sha1_transform_fn *) sha1_apply_transform_avx2); | |
228 | } | |
229 | ||
230 | static int sha1_avx2_final(struct shash_desc *desc, u8 *out) | |
231 | { | |
232 | return sha1_avx2_finup(desc, NULL, 0, out); | |
233 | } | |
234 | ||
235 | static struct shash_alg sha1_avx2_alg = { | |
236 | .digestsize = SHA1_DIGEST_SIZE, | |
237 | .init = sha1_base_init, | |
238 | .update = sha1_avx2_update, | |
239 | .final = sha1_avx2_final, | |
240 | .finup = sha1_avx2_finup, | |
241 | .descsize = sizeof(struct sha1_state), | |
242 | .base = { | |
243 | .cra_name = "sha1", | |
244 | .cra_driver_name = "sha1-avx2", | |
245 | .cra_priority = 170, | |
85c66ecd | 246 | .cra_blocksize = SHA1_BLOCK_SIZE, |
247 | .cra_module = THIS_MODULE, | |
7c1da8d0 | 248 | } |
85c66ecd | 249 | }; |
250 | ||
251 | static int register_sha1_avx2(void) | |
252 | { | |
253 | if (avx2_usable()) | |
254 | return crypto_register_shash(&sha1_avx2_alg); | |
255 | return 0; | |
256 | } | |
257 | ||
258 | static void unregister_sha1_avx2(void) | |
259 | { | |
260 | if (avx2_usable()) | |
261 | crypto_unregister_shash(&sha1_avx2_alg); | |
262 | } | |
263 | ||
264 | #else | |
265 | static inline int register_sha1_avx2(void) { return 0; } | |
266 | static inline void unregister_sha1_avx2(void) { } | |
6ca5afb8 | 267 | #endif |
85c66ecd | 268 | |
95fca7df | 269 | #ifdef CONFIG_AS_SHA1_NI |
85c66ecd | 270 | asmlinkage void sha1_ni_transform(u32 *digest, const char *data, |
271 | unsigned int rounds); | |
272 | ||
273 | static int sha1_ni_update(struct shash_desc *desc, const u8 *data, | |
274 | unsigned int len) | |
275 | { | |
276 | return sha1_update(desc, data, len, | |
277 | (sha1_transform_fn *) sha1_ni_transform); | |
278 | } | |
279 | ||
280 | static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, | |
281 | unsigned int len, u8 *out) | |
282 | { | |
283 | return sha1_finup(desc, data, len, out, | |
284 | (sha1_transform_fn *) sha1_ni_transform); | |
285 | } | |
286 | ||
287 | static int sha1_ni_final(struct shash_desc *desc, u8 *out) | |
288 | { | |
289 | return sha1_ni_finup(desc, NULL, 0, out); | |
290 | } | |
291 | ||
292 | static struct shash_alg sha1_ni_alg = { | |
293 | .digestsize = SHA1_DIGEST_SIZE, | |
294 | .init = sha1_base_init, | |
295 | .update = sha1_ni_update, | |
296 | .final = sha1_ni_final, | |
297 | .finup = sha1_ni_finup, | |
298 | .descsize = sizeof(struct sha1_state), | |
299 | .base = { | |
300 | .cra_name = "sha1", | |
301 | .cra_driver_name = "sha1-ni", | |
302 | .cra_priority = 250, | |
85c66ecd | 303 | .cra_blocksize = SHA1_BLOCK_SIZE, |
304 | .cra_module = THIS_MODULE, | |
95fca7df | 305 | } |
85c66ecd | 306 | }; |
307 | ||
308 | static int register_sha1_ni(void) | |
309 | { | |
310 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) | |
311 | return crypto_register_shash(&sha1_ni_alg); | |
312 | return 0; | |
313 | } | |
314 | ||
315 | static void unregister_sha1_ni(void) | |
316 | { | |
317 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) | |
318 | crypto_unregister_shash(&sha1_ni_alg); | |
319 | } | |
320 | ||
321 | #else | |
322 | static inline int register_sha1_ni(void) { return 0; } | |
323 | static inline void unregister_sha1_ni(void) { } | |
66be8951 MK |
324 | #endif |
325 | ||
326 | static int __init sha1_ssse3_mod_init(void) | |
327 | { | |
85c66ecd | 328 | if (register_sha1_ssse3()) |
329 | goto fail; | |
6ca5afb8 | 330 | |
85c66ecd | 331 | if (register_sha1_avx()) { |
332 | unregister_sha1_ssse3(); | |
333 | goto fail; | |
7c1da8d0 | 334 | } |
66be8951 | 335 | |
85c66ecd | 336 | if (register_sha1_avx2()) { |
337 | unregister_sha1_avx(); | |
338 | unregister_sha1_ssse3(); | |
339 | goto fail; | |
7c1da8d0 | 340 | } |
66be8951 | 341 | |
85c66ecd | 342 | if (register_sha1_ni()) { |
343 | unregister_sha1_avx2(); | |
344 | unregister_sha1_avx(); | |
345 | unregister_sha1_ssse3(); | |
346 | goto fail; | |
66be8951 | 347 | } |
66be8951 | 348 | |
85c66ecd | 349 | return 0; |
350 | fail: | |
66be8951 MK |
351 | return -ENODEV; |
352 | } | |
353 | ||
354 | static void __exit sha1_ssse3_mod_fini(void) | |
355 | { | |
85c66ecd | 356 | unregister_sha1_ni(); |
357 | unregister_sha1_avx2(); | |
358 | unregister_sha1_avx(); | |
359 | unregister_sha1_ssse3(); | |
66be8951 MK |
360 | } |
361 | ||
362 | module_init(sha1_ssse3_mod_init); | |
363 | module_exit(sha1_ssse3_mod_fini); | |
364 | ||
365 | MODULE_LICENSE("GPL"); | |
366 | MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); | |
367 | ||
5d26a105 | 368 | MODULE_ALIAS_CRYPTO("sha1"); |
1a445e8e SM |
369 | MODULE_ALIAS_CRYPTO("sha1-ssse3"); |
370 | MODULE_ALIAS_CRYPTO("sha1-avx"); | |
371 | MODULE_ALIAS_CRYPTO("sha1-avx2"); | |
372 | #ifdef CONFIG_AS_SHA1_NI | |
373 | MODULE_ALIAS_CRYPTO("sha1-ni"); | |
374 | #endif |