Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
d0a3431a AB |
2 | /* |
3 | * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions | |
4 | * | |
5 | * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> | |
d0a3431a AB |
6 | */ |
7 | ||
2a9faf8b | 8 | #include <linux/cpufeature.h> |
d0a3431a AB |
9 | #include <linux/crc32.h> |
10 | #include <linux/init.h> | |
11 | #include <linux/kernel.h> | |
12 | #include <linux/module.h> | |
13 | #include <linux/string.h> | |
14 | ||
15 | #include <crypto/internal/hash.h> | |
99680c5e | 16 | #include <crypto/internal/simd.h> |
d0a3431a AB |
17 | |
18 | #include <asm/hwcap.h> | |
19 | #include <asm/neon.h> | |
20 | #include <asm/simd.h> | |
21 | #include <asm/unaligned.h> | |
22 | ||
23 | #define PMULL_MIN_LEN 64L /* minimum size of buffer | |
24 | * for crc32_pmull_le_16 */ | |
25 | #define SCALE_F 16L /* size of NEON register */ | |
26 | ||
27 | asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); | |
28 | asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); | |
29 | ||
30 | asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); | |
31 | asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); | |
32 | ||
33 | static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len); | |
34 | static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len); | |
35 | ||
36 | static int crc32_cra_init(struct crypto_tfm *tfm) | |
37 | { | |
38 | u32 *key = crypto_tfm_ctx(tfm); | |
39 | ||
40 | *key = 0; | |
41 | return 0; | |
42 | } | |
43 | ||
44 | static int crc32c_cra_init(struct crypto_tfm *tfm) | |
45 | { | |
46 | u32 *key = crypto_tfm_ctx(tfm); | |
47 | ||
48 | *key = ~0; | |
49 | return 0; | |
50 | } | |
51 | ||
52 | static int crc32_setkey(struct crypto_shash *hash, const u8 *key, | |
53 | unsigned int keylen) | |
54 | { | |
55 | u32 *mctx = crypto_shash_ctx(hash); | |
56 | ||
674f368a | 57 | if (keylen != sizeof(u32)) |
d0a3431a | 58 | return -EINVAL; |
d0a3431a AB |
59 | *mctx = le32_to_cpup((__le32 *)key); |
60 | return 0; | |
61 | } | |
62 | ||
63 | static int crc32_init(struct shash_desc *desc) | |
64 | { | |
65 | u32 *mctx = crypto_shash_ctx(desc->tfm); | |
66 | u32 *crc = shash_desc_ctx(desc); | |
67 | ||
68 | *crc = *mctx; | |
69 | return 0; | |
70 | } | |
71 | ||
72 | static int crc32_update(struct shash_desc *desc, const u8 *data, | |
73 | unsigned int length) | |
74 | { | |
75 | u32 *crc = shash_desc_ctx(desc); | |
76 | ||
77 | *crc = crc32_armv8_le(*crc, data, length); | |
78 | return 0; | |
79 | } | |
80 | ||
81 | static int crc32c_update(struct shash_desc *desc, const u8 *data, | |
82 | unsigned int length) | |
83 | { | |
84 | u32 *crc = shash_desc_ctx(desc); | |
85 | ||
86 | *crc = crc32c_armv8_le(*crc, data, length); | |
87 | return 0; | |
88 | } | |
89 | ||
90 | static int crc32_final(struct shash_desc *desc, u8 *out) | |
91 | { | |
92 | u32 *crc = shash_desc_ctx(desc); | |
93 | ||
94 | put_unaligned_le32(*crc, out); | |
95 | return 0; | |
96 | } | |
97 | ||
98 | static int crc32c_final(struct shash_desc *desc, u8 *out) | |
99 | { | |
100 | u32 *crc = shash_desc_ctx(desc); | |
101 | ||
102 | put_unaligned_le32(~*crc, out); | |
103 | return 0; | |
104 | } | |
105 | ||
106 | static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, | |
107 | unsigned int length) | |
108 | { | |
109 | u32 *crc = shash_desc_ctx(desc); | |
110 | unsigned int l; | |
111 | ||
99680c5e | 112 | if (crypto_simd_usable()) { |
d0a3431a AB |
113 | if ((u32)data % SCALE_F) { |
114 | l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); | |
115 | ||
116 | *crc = fallback_crc32(*crc, data, l); | |
117 | ||
118 | data += l; | |
119 | length -= l; | |
120 | } | |
121 | ||
122 | if (length >= PMULL_MIN_LEN) { | |
123 | l = round_down(length, SCALE_F); | |
124 | ||
125 | kernel_neon_begin(); | |
126 | *crc = crc32_pmull_le(data, l, *crc); | |
127 | kernel_neon_end(); | |
128 | ||
129 | data += l; | |
130 | length -= l; | |
131 | } | |
132 | } | |
133 | ||
134 | if (length > 0) | |
135 | *crc = fallback_crc32(*crc, data, length); | |
136 | ||
137 | return 0; | |
138 | } | |
139 | ||
140 | static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data, | |
141 | unsigned int length) | |
142 | { | |
143 | u32 *crc = shash_desc_ctx(desc); | |
144 | unsigned int l; | |
145 | ||
99680c5e | 146 | if (crypto_simd_usable()) { |
d0a3431a AB |
147 | if ((u32)data % SCALE_F) { |
148 | l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); | |
149 | ||
150 | *crc = fallback_crc32c(*crc, data, l); | |
151 | ||
152 | data += l; | |
153 | length -= l; | |
154 | } | |
155 | ||
156 | if (length >= PMULL_MIN_LEN) { | |
157 | l = round_down(length, SCALE_F); | |
158 | ||
159 | kernel_neon_begin(); | |
160 | *crc = crc32c_pmull_le(data, l, *crc); | |
161 | kernel_neon_end(); | |
162 | ||
163 | data += l; | |
164 | length -= l; | |
165 | } | |
166 | } | |
167 | ||
168 | if (length > 0) | |
169 | *crc = fallback_crc32c(*crc, data, length); | |
170 | ||
171 | return 0; | |
172 | } | |
173 | ||
174 | static struct shash_alg crc32_pmull_algs[] = { { | |
175 | .setkey = crc32_setkey, | |
176 | .init = crc32_init, | |
177 | .update = crc32_update, | |
178 | .final = crc32_final, | |
179 | .descsize = sizeof(u32), | |
180 | .digestsize = sizeof(u32), | |
181 | ||
182 | .base.cra_ctxsize = sizeof(u32), | |
183 | .base.cra_init = crc32_cra_init, | |
184 | .base.cra_name = "crc32", | |
185 | .base.cra_driver_name = "crc32-arm-ce", | |
186 | .base.cra_priority = 200, | |
a208fa8f | 187 | .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, |
d0a3431a AB |
188 | .base.cra_blocksize = 1, |
189 | .base.cra_module = THIS_MODULE, | |
190 | }, { | |
191 | .setkey = crc32_setkey, | |
192 | .init = crc32_init, | |
193 | .update = crc32c_update, | |
194 | .final = crc32c_final, | |
195 | .descsize = sizeof(u32), | |
196 | .digestsize = sizeof(u32), | |
197 | ||
198 | .base.cra_ctxsize = sizeof(u32), | |
199 | .base.cra_init = crc32c_cra_init, | |
200 | .base.cra_name = "crc32c", | |
201 | .base.cra_driver_name = "crc32c-arm-ce", | |
202 | .base.cra_priority = 200, | |
a208fa8f | 203 | .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, |
d0a3431a AB |
204 | .base.cra_blocksize = 1, |
205 | .base.cra_module = THIS_MODULE, | |
206 | } }; | |
207 | ||
208 | static int __init crc32_pmull_mod_init(void) | |
209 | { | |
210 | if (elf_hwcap2 & HWCAP2_PMULL) { | |
211 | crc32_pmull_algs[0].update = crc32_pmull_update; | |
212 | crc32_pmull_algs[1].update = crc32c_pmull_update; | |
213 | ||
214 | if (elf_hwcap2 & HWCAP2_CRC32) { | |
215 | fallback_crc32 = crc32_armv8_le; | |
216 | fallback_crc32c = crc32c_armv8_le; | |
217 | } else { | |
218 | fallback_crc32 = crc32_le; | |
219 | fallback_crc32c = __crc32c_le; | |
220 | } | |
221 | } else if (!(elf_hwcap2 & HWCAP2_CRC32)) { | |
222 | return -ENODEV; | |
223 | } | |
224 | ||
225 | return crypto_register_shashes(crc32_pmull_algs, | |
226 | ARRAY_SIZE(crc32_pmull_algs)); | |
227 | } | |
228 | ||
229 | static void __exit crc32_pmull_mod_exit(void) | |
230 | { | |
231 | crypto_unregister_shashes(crc32_pmull_algs, | |
232 | ARRAY_SIZE(crc32_pmull_algs)); | |
233 | } | |
234 | ||
cd560235 | 235 | static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = { |
2a9faf8b AB |
236 | { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { } |
237 | }; | |
238 | MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature); | |
239 | ||
d0a3431a AB |
240 | module_init(crc32_pmull_mod_init); |
241 | module_exit(crc32_pmull_mod_exit); | |
242 | ||
243 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); | |
244 | MODULE_LICENSE("GPL v2"); | |
245 | MODULE_ALIAS_CRYPTO("crc32"); | |
246 | MODULE_ALIAS_CRYPTO("crc32c"); |