Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
56d76c96 JK |
2 | /* |
3 | * Glue Code for x86_64/AVX2 assembler optimized version of Serpent | |
4 | * | |
5 | * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> | |
56d76c96 JK |
6 | */ |
7 | ||
8 | #include <linux/module.h> | |
9 | #include <linux/types.h> | |
10 | #include <linux/crypto.h> | |
11 | #include <linux/err.h> | |
12 | #include <crypto/algapi.h> | |
e16bf974 | 13 | #include <crypto/internal/simd.h> |
56d76c96 | 14 | #include <crypto/serpent.h> |
e16bf974 | 15 | #include <crypto/xts.h> |
56d76c96 | 16 | #include <asm/crypto/glue_helper.h> |
e16bf974 | 17 | #include <asm/crypto/serpent-avx.h> |
56d76c96 JK |
18 | |
19 | #define SERPENT_AVX2_PARALLEL_BLOCKS 16 | |
20 | ||
21 | /* 16-way AVX2 parallel cipher functions */ | |
22 | asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst, | |
23 | const u8 *src); | |
24 | asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst, | |
25 | const u8 *src); | |
26 | asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src); | |
27 | ||
28 | asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src, | |
29 | le128 *iv); | |
30 | asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst, | |
31 | const u8 *src, le128 *iv); | |
32 | asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst, | |
33 | const u8 *src, le128 *iv); | |
34 | ||
e16bf974 EB |
35 | static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, |
36 | const u8 *key, unsigned int keylen) | |
37 | { | |
38 | return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); | |
39 | } | |
40 | ||
56d76c96 JK |
41 | static const struct common_glue_ctx serpent_enc = { |
42 | .num_funcs = 3, | |
43 | .fpu_blocks_limit = 8, | |
44 | ||
45 | .funcs = { { | |
46 | .num_blocks = 16, | |
47 | .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) } | |
48 | }, { | |
49 | .num_blocks = 8, | |
50 | .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) } | |
51 | }, { | |
52 | .num_blocks = 1, | |
53 | .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) } | |
54 | } } | |
55 | }; | |
56 | ||
57 | static const struct common_glue_ctx serpent_ctr = { | |
58 | .num_funcs = 3, | |
59 | .fpu_blocks_limit = 8, | |
60 | ||
61 | .funcs = { { | |
62 | .num_blocks = 16, | |
63 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) } | |
64 | }, { | |
65 | .num_blocks = 8, | |
66 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) } | |
67 | }, { | |
68 | .num_blocks = 1, | |
69 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) } | |
70 | } } | |
71 | }; | |
72 | ||
73 | static const struct common_glue_ctx serpent_enc_xts = { | |
74 | .num_funcs = 3, | |
75 | .fpu_blocks_limit = 8, | |
76 | ||
77 | .funcs = { { | |
78 | .num_blocks = 16, | |
79 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) } | |
80 | }, { | |
81 | .num_blocks = 8, | |
82 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) } | |
83 | }, { | |
84 | .num_blocks = 1, | |
85 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) } | |
86 | } } | |
87 | }; | |
88 | ||
89 | static const struct common_glue_ctx serpent_dec = { | |
90 | .num_funcs = 3, | |
91 | .fpu_blocks_limit = 8, | |
92 | ||
93 | .funcs = { { | |
94 | .num_blocks = 16, | |
95 | .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) } | |
96 | }, { | |
97 | .num_blocks = 8, | |
98 | .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) } | |
99 | }, { | |
100 | .num_blocks = 1, | |
101 | .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) } | |
102 | } } | |
103 | }; | |
104 | ||
105 | static const struct common_glue_ctx serpent_dec_cbc = { | |
106 | .num_funcs = 3, | |
107 | .fpu_blocks_limit = 8, | |
108 | ||
109 | .funcs = { { | |
110 | .num_blocks = 16, | |
111 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) } | |
112 | }, { | |
113 | .num_blocks = 8, | |
114 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) } | |
115 | }, { | |
116 | .num_blocks = 1, | |
117 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) } | |
118 | } } | |
119 | }; | |
120 | ||
121 | static const struct common_glue_ctx serpent_dec_xts = { | |
122 | .num_funcs = 3, | |
123 | .fpu_blocks_limit = 8, | |
124 | ||
125 | .funcs = { { | |
126 | .num_blocks = 16, | |
127 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) } | |
128 | }, { | |
129 | .num_blocks = 8, | |
130 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) } | |
131 | }, { | |
132 | .num_blocks = 1, | |
133 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) } | |
134 | } } | |
135 | }; | |
136 | ||
e16bf974 | 137 | static int ecb_encrypt(struct skcipher_request *req) |
56d76c96 | 138 | { |
e16bf974 | 139 | return glue_ecb_req_128bit(&serpent_enc, req); |
56d76c96 JK |
140 | } |
141 | ||
e16bf974 | 142 | static int ecb_decrypt(struct skcipher_request *req) |
56d76c96 | 143 | { |
e16bf974 | 144 | return glue_ecb_req_128bit(&serpent_dec, req); |
56d76c96 JK |
145 | } |
146 | ||
e16bf974 | 147 | static int cbc_encrypt(struct skcipher_request *req) |
56d76c96 | 148 | { |
e16bf974 EB |
149 | return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt), |
150 | req); | |
56d76c96 JK |
151 | } |
152 | ||
e16bf974 | 153 | static int cbc_decrypt(struct skcipher_request *req) |
56d76c96 | 154 | { |
e16bf974 | 155 | return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); |
56d76c96 JK |
156 | } |
157 | ||
e16bf974 | 158 | static int ctr_crypt(struct skcipher_request *req) |
56d76c96 | 159 | { |
e16bf974 | 160 | return glue_ctr_req_128bit(&serpent_ctr, req); |
56d76c96 JK |
161 | } |
162 | ||
e16bf974 | 163 | static int xts_encrypt(struct skcipher_request *req) |
56d76c96 | 164 | { |
e16bf974 EB |
165 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
166 | struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
56d76c96 | 167 | |
e16bf974 EB |
168 | return glue_xts_req_128bit(&serpent_enc_xts, req, |
169 | XTS_TWEAK_CAST(__serpent_encrypt), | |
8ce5fac2 | 170 | &ctx->tweak_ctx, &ctx->crypt_ctx, false); |
56d76c96 JK |
171 | } |
172 | ||
e16bf974 | 173 | static int xts_decrypt(struct skcipher_request *req) |
56d76c96 | 174 | { |
e16bf974 EB |
175 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
176 | struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
56d76c96 | 177 | |
e16bf974 EB |
178 | return glue_xts_req_128bit(&serpent_dec_xts, req, |
179 | XTS_TWEAK_CAST(__serpent_encrypt), | |
8ce5fac2 | 180 | &ctx->tweak_ctx, &ctx->crypt_ctx, true); |
56d76c96 JK |
181 | } |
182 | ||
e16bf974 EB |
183 | static struct skcipher_alg serpent_algs[] = { |
184 | { | |
185 | .base.cra_name = "__ecb(serpent)", | |
186 | .base.cra_driver_name = "__ecb-serpent-avx2", | |
187 | .base.cra_priority = 600, | |
188 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
189 | .base.cra_blocksize = SERPENT_BLOCK_SIZE, | |
190 | .base.cra_ctxsize = sizeof(struct serpent_ctx), | |
191 | .base.cra_module = THIS_MODULE, | |
192 | .min_keysize = SERPENT_MIN_KEY_SIZE, | |
193 | .max_keysize = SERPENT_MAX_KEY_SIZE, | |
194 | .setkey = serpent_setkey_skcipher, | |
195 | .encrypt = ecb_encrypt, | |
196 | .decrypt = ecb_decrypt, | |
197 | }, { | |
198 | .base.cra_name = "__cbc(serpent)", | |
199 | .base.cra_driver_name = "__cbc-serpent-avx2", | |
200 | .base.cra_priority = 600, | |
201 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
202 | .base.cra_blocksize = SERPENT_BLOCK_SIZE, | |
203 | .base.cra_ctxsize = sizeof(struct serpent_ctx), | |
204 | .base.cra_module = THIS_MODULE, | |
205 | .min_keysize = SERPENT_MIN_KEY_SIZE, | |
206 | .max_keysize = SERPENT_MAX_KEY_SIZE, | |
207 | .ivsize = SERPENT_BLOCK_SIZE, | |
208 | .setkey = serpent_setkey_skcipher, | |
209 | .encrypt = cbc_encrypt, | |
210 | .decrypt = cbc_decrypt, | |
211 | }, { | |
212 | .base.cra_name = "__ctr(serpent)", | |
213 | .base.cra_driver_name = "__ctr-serpent-avx2", | |
214 | .base.cra_priority = 600, | |
215 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
216 | .base.cra_blocksize = 1, | |
217 | .base.cra_ctxsize = sizeof(struct serpent_ctx), | |
218 | .base.cra_module = THIS_MODULE, | |
219 | .min_keysize = SERPENT_MIN_KEY_SIZE, | |
220 | .max_keysize = SERPENT_MAX_KEY_SIZE, | |
221 | .ivsize = SERPENT_BLOCK_SIZE, | |
222 | .chunksize = SERPENT_BLOCK_SIZE, | |
223 | .setkey = serpent_setkey_skcipher, | |
224 | .encrypt = ctr_crypt, | |
225 | .decrypt = ctr_crypt, | |
226 | }, { | |
227 | .base.cra_name = "__xts(serpent)", | |
228 | .base.cra_driver_name = "__xts-serpent-avx2", | |
229 | .base.cra_priority = 600, | |
230 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
231 | .base.cra_blocksize = SERPENT_BLOCK_SIZE, | |
232 | .base.cra_ctxsize = sizeof(struct serpent_xts_ctx), | |
233 | .base.cra_module = THIS_MODULE, | |
234 | .min_keysize = 2 * SERPENT_MIN_KEY_SIZE, | |
235 | .max_keysize = 2 * SERPENT_MAX_KEY_SIZE, | |
236 | .ivsize = SERPENT_BLOCK_SIZE, | |
237 | .setkey = xts_serpent_setkey, | |
238 | .encrypt = xts_encrypt, | |
239 | .decrypt = xts_decrypt, | |
56d76c96 | 240 | }, |
e16bf974 EB |
241 | }; |
242 | ||
243 | static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; | |
56d76c96 JK |
244 | |
245 | static int __init init(void) | |
246 | { | |
534ff06e | 247 | const char *feature_name; |
56d76c96 | 248 | |
abcfdfe0 | 249 | if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) { |
b54b4bbb IM |
250 | pr_info("AVX2 instructions are not detected.\n"); |
251 | return -ENODEV; | |
252 | } | |
d91cab78 DH |
253 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
254 | &feature_name)) { | |
534ff06e | 255 | pr_info("CPU feature '%s' is not supported.\n", feature_name); |
56d76c96 JK |
256 | return -ENODEV; |
257 | } | |
258 | ||
e16bf974 EB |
259 | return simd_register_skciphers_compat(serpent_algs, |
260 | ARRAY_SIZE(serpent_algs), | |
261 | serpent_simd_algs); | |
56d76c96 JK |
262 | } |
263 | ||
264 | static void __exit fini(void) | |
265 | { | |
e16bf974 EB |
266 | simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), |
267 | serpent_simd_algs); | |
56d76c96 JK |
268 | } |
269 | ||
270 | module_init(init); | |
271 | module_exit(fini); | |
272 | ||
273 | MODULE_LICENSE("GPL"); | |
274 | MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized"); | |
5d26a105 KC |
275 | MODULE_ALIAS_CRYPTO("serpent"); |
276 | MODULE_ALIAS_CRYPTO("serpent-asm"); |