Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
107778b5 JG |
2 | /* |
3 | * Glue Code for AVX assembler version of Twofish Cipher | |
4 | * | |
5 | * Copyright (C) 2012 Johannes Goetzfried | |
6 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | |
7 | * | |
18be4527 | 8 | * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> |
107778b5 JG |
9 | */ |
10 | ||
11 | #include <linux/module.h> | |
107778b5 JG |
12 | #include <linux/types.h> |
13 | #include <linux/crypto.h> | |
14 | #include <linux/err.h> | |
15 | #include <crypto/algapi.h> | |
0e6ab46d | 16 | #include <crypto/internal/simd.h> |
107778b5 | 17 | #include <crypto/twofish.h> |
107778b5 | 18 | #include <crypto/xts.h> |
a7378d4e | 19 | #include <asm/crypto/glue_helper.h> |
0e6ab46d | 20 | #include <asm/crypto/twofish.h> |
107778b5 | 21 | |
107778b5 JG |
22 | #define TWOFISH_PARALLEL_BLOCKS 8 |
23 | ||
107778b5 | 24 | /* 8-way parallel cipher functions */ |
8435a3c3 JK |
25 | asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, |
26 | const u8 *src); | |
27 | asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, | |
107778b5 JG |
28 | const u8 *src); |
29 | ||
8435a3c3 JK |
30 | asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, |
31 | const u8 *src); | |
32 | asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, | |
33 | const u8 *src, le128 *iv); | |
107778b5 | 34 | |
18be4527 JK |
35 | asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, |
36 | const u8 *src, le128 *iv); | |
37 | asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, | |
38 | const u8 *src, le128 *iv); | |
39 | ||
0e6ab46d EB |
40 | static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, |
41 | const u8 *key, unsigned int keylen) | |
42 | { | |
43 | return twofish_setkey(&tfm->base, key, keylen); | |
44 | } | |
45 | ||
8435a3c3 | 46 | static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, |
107778b5 JG |
47 | const u8 *src) |
48 | { | |
8435a3c3 | 49 | __twofish_enc_blk_3way(ctx, dst, src, false); |
107778b5 JG |
50 | } |
51 | ||
99f42f93 | 52 | static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) |
18be4527 JK |
53 | { |
54 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, | |
55 | GLUE_FUNC_CAST(twofish_enc_blk)); | |
56 | } | |
57 | ||
99f42f93 | 58 | static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) |
18be4527 JK |
59 | { |
60 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, | |
61 | GLUE_FUNC_CAST(twofish_dec_blk)); | |
62 | } | |
63 | ||
ebeea983 EB |
64 | struct twofish_xts_ctx { |
65 | struct twofish_ctx tweak_ctx; | |
66 | struct twofish_ctx crypt_ctx; | |
67 | }; | |
68 | ||
0e6ab46d | 69 | static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, |
ebeea983 EB |
70 | unsigned int keylen) |
71 | { | |
0e6ab46d EB |
72 | struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
73 | u32 *flags = &tfm->base.crt_flags; | |
ebeea983 EB |
74 | int err; |
75 | ||
0e6ab46d | 76 | err = xts_verify_key(tfm, key, keylen); |
ebeea983 EB |
77 | if (err) |
78 | return err; | |
79 | ||
80 | /* first half of xts-key is for crypt */ | |
81 | err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); | |
82 | if (err) | |
83 | return err; | |
84 | ||
85 | /* second half of xts-key is for tweak */ | |
86 | return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, | |
87 | flags); | |
88 | } | |
107778b5 | 89 | |
a7378d4e JK |
90 | static const struct common_glue_ctx twofish_enc = { |
91 | .num_funcs = 3, | |
92 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
93 | ||
94 | .funcs = { { | |
95 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 96 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) } |
a7378d4e JK |
97 | }, { |
98 | .num_blocks = 3, | |
99 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } | |
100 | }, { | |
101 | .num_blocks = 1, | |
102 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } | |
103 | } } | |
104 | }; | |
105 | ||
106 | static const struct common_glue_ctx twofish_ctr = { | |
107 | .num_funcs = 3, | |
108 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
109 | ||
110 | .funcs = { { | |
111 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 112 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) } |
a7378d4e JK |
113 | }, { |
114 | .num_blocks = 3, | |
115 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } | |
116 | }, { | |
117 | .num_blocks = 1, | |
118 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } | |
119 | } } | |
120 | }; | |
121 | ||
18be4527 JK |
122 | static const struct common_glue_ctx twofish_enc_xts = { |
123 | .num_funcs = 2, | |
124 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
125 | ||
126 | .funcs = { { | |
127 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
128 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) } | |
129 | }, { | |
130 | .num_blocks = 1, | |
131 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) } | |
132 | } } | |
133 | }; | |
134 | ||
a7378d4e JK |
135 | static const struct common_glue_ctx twofish_dec = { |
136 | .num_funcs = 3, | |
137 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
138 | ||
139 | .funcs = { { | |
140 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 141 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) } |
a7378d4e JK |
142 | }, { |
143 | .num_blocks = 3, | |
144 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } | |
145 | }, { | |
146 | .num_blocks = 1, | |
147 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } | |
148 | } } | |
149 | }; | |
150 | ||
151 | static const struct common_glue_ctx twofish_dec_cbc = { | |
152 | .num_funcs = 3, | |
153 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
154 | ||
155 | .funcs = { { | |
156 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 157 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) } |
a7378d4e JK |
158 | }, { |
159 | .num_blocks = 3, | |
160 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } | |
161 | }, { | |
162 | .num_blocks = 1, | |
163 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } | |
164 | } } | |
165 | }; | |
166 | ||
18be4527 JK |
167 | static const struct common_glue_ctx twofish_dec_xts = { |
168 | .num_funcs = 2, | |
169 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
170 | ||
171 | .funcs = { { | |
172 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
173 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) } | |
174 | }, { | |
175 | .num_blocks = 1, | |
176 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) } | |
177 | } } | |
178 | }; | |
179 | ||
0e6ab46d | 180 | static int ecb_encrypt(struct skcipher_request *req) |
107778b5 | 181 | { |
0e6ab46d | 182 | return glue_ecb_req_128bit(&twofish_enc, req); |
107778b5 JG |
183 | } |
184 | ||
0e6ab46d | 185 | static int ecb_decrypt(struct skcipher_request *req) |
107778b5 | 186 | { |
0e6ab46d | 187 | return glue_ecb_req_128bit(&twofish_dec, req); |
107778b5 JG |
188 | } |
189 | ||
0e6ab46d | 190 | static int cbc_encrypt(struct skcipher_request *req) |
107778b5 | 191 | { |
0e6ab46d EB |
192 | return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk), |
193 | req); | |
107778b5 JG |
194 | } |
195 | ||
0e6ab46d | 196 | static int cbc_decrypt(struct skcipher_request *req) |
107778b5 | 197 | { |
0e6ab46d | 198 | return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); |
107778b5 JG |
199 | } |
200 | ||
0e6ab46d | 201 | static int ctr_crypt(struct skcipher_request *req) |
107778b5 | 202 | { |
0e6ab46d | 203 | return glue_ctr_req_128bit(&twofish_ctr, req); |
107778b5 JG |
204 | } |
205 | ||
0e6ab46d | 206 | static int xts_encrypt(struct skcipher_request *req) |
107778b5 | 207 | { |
0e6ab46d EB |
208 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
209 | struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
107778b5 | 210 | |
0e6ab46d EB |
211 | return glue_xts_req_128bit(&twofish_enc_xts, req, |
212 | XTS_TWEAK_CAST(twofish_enc_blk), | |
213 | &ctx->tweak_ctx, &ctx->crypt_ctx); | |
107778b5 JG |
214 | } |
215 | ||
0e6ab46d | 216 | static int xts_decrypt(struct skcipher_request *req) |
107778b5 | 217 | { |
0e6ab46d EB |
218 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
219 | struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
107778b5 | 220 | |
0e6ab46d EB |
221 | return glue_xts_req_128bit(&twofish_dec_xts, req, |
222 | XTS_TWEAK_CAST(twofish_enc_blk), | |
223 | &ctx->tweak_ctx, &ctx->crypt_ctx); | |
107778b5 JG |
224 | } |
225 | ||
0e6ab46d EB |
226 | static struct skcipher_alg twofish_algs[] = { |
227 | { | |
228 | .base.cra_name = "__ecb(twofish)", | |
229 | .base.cra_driver_name = "__ecb-twofish-avx", | |
230 | .base.cra_priority = 400, | |
231 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
232 | .base.cra_blocksize = TF_BLOCK_SIZE, | |
233 | .base.cra_ctxsize = sizeof(struct twofish_ctx), | |
234 | .base.cra_module = THIS_MODULE, | |
235 | .min_keysize = TF_MIN_KEY_SIZE, | |
236 | .max_keysize = TF_MAX_KEY_SIZE, | |
237 | .setkey = twofish_setkey_skcipher, | |
238 | .encrypt = ecb_encrypt, | |
239 | .decrypt = ecb_decrypt, | |
240 | }, { | |
241 | .base.cra_name = "__cbc(twofish)", | |
242 | .base.cra_driver_name = "__cbc-twofish-avx", | |
243 | .base.cra_priority = 400, | |
244 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
245 | .base.cra_blocksize = TF_BLOCK_SIZE, | |
246 | .base.cra_ctxsize = sizeof(struct twofish_ctx), | |
247 | .base.cra_module = THIS_MODULE, | |
248 | .min_keysize = TF_MIN_KEY_SIZE, | |
249 | .max_keysize = TF_MAX_KEY_SIZE, | |
250 | .ivsize = TF_BLOCK_SIZE, | |
251 | .setkey = twofish_setkey_skcipher, | |
252 | .encrypt = cbc_encrypt, | |
253 | .decrypt = cbc_decrypt, | |
254 | }, { | |
255 | .base.cra_name = "__ctr(twofish)", | |
256 | .base.cra_driver_name = "__ctr-twofish-avx", | |
257 | .base.cra_priority = 400, | |
258 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
259 | .base.cra_blocksize = 1, | |
260 | .base.cra_ctxsize = sizeof(struct twofish_ctx), | |
261 | .base.cra_module = THIS_MODULE, | |
262 | .min_keysize = TF_MIN_KEY_SIZE, | |
263 | .max_keysize = TF_MAX_KEY_SIZE, | |
264 | .ivsize = TF_BLOCK_SIZE, | |
265 | .chunksize = TF_BLOCK_SIZE, | |
266 | .setkey = twofish_setkey_skcipher, | |
267 | .encrypt = ctr_crypt, | |
268 | .decrypt = ctr_crypt, | |
269 | }, { | |
270 | .base.cra_name = "__xts(twofish)", | |
271 | .base.cra_driver_name = "__xts-twofish-avx", | |
272 | .base.cra_priority = 400, | |
273 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
274 | .base.cra_blocksize = TF_BLOCK_SIZE, | |
275 | .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), | |
276 | .base.cra_module = THIS_MODULE, | |
277 | .min_keysize = 2 * TF_MIN_KEY_SIZE, | |
278 | .max_keysize = 2 * TF_MAX_KEY_SIZE, | |
279 | .ivsize = TF_BLOCK_SIZE, | |
280 | .setkey = xts_twofish_setkey, | |
281 | .encrypt = xts_encrypt, | |
282 | .decrypt = xts_decrypt, | |
107778b5 | 283 | }, |
0e6ab46d EB |
284 | }; |
285 | ||
286 | static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; | |
107778b5 JG |
287 | |
288 | static int __init twofish_init(void) | |
289 | { | |
4eecd261 | 290 | const char *feature_name; |
107778b5 | 291 | |
158ecc39 | 292 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { |
4eecd261 | 293 | pr_info("CPU feature '%s' is not supported.\n", feature_name); |
107778b5 JG |
294 | return -ENODEV; |
295 | } | |
296 | ||
0e6ab46d EB |
297 | return simd_register_skciphers_compat(twofish_algs, |
298 | ARRAY_SIZE(twofish_algs), | |
299 | twofish_simd_algs); | |
107778b5 JG |
300 | } |
301 | ||
302 | static void __exit twofish_exit(void) | |
303 | { | |
0e6ab46d EB |
304 | simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), |
305 | twofish_simd_algs); | |
107778b5 JG |
306 | } |
307 | ||
308 | module_init(twofish_init); | |
309 | module_exit(twofish_exit); | |
310 | ||
311 | MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); | |
312 | MODULE_LICENSE("GPL"); | |
5d26a105 | 313 | MODULE_ALIAS_CRYPTO("twofish"); |