Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
4ea1277d | 2 | /* |
a97673a1 | 3 | * Glue Code for the AVX assembler implementation of the Cast6 Cipher |
4ea1277d JG |
4 | * |
5 | * Copyright (C) 2012 Johannes Goetzfried | |
6 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | |
7 | * | |
70177286 | 8 | * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> |
4ea1277d JG |
9 | */ |
10 | ||
11 | #include <linux/module.h> | |
4ea1277d JG |
12 | #include <linux/types.h> |
13 | #include <linux/crypto.h> | |
14 | #include <linux/err.h> | |
15 | #include <crypto/algapi.h> | |
16 | #include <crypto/cast6.h> | |
4bd96924 | 17 | #include <crypto/internal/simd.h> |
4ea1277d | 18 | #include <crypto/xts.h> |
4ea1277d JG |
19 | #include <asm/crypto/glue_helper.h> |
20 | ||
21 | #define CAST6_PARALLEL_BLOCKS 8 | |
22 | ||
9c1e8836 KC |
23 | asmlinkage void cast6_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); |
24 | asmlinkage void cast6_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); | |
25 | ||
26 | asmlinkage void cast6_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); | |
27 | asmlinkage void cast6_ctr_8way(const void *ctx, u8 *dst, const u8 *src, | |
cba1cce0 | 28 | le128 *iv); |
4ea1277d | 29 | |
9c1e8836 KC |
30 | asmlinkage void cast6_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, |
31 | le128 *iv); | |
32 | asmlinkage void cast6_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, | |
33 | le128 *iv); | |
70177286 | 34 | |
4bd96924 EB |
35 | static int cast6_setkey_skcipher(struct crypto_skcipher *tfm, |
36 | const u8 *key, unsigned int keylen) | |
37 | { | |
38 | return cast6_setkey(&tfm->base, key, keylen); | |
39 | } | |
40 | ||
9c1e8836 | 41 | static void cast6_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) |
70177286 | 42 | { |
9c1e8836 | 43 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, __cast6_encrypt); |
70177286 JK |
44 | } |
45 | ||
9c1e8836 | 46 | static void cast6_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) |
70177286 | 47 | { |
9c1e8836 | 48 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, __cast6_decrypt); |
70177286 JK |
49 | } |
50 | ||
9c1e8836 | 51 | static void cast6_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) |
4ea1277d JG |
52 | { |
53 | be128 ctrblk; | |
9c1e8836 KC |
54 | u128 *dst = (u128 *)d; |
55 | const u128 *src = (const u128 *)s; | |
4ea1277d | 56 | |
58990986 JK |
57 | le128_to_be128(&ctrblk, iv); |
58 | le128_inc(iv); | |
4ea1277d JG |
59 | |
60 | __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); | |
61 | u128_xor(dst, src, (u128 *)&ctrblk); | |
62 | } | |
63 | ||
4ea1277d JG |
64 | static const struct common_glue_ctx cast6_enc = { |
65 | .num_funcs = 2, | |
66 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | |
67 | ||
68 | .funcs = { { | |
69 | .num_blocks = CAST6_PARALLEL_BLOCKS, | |
9c1e8836 | 70 | .fn_u = { .ecb = cast6_ecb_enc_8way } |
4ea1277d JG |
71 | }, { |
72 | .num_blocks = 1, | |
9c1e8836 | 73 | .fn_u = { .ecb = __cast6_encrypt } |
4ea1277d JG |
74 | } } |
75 | }; | |
76 | ||
77 | static const struct common_glue_ctx cast6_ctr = { | |
78 | .num_funcs = 2, | |
79 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | |
80 | ||
81 | .funcs = { { | |
82 | .num_blocks = CAST6_PARALLEL_BLOCKS, | |
9c1e8836 | 83 | .fn_u = { .ctr = cast6_ctr_8way } |
4ea1277d JG |
84 | }, { |
85 | .num_blocks = 1, | |
9c1e8836 | 86 | .fn_u = { .ctr = cast6_crypt_ctr } |
4ea1277d JG |
87 | } } |
88 | }; | |
89 | ||
70177286 JK |
90 | static const struct common_glue_ctx cast6_enc_xts = { |
91 | .num_funcs = 2, | |
92 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | |
93 | ||
94 | .funcs = { { | |
95 | .num_blocks = CAST6_PARALLEL_BLOCKS, | |
9c1e8836 | 96 | .fn_u = { .xts = cast6_xts_enc_8way } |
70177286 JK |
97 | }, { |
98 | .num_blocks = 1, | |
9c1e8836 | 99 | .fn_u = { .xts = cast6_xts_enc } |
70177286 JK |
100 | } } |
101 | }; | |
102 | ||
4ea1277d JG |
103 | static const struct common_glue_ctx cast6_dec = { |
104 | .num_funcs = 2, | |
105 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | |
106 | ||
107 | .funcs = { { | |
108 | .num_blocks = CAST6_PARALLEL_BLOCKS, | |
9c1e8836 | 109 | .fn_u = { .ecb = cast6_ecb_dec_8way } |
4ea1277d JG |
110 | }, { |
111 | .num_blocks = 1, | |
9c1e8836 | 112 | .fn_u = { .ecb = __cast6_decrypt } |
4ea1277d JG |
113 | } } |
114 | }; | |
115 | ||
116 | static const struct common_glue_ctx cast6_dec_cbc = { | |
117 | .num_funcs = 2, | |
118 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | |
119 | ||
120 | .funcs = { { | |
121 | .num_blocks = CAST6_PARALLEL_BLOCKS, | |
9c1e8836 | 122 | .fn_u = { .cbc = cast6_cbc_dec_8way } |
4ea1277d JG |
123 | }, { |
124 | .num_blocks = 1, | |
9c1e8836 | 125 | .fn_u = { .cbc = __cast6_decrypt } |
4ea1277d JG |
126 | } } |
127 | }; | |
128 | ||
70177286 JK |
129 | static const struct common_glue_ctx cast6_dec_xts = { |
130 | .num_funcs = 2, | |
131 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | |
132 | ||
133 | .funcs = { { | |
134 | .num_blocks = CAST6_PARALLEL_BLOCKS, | |
9c1e8836 | 135 | .fn_u = { .xts = cast6_xts_dec_8way } |
70177286 JK |
136 | }, { |
137 | .num_blocks = 1, | |
9c1e8836 | 138 | .fn_u = { .xts = cast6_xts_dec } |
70177286 JK |
139 | } } |
140 | }; | |
141 | ||
4bd96924 | 142 | static int ecb_encrypt(struct skcipher_request *req) |
4ea1277d | 143 | { |
4bd96924 | 144 | return glue_ecb_req_128bit(&cast6_enc, req); |
4ea1277d JG |
145 | } |
146 | ||
4bd96924 | 147 | static int ecb_decrypt(struct skcipher_request *req) |
4ea1277d | 148 | { |
4bd96924 | 149 | return glue_ecb_req_128bit(&cast6_dec, req); |
4ea1277d JG |
150 | } |
151 | ||
4bd96924 | 152 | static int cbc_encrypt(struct skcipher_request *req) |
4ea1277d | 153 | { |
9c1e8836 | 154 | return glue_cbc_encrypt_req_128bit(__cast6_encrypt, req); |
4ea1277d JG |
155 | } |
156 | ||
4bd96924 | 157 | static int cbc_decrypt(struct skcipher_request *req) |
4ea1277d | 158 | { |
4bd96924 | 159 | return glue_cbc_decrypt_req_128bit(&cast6_dec_cbc, req); |
4ea1277d JG |
160 | } |
161 | ||
4bd96924 | 162 | static int ctr_crypt(struct skcipher_request *req) |
4ea1277d | 163 | { |
4bd96924 | 164 | return glue_ctr_req_128bit(&cast6_ctr, req); |
4ea1277d JG |
165 | } |
166 | ||
4ea1277d JG |
167 | struct cast6_xts_ctx { |
168 | struct cast6_ctx tweak_ctx; | |
169 | struct cast6_ctx crypt_ctx; | |
170 | }; | |
171 | ||
4bd96924 EB |
172 | static int xts_cast6_setkey(struct crypto_skcipher *tfm, const u8 *key, |
173 | unsigned int keylen) | |
4ea1277d | 174 | { |
4bd96924 | 175 | struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
4ea1277d JG |
176 | int err; |
177 | ||
4bd96924 | 178 | err = xts_verify_key(tfm, key, keylen); |
28856a9e SM |
179 | if (err) |
180 | return err; | |
4ea1277d JG |
181 | |
182 | /* first half of xts-key is for crypt */ | |
674f368a | 183 | err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2); |
4ea1277d JG |
184 | if (err) |
185 | return err; | |
186 | ||
187 | /* second half of xts-key is for tweak */ | |
674f368a | 188 | return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); |
4ea1277d JG |
189 | } |
190 | ||
4bd96924 | 191 | static int xts_encrypt(struct skcipher_request *req) |
4ea1277d | 192 | { |
4bd96924 EB |
193 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
194 | struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
4ea1277d | 195 | |
9c1e8836 | 196 | return glue_xts_req_128bit(&cast6_enc_xts, req, __cast6_encrypt, |
8ce5fac2 | 197 | &ctx->tweak_ctx, &ctx->crypt_ctx, false); |
4ea1277d JG |
198 | } |
199 | ||
4bd96924 | 200 | static int xts_decrypt(struct skcipher_request *req) |
4ea1277d | 201 | { |
4bd96924 EB |
202 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
203 | struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); | |
4ea1277d | 204 | |
9c1e8836 | 205 | return glue_xts_req_128bit(&cast6_dec_xts, req, __cast6_encrypt, |
8ce5fac2 | 206 | &ctx->tweak_ctx, &ctx->crypt_ctx, true); |
4ea1277d JG |
207 | } |
208 | ||
4bd96924 EB |
209 | static struct skcipher_alg cast6_algs[] = { |
210 | { | |
211 | .base.cra_name = "__ecb(cast6)", | |
212 | .base.cra_driver_name = "__ecb-cast6-avx", | |
213 | .base.cra_priority = 200, | |
214 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
215 | .base.cra_blocksize = CAST6_BLOCK_SIZE, | |
216 | .base.cra_ctxsize = sizeof(struct cast6_ctx), | |
217 | .base.cra_module = THIS_MODULE, | |
218 | .min_keysize = CAST6_MIN_KEY_SIZE, | |
219 | .max_keysize = CAST6_MAX_KEY_SIZE, | |
220 | .setkey = cast6_setkey_skcipher, | |
221 | .encrypt = ecb_encrypt, | |
222 | .decrypt = ecb_decrypt, | |
223 | }, { | |
224 | .base.cra_name = "__cbc(cast6)", | |
225 | .base.cra_driver_name = "__cbc-cast6-avx", | |
226 | .base.cra_priority = 200, | |
227 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
228 | .base.cra_blocksize = CAST6_BLOCK_SIZE, | |
229 | .base.cra_ctxsize = sizeof(struct cast6_ctx), | |
230 | .base.cra_module = THIS_MODULE, | |
231 | .min_keysize = CAST6_MIN_KEY_SIZE, | |
232 | .max_keysize = CAST6_MAX_KEY_SIZE, | |
233 | .ivsize = CAST6_BLOCK_SIZE, | |
234 | .setkey = cast6_setkey_skcipher, | |
235 | .encrypt = cbc_encrypt, | |
236 | .decrypt = cbc_decrypt, | |
237 | }, { | |
238 | .base.cra_name = "__ctr(cast6)", | |
239 | .base.cra_driver_name = "__ctr-cast6-avx", | |
240 | .base.cra_priority = 200, | |
241 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
242 | .base.cra_blocksize = 1, | |
243 | .base.cra_ctxsize = sizeof(struct cast6_ctx), | |
244 | .base.cra_module = THIS_MODULE, | |
245 | .min_keysize = CAST6_MIN_KEY_SIZE, | |
246 | .max_keysize = CAST6_MAX_KEY_SIZE, | |
247 | .ivsize = CAST6_BLOCK_SIZE, | |
248 | .chunksize = CAST6_BLOCK_SIZE, | |
249 | .setkey = cast6_setkey_skcipher, | |
250 | .encrypt = ctr_crypt, | |
251 | .decrypt = ctr_crypt, | |
252 | }, { | |
253 | .base.cra_name = "__xts(cast6)", | |
254 | .base.cra_driver_name = "__xts-cast6-avx", | |
255 | .base.cra_priority = 200, | |
256 | .base.cra_flags = CRYPTO_ALG_INTERNAL, | |
257 | .base.cra_blocksize = CAST6_BLOCK_SIZE, | |
258 | .base.cra_ctxsize = sizeof(struct cast6_xts_ctx), | |
259 | .base.cra_module = THIS_MODULE, | |
260 | .min_keysize = 2 * CAST6_MIN_KEY_SIZE, | |
261 | .max_keysize = 2 * CAST6_MAX_KEY_SIZE, | |
262 | .ivsize = CAST6_BLOCK_SIZE, | |
263 | .setkey = xts_cast6_setkey, | |
264 | .encrypt = xts_encrypt, | |
265 | .decrypt = xts_decrypt, | |
4ea1277d | 266 | }, |
4bd96924 EB |
267 | }; |
268 | ||
269 | static struct simd_skcipher_alg *cast6_simd_algs[ARRAY_SIZE(cast6_algs)]; | |
4ea1277d JG |
270 | |
271 | static int __init cast6_init(void) | |
272 | { | |
1debf7db | 273 | const char *feature_name; |
4ea1277d | 274 | |
d91cab78 DH |
275 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
276 | &feature_name)) { | |
1debf7db | 277 | pr_info("CPU feature '%s' is not supported.\n", feature_name); |
4ea1277d JG |
278 | return -ENODEV; |
279 | } | |
280 | ||
4bd96924 EB |
281 | return simd_register_skciphers_compat(cast6_algs, |
282 | ARRAY_SIZE(cast6_algs), | |
283 | cast6_simd_algs); | |
4ea1277d JG |
284 | } |
285 | ||
286 | static void __exit cast6_exit(void) | |
287 | { | |
4bd96924 EB |
288 | simd_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs), |
289 | cast6_simd_algs); | |
4ea1277d JG |
290 | } |
291 | ||
292 | module_init(cast6_init); | |
293 | module_exit(cast6_exit); | |
294 | ||
295 | MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized"); | |
296 | MODULE_LICENSE("GPL"); | |
5d26a105 | 297 | MODULE_ALIAS_CRYPTO("cast6"); |