1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Shared glue code for 128bit block ciphers
5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
7 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
9 * CTR part based on code (crypto/ctr.c) by:
10 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
13 #include <linux/module.h>
14 #include <crypto/b128ops.h>
15 #include <crypto/gf128mul.h>
16 #include <crypto/internal/skcipher.h>
17 #include <crypto/scatterwalk.h>
18 #include <crypto/xts.h>
19 #include <asm/crypto/glue_helper.h>
21 int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
22 struct skcipher_request *req)
24 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
25 const unsigned int bsize = 128 / 8;
26 struct skcipher_walk walk;
27 bool fpu_enabled = false;
31 err = skcipher_walk_virt(&walk, req, false);
33 while ((nbytes = walk.nbytes)) {
34 const u8 *src = walk.src.virt.addr;
35 u8 *dst = walk.dst.virt.addr;
36 unsigned int func_bytes;
39 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
40 &walk, fpu_enabled, nbytes);
41 for (i = 0; i < gctx->num_funcs; i++) {
42 func_bytes = bsize * gctx->funcs[i].num_blocks;
44 if (nbytes < func_bytes)
47 /* Process multi-block batch */
49 gctx->funcs[i].fn_u.ecb(ctx, dst, src);
53 } while (nbytes >= func_bytes);
58 err = skcipher_walk_done(&walk, nbytes);
61 glue_fpu_end(fpu_enabled);
64 EXPORT_SYMBOL_GPL(glue_ecb_req_128bit);
66 int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
67 struct skcipher_request *req)
69 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
70 const unsigned int bsize = 128 / 8;
71 struct skcipher_walk walk;
75 err = skcipher_walk_virt(&walk, req, false);
77 while ((nbytes = walk.nbytes)) {
78 const u128 *src = (u128 *)walk.src.virt.addr;
79 u128 *dst = (u128 *)walk.dst.virt.addr;
80 u128 *iv = (u128 *)walk.iv;
83 u128_xor(dst, src, iv);
84 fn(ctx, (u8 *)dst, (u8 *)dst);
89 } while (nbytes >= bsize);
91 *(u128 *)walk.iv = *iv;
92 err = skcipher_walk_done(&walk, nbytes);
96 EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit);
98 int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
99 struct skcipher_request *req)
101 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
102 const unsigned int bsize = 128 / 8;
103 struct skcipher_walk walk;
104 bool fpu_enabled = false;
108 err = skcipher_walk_virt(&walk, req, false);
110 while ((nbytes = walk.nbytes)) {
111 const u128 *src = walk.src.virt.addr;
112 u128 *dst = walk.dst.virt.addr;
113 unsigned int func_bytes, num_blocks;
117 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
118 &walk, fpu_enabled, nbytes);
119 /* Start of the last block. */
120 src += nbytes / bsize - 1;
121 dst += nbytes / bsize - 1;
125 for (i = 0; i < gctx->num_funcs; i++) {
126 num_blocks = gctx->funcs[i].num_blocks;
127 func_bytes = bsize * num_blocks;
129 if (nbytes < func_bytes)
132 /* Process multi-block batch */
134 src -= num_blocks - 1;
135 dst -= num_blocks - 1;
137 gctx->funcs[i].fn_u.cbc(ctx, dst, src);
139 nbytes -= func_bytes;
143 u128_xor(dst, dst, --src);
145 } while (nbytes >= func_bytes);
148 u128_xor(dst, dst, (u128 *)walk.iv);
149 *(u128 *)walk.iv = last_iv;
150 err = skcipher_walk_done(&walk, nbytes);
153 glue_fpu_end(fpu_enabled);
156 EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit);
158 int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
159 struct skcipher_request *req)
161 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
162 const unsigned int bsize = 128 / 8;
163 struct skcipher_walk walk;
164 bool fpu_enabled = false;
168 err = skcipher_walk_virt(&walk, req, false);
170 while ((nbytes = walk.nbytes) >= bsize) {
171 const u128 *src = walk.src.virt.addr;
172 u128 *dst = walk.dst.virt.addr;
173 unsigned int func_bytes, num_blocks;
177 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
178 &walk, fpu_enabled, nbytes);
180 be128_to_le128(&ctrblk, (be128 *)walk.iv);
182 for (i = 0; i < gctx->num_funcs; i++) {
183 num_blocks = gctx->funcs[i].num_blocks;
184 func_bytes = bsize * num_blocks;
186 if (nbytes < func_bytes)
189 /* Process multi-block batch */
191 gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
194 nbytes -= func_bytes;
195 } while (nbytes >= func_bytes);
201 le128_to_be128((be128 *)walk.iv, &ctrblk);
202 err = skcipher_walk_done(&walk, nbytes);
205 glue_fpu_end(fpu_enabled);
211 be128_to_le128(&ctrblk, (be128 *)walk.iv);
212 memcpy(&tmp, walk.src.virt.addr, nbytes);
213 gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, &tmp, &tmp,
215 memcpy(walk.dst.virt.addr, &tmp, nbytes);
216 le128_to_be128((be128 *)walk.iv, &ctrblk);
218 err = skcipher_walk_done(&walk, 0);
223 EXPORT_SYMBOL_GPL(glue_ctr_req_128bit);
225 static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
227 struct skcipher_walk *walk)
229 const unsigned int bsize = 128 / 8;
230 unsigned int nbytes = walk->nbytes;
231 u128 *src = walk->src.virt.addr;
232 u128 *dst = walk->dst.virt.addr;
233 unsigned int num_blocks, func_bytes;
236 /* Process multi-block batch */
237 for (i = 0; i < gctx->num_funcs; i++) {
238 num_blocks = gctx->funcs[i].num_blocks;
239 func_bytes = bsize * num_blocks;
241 if (nbytes >= func_bytes) {
243 gctx->funcs[i].fn_u.xts(ctx, dst, src,
248 nbytes -= func_bytes;
249 } while (nbytes >= func_bytes);
260 int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
261 struct skcipher_request *req,
262 common_glue_func_t tweak_fn, void *tweak_ctx,
263 void *crypt_ctx, bool decrypt)
265 const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);
266 const unsigned int bsize = 128 / 8;
267 struct skcipher_request subreq;
268 struct skcipher_walk walk;
269 bool fpu_enabled = false;
270 unsigned int nbytes, tail;
273 if (req->cryptlen < XTS_BLOCK_SIZE)
277 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
279 tail = req->cryptlen % XTS_BLOCK_SIZE + XTS_BLOCK_SIZE;
281 skcipher_request_set_tfm(&subreq, tfm);
282 skcipher_request_set_callback(&subreq,
283 crypto_skcipher_get_flags(tfm),
285 skcipher_request_set_crypt(&subreq, req->src, req->dst,
286 req->cryptlen - tail, req->iv);
290 err = skcipher_walk_virt(&walk, req, false);
291 nbytes = walk.nbytes;
295 /* set minimum length to bsize, for tweak_fn */
296 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
298 nbytes < bsize ? bsize : nbytes);
300 /* calculate first value of T */
301 tweak_fn(tweak_ctx, walk.iv, walk.iv);
304 nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk);
306 err = skcipher_walk_done(&walk, nbytes);
307 nbytes = walk.nbytes;
311 u8 *next_tweak, *final_tweak = req->iv;
312 struct scatterlist *src, *dst;
313 struct scatterlist s[2], d[2];
316 dst = src = scatterwalk_ffwd(s, req->src, req->cryptlen);
317 if (req->dst != req->src)
318 dst = scatterwalk_ffwd(d, req->dst, req->cryptlen);
321 next_tweak = memcpy(b, req->iv, XTS_BLOCK_SIZE);
322 gf128mul_x_ble(b, b);
324 next_tweak = req->iv;
327 skcipher_request_set_crypt(&subreq, src, dst, XTS_BLOCK_SIZE,
330 err = skcipher_walk_virt(&walk, req, false) ?:
331 skcipher_walk_done(&walk,
332 __glue_xts_req_128bit(gctx, crypt_ctx, &walk));
336 scatterwalk_map_and_copy(b, dst, 0, XTS_BLOCK_SIZE, 0);
337 memcpy(b + 1, b, tail - XTS_BLOCK_SIZE);
338 scatterwalk_map_and_copy(b, src, XTS_BLOCK_SIZE,
339 tail - XTS_BLOCK_SIZE, 0);
340 scatterwalk_map_and_copy(b, dst, 0, tail, 1);
342 skcipher_request_set_crypt(&subreq, dst, dst, XTS_BLOCK_SIZE,
345 err = skcipher_walk_virt(&walk, req, false) ?:
346 skcipher_walk_done(&walk,
347 __glue_xts_req_128bit(gctx, crypt_ctx, &walk));
351 glue_fpu_end(fpu_enabled);
355 EXPORT_SYMBOL_GPL(glue_xts_req_128bit);
357 void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
358 common_glue_func_t fn)
362 /* generate next IV */
363 gf128mul_x_ble(iv, &ivblk);
366 u128_xor(dst, src, (u128 *)&ivblk);
368 /* PP <- D(Key2,CC) */
369 fn(ctx, (u8 *)dst, (u8 *)dst);
372 u128_xor(dst, dst, (u128 *)&ivblk);
374 EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
376 MODULE_LICENSE("GPL");